Author: hfnukal Date: 2011-09-09 06:06:32 -0400 (Fri, 09 Sep 2011) New Revision: 7343 Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-artifact-conflicts.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-gav-conflicts.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-poms.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-tags.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-artifacts epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-snapshots epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-file epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-maven epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-poms.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/koji-maven-migrate epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/list-tag-deps.py epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-load-build-dependencies epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-test-spec-fragment Modified: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/build.xml epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/gatein-portal/pom.xml epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/mead.parent/pom.xml epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/pom.xml Log: JBEPP-1091 mead build import dependencies Modified: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/build.xml =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/build.xml 2011-09-08 16:34:31 UTC (rev 7342) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/build.xml 2011-09-09 10:06:32 UTC (rev 7343) @@ -16,7 +16,7 @@ </ant> execute - mvn package -P mead-all [-Dbrew.exec [-Dbrew.not.wait]] [-Dbrew.not.scratch] + mvn package -P mead-all [-Dbrew.exec [-Dbrew.not.wait]] [-Dbrew.not.scratch] [-Dimport.dependencies] brew.exec - will execute Brew bild. Brew binary must be installed om build mashine. brew.no.wait - If not declared, build waits until task is done and synchronized @@ -96,7 +96,7 @@ </loadfile> <echo>Patch Revision: "${svnpatch.revision}"</echo> - <property name="mead.patch">--patches '${svn.patch.base}?${mead.patch.path}#${svnpatch.revision}'</property> + <property name="mead.patch">--patches 'svn+${svn.patch.base}?${mead.patch.path}#${svnpatch.revision}'</property> </target> <target name="patch-cond-else" depends="patch-cond" unless="patch-cond-is-true"> @@ -189,12 +189,15 @@ <antcall target="brew-build-wait"/> </target> + + <!-- SCRATCH --> <target name="brew-build-scratch-no" if="brew.not.scratch"> <property name="brew.build.scratch"> </property> </target> <target name="brew-build-scratch-yes" unless="brew.not.scratch"> <property name="brew.build.scratch">--scratch</property> </target> + <target name="brew-build-wait" unless="brew.not.wait"> <echo>Task ID: "${mead.task.id}"</echo> @@ -238,4 +241,61 @@ <echo>EXEC: brew wait-repo --target jboss-epp-5 --build ${mead.build.id}</echo> </target> + + <!-- Import dependencies --> + <target name="mead-import-dependencies"> + <property name="svn.path">${ant.build.dir}/import-svn/</property> + + <echo>Checkout from SVN ${mead.svn.base}/${mead.svn.repo}/${mead.svn.path}</echo> + <!--<exec executable="svn" output="${ant.build.dir}/svn.log" failonerror="true" failifexecutionfails="">--> + <!--<arg value="co"/>--> + <!--<arg value="${mead.svn.base}/${mead.svn.repo}/${mead.svn.path}" />--> + <!--<arg line="${svn.path}" />--> + <!--</exec>--> + + <echo>Resolve dependencies</echo> + <!--<exec executable="mvn" dir="${svn.path}" output="${ant.build.dir}/mvn.resolve.log" failonerror="true" failifexecutionfails="">--> + <!--<arg value="dependency:resolve"/>--> + <!--</exec>--> + + <echo>Dependency list</echo> + <!--<exec executable="mvn" dir="${svn.path}" output="${ant.build.dir}/mvn.dependency.list" failonerror="true" failifexecutionfails="">--> + <!--<arg value="dependency:list"/>--> + <!--</exec>--> + + <echo>Parse file</echo> + <loadfile property="mvn.dependency.list" srcFile="${ant.build.dir}/mvn.dependency.list"> + <filterchain> + <linecontainsregexp> + <regexp pattern=":[a-zA-Z0-9\.]*:[a-zA-Z0-9\.]*:[a-zA-Z0-9\.]*" /> + </linecontainsregexp> + <!--<deletecharacters chars="\[INFO\]"/>--> + <replaceregex pattern="\[INFO\]" replace=""/> + <replaceregex pattern=":[a-z]*$" replace=""/> + <trim/> + <sortfilter/> + <uniqfilter/> + <linecontains negate="true"> + <contains value="org.gatein"/> + </linecontains> + <linecontains negate="true"> + <contains value="org.exoplatform"/> + </linecontains> + <linecontains negate="true"> + <contains value="org.picketlink"/> + </linecontains> + </filterchain> + </loadfile> + + <echo file="${ant.build.dir}/mvn.dependency.list.filtered">${mvn.dependency.list}</echo> + + <exec dir="../src/brewscripts/" executable="pwd"></exec> + + <!--Execute import --> + <exec executable="bash" dir="../src/brewscripts/" output="${ant.build.dir}/import.log" failonerror="true" failifexecutionfails=""> + <arg value="import-file"/> + <arg value="${ant.build.dir}/mvn.dependency.list.filtered"/> + </exec> + </target> + </project> \ No newline at end of file Modified: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/gatein-portal/pom.xml =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/gatein-portal/pom.xml 2011-09-08 16:34:31 UTC (rev 7342) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/gatein-portal/pom.xml 2011-09-09 10:06:32 UTC (rev 7343) @@ -20,9 +20,9 @@ <mead.svn.repo>${svn.gatein.repo}</mead.svn.repo> <mead.svn.path>${gatein-portal.svnpath}</mead.svn.path> - <mead.build.profiles>release pkg-jbossas</mead.build.profiles> - <mead.build.properties>exo.projects.directory.dependencies=/maven/build/test-containers</mead.build.properties> - <mead.patch.path>patches/org.exoplatform.portal-exo.portal.parent/tags/epp-5.1.0-ER03</mead.patch.path> + <mead.build.profiles>release epp-distribution</mead.build.profiles> + <mead.build.properties></mead.build.properties> + <mead.patch.path></mead.patch.path> </properties> </project> \ No newline at end of file Modified: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/mead.parent/pom.xml =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/mead.parent/pom.xml 2011-09-08 16:34:31 UTC (rev 7342) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/mead.parent/pom.xml 2011-09-09 10:06:32 UTC (rev 7343) @@ -1,14 +1,14 @@ <?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.exoplatform.portal.mead</groupId> <version>5.2.0-epp-DEV04-SNAPSHOT</version> <artifactId>mead-tools</artifactId> - <relativePath>../pom.xml</relativePath> + <relativePath>../pom.xml</relativePath> </parent> <artifactId>mead.parent</artifactId> @@ -20,31 +20,68 @@ <ant.build.file>../build.xml</ant.build.file> <ant.target>mead-build</ant.target> </properties> - - <build> - <plugins> - <!-- Generate MEAD commands --> - <plugin> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.6</version> - <executions> - <execution> - <id>mead-build-${project.artifactId}</id> - <phase>compile</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <ant antfile="${ant.build.file}" inheritRefs="true" inheritAll="true"> - <target name="${ant.target}"/> - </ant> - </target> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> + + <profiles> + <profile> + <id>mead-build</id> + <build> + <plugins> + <!-- Generate MEAD commands --> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.6</version> + <executions> + <execution> + <id>mead-build-${project.artifactId}</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <ant antfile="${ant.build.file}" inheritRefs="true" inheritAll="true"> + <target name="${ant.target}"/> + </ant> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + + <profile> + <id>mead-import</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <build> + <plugins> + <!-- Generate MEAD commands --> + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.6</version> + <executions> + <execution> + <id>mead-import-${project.artifactId}</id> + <phase>compile</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <ant antfile="${ant.build.file}" inheritRefs="true" inheritAll="true"> + <target name="mead-import-dependencies"/> + </ant> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> \ No newline at end of file Modified: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/pom.xml =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/pom.xml 2011-09-08 16:34:31 UTC (rev 7342) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/pom.xml 2011-09-09 10:06:32 UTC (rev 7343) @@ -68,7 +68,7 @@ <!-- # eXo components --> <svn.exo.repo>exo-jcr</svn.exo.repo> - <exo-parent.svnpath>parent/tags/7</exo-parent.svnpath> + <exo-parent.svnpath>parent/tags/9</exo-parent.svnpath> <exo-doc.svnpath>docs-style/tags/1</exo-doc.svnpath> <exo-junit.svnpath>junit.framework/tags/1.2.1-GA</exo-junit.svnpath> @@ -83,27 +83,29 @@ <profile> <id>mead-all</id> <modules> + <module>exo-parent </module> + <module>exo-doc </module> + <module>exo-junit </module> + + <module>exo-kernel </module> + + <module>exo-core </module> + <module>exo-ws </module> + <module>exo-jcr </module> <module>gatein-parent </module> - <module>gatein-portal </module> - <module>gatein-packager </module> <module>gatein-dep </module> <module>gatein-common </module> <module>gatein-wci </module> <module>gatein-pc </module> <module>gatein-wsrp </module> <module>gatein-mop </module> - <module>gatein-sso </module> <module>gatein-simplecaptcha </module> <module>gatein-shindig </module> + <module>gatein-packager </module> <module>jboss-picketlink-idm </module> + <module>gatein-portal </module> + <module>gatein-sso </module> <module>jboss-portletbridge </module> - <module>exo-parent </module> - <module>exo-doc </module> - <module>exo-junit </module> - <module>exo-kernel </module> - <module>exo-core </module> - <module>exo-ws </module> - <module>exo-jcr </module> </modules> </profile> Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-artifact-conflicts.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-artifact-conflicts.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-artifact-conflicts.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,31 @@ +#!/usr/bin/python + +import sys +import koji + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +tag = sys.argv[1] + +builds = session.listTagged(tag, inherit=True) +print 'Processing', len(builds), 'builds...' + +all_builds = {} +all_archives = {} + +for build in builds: + all_builds[build['id']] = build + archives = session.listArchives(buildID=build['id'], type='maven') + for archive in archives: + path = koji.pathinfo.mavenrepo(build, archive) + '/' + archive['filename'] + archive_list = all_archives.setdefault(path, []) + if archive_list: + prev_archive = archive_list[len(archive_list) - 1] + prev_build = all_builds[prev_archive['build_id']] + if build['name'] == prev_build['name'] and build['version'] == prev_build['version']: + # only the latest release of a package is included, so this isn't actually a conflict + continue + print path, 'provided by both' + print ' ', koji.buildLabel(prev_build), 'and' + print ' ', koji.buildLabel(build) + archive_list.append(archive) Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-artifact-conflicts.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-gav-conflicts.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-gav-conflicts.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-gav-conflicts.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,31 @@ +#!/usr/bin/python + +import sys +import koji + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +tag = sys.argv[1] + +builds = session.listTagged(tag, inherit=True) +print 'Processing', len(builds), 'builds...' + +all_builds = dict([(build['id'], build) for build in builds]) +all_gavs = {} + +for build in builds: + archives = session.listArchives(buildID=build['id'], type='maven') + for archive in archives: + gav = koji.mavenLabel(archive) + build_id = all_gavs.get(gav) + if build_id: + if build_id != archive['build_id']: + prev_build = all_builds[build_id] + curr_build = all_builds[archive['build_id']] + if prev_build['name'] != curr_build['name']: + print 'build %s and %s are both putting artifacts in %s' % \ + (koji.buildLabel(prev_build), + koji.buildLabel(curr_build), + gav) + else: + all_gavs[gav] = archive['build_id'] Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-gav-conflicts.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-poms.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-poms.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-poms.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,26 @@ +#!/usr/bin/python + +import koji + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') +session.multicall = True + +poms = file('/tmp/poms.txt').readlines() + +nvrs = [] + +for pom in poms: + pom = pom.split('/') + nvrs.append(pom) + terms = '.'.join(pom[:-3]) + '-' + pom[-3] + '-' + pom[-2].replace('-', '_') + '-*' + print terms + session.search(terms, 'build', 'glob', queryOpts={'countOnly': True}) + +results = session.multiCall() + +for pom, [result] in zip(nvrs, results): + if result == 0: + print '.'.join(pom[:-3]) + ':' + pom[-3] + ':jar:' + pom[-2], 'has no versions imported' + else: + # print '.'.join(pom[:-3]) + ':' + pom[-3] + ':jar:' + pom[-2], 'has', result, 'versions imported' + pass Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-poms.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-tags.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-tags.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-tags.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,17 @@ +#!/usr/bin/python + +import koji + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +builds = session.listBuilds(mavenOnly=True) + +for build in builds: + if build['state'] != koji.BUILD_STATES['COMPLETE']: + continue + tags = session.listTags(build=build['build_id']) + for tag in tags: + if tag['name'] == 'mead-import-maven-all': + break + else: + print build['nvr'], 'is not in mead-import-maven-all' Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/check-tags.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-artifacts =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-artifacts (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-artifacts 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,60 @@ +#!/usr/bin/python + +import sys +import re +import urlgrabber +import urlgrabber.progress +import koji +import optparse + +KOJI_HUB = 'http://brewhub.devel.redhat.com/brewhub' + +MAVEN_BASE = 'http://repo1.maven.org/maven2' +JBOSS_BASE = 'http://repository.jboss.org/nexus/content/groups/public-jboss' +OLD_JBOSS_BASE = 'http://repository.jboss.org/maven2' + +LINK_RE = re.compile(r'href="([^"]+)"') + +parser = optparse.OptionParser(usage='%prog groupId:artifactId:type:version') +parser.add_option('--repo', dest='repos', help='Additional repo to check', action='append', + default=[MAVEN_BASE, JBOSS_BASE, OLD_JBOSS_BASE]) +options, args = parser.parse_args() + +SUPPORTED_EXTS = [] +session = koji.ClientSession(KOJI_HUB) +for archive_type in session.getArchiveTypes(): + SUPPORTED_EXTS.extend(archive_type['extensions'].split()) + +if len(args) == 1: + tokens = args[0].split(':') + group_id, artifact_id, dummy, version = tokens +elif len(args) == 3: + group_id, artifact_id, version = args +else: + parser.error('You must specify a groupId:artifactId:type:version tuple, or pass groupId, artifactId, and version as parameters') + +path = '%s/%s/%s' % (group_id.replace('.', '/'), artifact_id, version) +for base in options.repos: + baseurl = '%s/%s' % (base, path) + + try: + contents = urlgrabber.urlread(baseurl) + print 'Downloading from %s' % baseurl + except: + print 'Could not read %s, continuing...' % baseurl + continue + + links = LINK_RE.findall(contents) + for link in links: + if link.endswith('maven-metadata.xml'): + continue + if [ext for ext in SUPPORTED_EXTS if link.endswith(ext)]: + if '://' in link: + url = link + else: + url = '%s/%s' % (baseurl, link) + urlgrabber.urlgrab(url, progress_obj=urlgrabber.progress.TextMeter()) + break +else: + print 'Could not download artifacts for %s:%s:%s' % (group_id, artifact_id, version) + sys.exit(1) Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-artifacts ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-snapshots =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-snapshots (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-snapshots 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,61 @@ +#!/usr/bin/python + +import os.path +import sys +import re +import urlgrabber +import urlgrabber.progress + +JBOSS_BASE = 'http://snapshots.jboss.org/maven2' +JBOSS_NEXUS = 'https://repository.jboss.org/nexus/content/groups/developer' +SONATYPE_BASE = 'http://oss.sonatype.org/content/repositories/snapshots' + +LINK_RE = re.compile(r'href="([^"]+)"') +SNAPSHOT_RE = re.compile(r'\d{8}\.\d{6}-\d+') + +if len(sys.argv) < 2: + print 'You must specify an artifact to download' + sys.exit(1) +elif len(sys.argv) > 2: + print 'You may only specify one artifact to download/import at a time' + sys.exit(1) + +arg = sys.argv[1] + +artifacts = {} + +group_id, artifact_id, dummy, version = arg.split(':') +path = '%s/%s/%s' % (group_id.replace('.', '/'), artifact_id, version) +for base in (JBOSS_BASE, JBOSS_NEXUS, SONATYPE_BASE): + baseurl = '%s/%s' % (base, path) + + try: + contents = urlgrabber.urlread(baseurl) + print 'Downloading from %s' % baseurl + except: + print 'Could not read %s, continuing...' % baseurl + continue + + links = LINK_RE.findall(contents) + for link in links: + if link.endswith('.pom') or link.endswith('.jar') or link.endswith('.war') or \ + link.endswith('.ear') or link.endswith('.sar') or link.endswith('.rar') or \ + link.endswith('.zip'): + url = '%s/%s' % (baseurl, link) + canon_base = SNAPSHOT_RE.sub('SNAPSHOT', link) + if canon_base in artifacts: + prev_link = artifacts[canon_base] + prev_base = os.path.basename(prev_link) + if os.path.basename(url) > prev_base: + artifacts[canon_base] = url + else: + artifacts[canon_base] = url + +if artifacts: + for canon_base, url in artifacts.items(): + urlgrabber.urlgrab(url, progress_obj=urlgrabber.progress.TextMeter()) + print 'Renaming %s to %s' % (os.path.basename(url), canon_base) + os.rename(os.path.basename(url), canon_base) +else: + print 'Could not download artifacts for %s' % arg + sys.exit(1) Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/get-maven-snapshots ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,14 @@ +#!/bin/sh +WORKDIR=`pwd` +TMPDIR=../../target/import-tmp +for i in $* +do + #will import all deps passed as parameters + echo "Importing $i" + mkdir $TMPDIR + cd $TMPDIR + $WORKDIR/get-maven-artifacts $i + $WORKDIR/import-maven * + cd $WORKDIR + rm -Rf $TMPDIR +done Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-file =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-file (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-file 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,2 @@ +#!/bin/sh +cat $1 | xargs ./import \ No newline at end of file Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-file ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-maven =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-maven (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-maven 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,83 @@ +#!/usr/bin/python + +import koji +import optparse +import os +import sys + +KOJI_HUB = 'http://brewhub.devel.redhat.com/brewhub' + +def run(cmd, fail=True): + print cmd + ret = os.system(cmd) + if ret != 0: + print 'Error running command' + if fail: + sys.exit(1) + +session = koji.ClientSession(KOJI_HUB) + +parser = optparse.OptionParser(usage='%prog [options] artifact [artifact...]') +parser.add_option('--tag', default='mead-import-maven-all', help='Tag to import into') +parser.add_option('--owner', default=os.environ['USER'], help='Owner of the package') +parser.add_option('--release', default='1', help='Release of the build to create') +options, args = parser.parse_args() + +if not options.tag: + parser.error('You must specify the tag to import to with --tag') + assert False + +pom = None +for arg in args[:]: + if arg.endswith('.pom'): + if not pom: + pom = arg + else: + sys.exit('Multiple .pom files specified. You may only import one Maven build at a time.') + +if not pom: + sys.exit('No .pom file specified') + +pom_info = koji.parse_pom(pom) +maven_info = koji.pom_to_maven_info(pom_info) +build_info = koji.maven_info_to_nvr(maven_info) +build_info['release'] = options.release + +brew_build = session.getBuild(build_info) +if brew_build: + if brew_build['task_id']: + print '%s was built in MEAD, cannot import additional artifacts to it' % koji.buildLabel(brew_build) + sys.exit(0) + + brew_artifacts = session.listArchives(buildID=brew_build['id'], type='maven') + if brew_artifacts: + for arg in args[:]: + if os.path.basename(arg) in [f['filename'] for f in brew_artifacts]: + print os.path.basename(arg), 'has already been uploaded' + args.remove(arg) + +if args: + cmd = 'brew import-archive --create-build --type maven --type-info %s %s %s' % (pom, koji.buildLabel(build_info), ' '.join(args)) + run(cmd) +else: + print 'All artifacts for %s have already been imported, skipping...' % koji.buildLabel(build_info) + +tag = session.getTag(options.tag, strict=True) +package = session.getPackage(build_info['name']) +tag_pkgs = [] +if package: + tag_pkgs = session.listPackages(tagID=tag['id'], pkgID=package['id'], inherited=True) + +if tag_pkgs: + print '%s already exists in %s, not adding...' % (package['name'], tag['name']) +else: + cmd = 'brew add-pkg --owner %s %s %s' % (options.owner, options.tag, build_info['name']) + run(cmd) + +if options.tag in [t['name'] for t in session.listTags(build=koji.buildLabel(build_info))]: + print '%s already tagged with %s, skipping...' % (koji.buildLabel(build_info), options.tag) +else: + cmd = 'brew tag-pkg --nowait %s %s' % (options.tag, koji.buildLabel(build_info)) + run(cmd) + +print 'Imported and tagged %s' % koji.buildLabel(build_info) Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-maven ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-poms.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-poms.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-poms.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,35 @@ +#!/usr/bin/python + +import koji +import os +import time + +def run(cmd): + print cmd + return os.system(cmd) + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +poms = file('/tmp/missing-poms.txt').readlines() + +for pom in poms: + pom = pom.split()[0] + parts = pom.split(':') + nvr = parts[0] + '-' + parts[1] + '-' + parts[3].replace('-', '_') + '-1' + build = session.getBuild(nvr) + if not build: + print nvr, 'does not exist in Brew' + else: + print nvr, 'has already been imported' + continue + + rv = run('/mnt/redhat/devel/mikeb/mead/scripts/get-maven-artifacts ' + pom) + if rv or not os.listdir(os.getcwd()): + print 'Error getting artifacts for', pom + continue + + rv = run('/mnt/redhat/devel/mikeb/mead/scripts/import-maven --tag maven-all *') + if rv: + print 'Error importing', pom + + run('rm *') Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/import-poms.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/koji-maven-migrate =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/koji-maven-migrate (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/koji-maven-migrate 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,65 @@ +#!/usr/bin/python + +# Script to migrate a Koji installation using Maven support from the +# old filesystem layout to the new layout. + +import koji +import sys +import optparse +import os +import os.path +import shutil + +parser = optparse.OptionParser(usage='%prog [options] HUB_URL') +parser.add_option('-d', '--topdir', action='store', default='/mnt/koji') +parser.add_option('--commit', action='store_true', help='Really do it') +parser.add_option('--debug', action='store_true') + +options, args = parser.parse_args() +if not args: + parser.error('You must specify a hub URL') + +session = koji.ClientSession(args[0]) +pi = koji.PathInfo(options.topdir) + +builds = {} + +archives = session.listArchives(type='maven', queryOpts={'order': 'id'}) +for archive in archives: + info = builds.get(archive['build_id']) + if info: + build, maven_build = info + else: + build = session.getBuild(archive['build_id'], strict=True) + maven_build = session.getMavenBuild(archive['build_id'], strict=True) + builds[archive['build_id']] = (build, maven_build) + + if build['state'] != koji.BUILD_STATES['COMPLETE']: + if options.debug: + print 'Skipping %s build: %s' % (koji.BUILD_STATES[build['state']], koji.buildLabel(build)) + continue + src = os.path.join(options.topdir, 'maven2', + maven_build['group_id'].replace('.', '/'), + maven_build['artifact_id'], + maven_build['version'], + build['release'], + archive['filename']) + + dest = os.path.join(pi.build(build), + 'maven', + archive['group_id'].replace('.', '/'), + archive['artifact_id'], + archive['version'], + archive['filename']) + + for ext in ['', '.md5', '.sha1']: + if os.path.exists(dest + ext): + if options.debug: + print 'File has already been moved:', dest + ext + elif os.path.exists(src + ext): + print 'Moving file from %s to %s' % (src + ext, dest + ext) + if options.commit: + koji.ensuredir(os.path.dirname(dest)) + shutil.move(src + ext, dest + ext) + else: + print 'Missing file: %s' % src + ext Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/koji-maven-migrate ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/list-tag-deps.py =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/list-tag-deps.py (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/list-tag-deps.py 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,47 @@ +#!/usr/bin/python + +import koji +import sys +import os + +if os.isatty(sys.stdout.fileno()): + RED = '\x1B[91;1m' + YELLOW = '\x1B[93;1m' + GREEN = '\x1B[92;1m' + RESET = '\x1B[0m' +else: + RED = YELLOW = GREEN = RESET = '' + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +tag_name = sys.argv[1] +fobj = file(sys.argv[2]) + +for line in fobj: + build_id = 0 + tokens = line.strip().split('/') + version = tokens[-1] + artifact_id = tokens[-2] + group_id = '.'.join(tokens[:-2]) + archives = session.listArchives(type='maven', typeInfo={'group_id': group_id, + 'artifact_id': artifact_id, + 'version': version}) + + for archive in archives: + if archive['build_id'] > build_id: + build_id = archive['build_id'] + + if build_id == 0: + print '%s%s:%s:dummy:%s%s needs to be imported' % (RED, group_id, artifact_id, version, RESET) + continue + + build = session.getBuild(build_id, strict=True) + tags = session.listTags(build=build['id']) + for tag in tags: + if tag['name'] == tag_name: + print '%s%s%s is tagged into %s' % (GREEN, build['nvr'], RESET, tag_name) + break + else: + print '%s%s%s needs to be tagged into %s' % (YELLOW, build['nvr'], RESET, tag_name) + +fobj.close() Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/list-tag-deps.py ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-load-build-dependencies =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-load-build-dependencies (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-load-build-dependencies 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,279 @@ +#!/usr/bin/python + +import os.path +import urlgrabber +import urlgrabber.progress +import koji +import re +import shutil +import sys +import tempfile +import optparse +import traceback + +KOJI_HUB = 'http://brewhub.devel.redhat.com/brewhub' + +LINK_RE = re.compile(r'href="([^"]+)"', re.I) +NORM_RE = re.compile(r'([^:])//+') +METER = urlgrabber.progress.TextMeter() +BAD_ARTIFACTS = ['pom.xml', + 'maven-metadata.xml', + 'maven-plugin-tools-2.0-javadoc.pom', + 'struts-1.2.8-el.pom', + 'maven-reporting-2.0-javadoc.pom', + 'maven-2.0-javadoc.pom', + 'connector-1.5.pom', + 'jbosscache-core-3.0.2.GA.pom', + 'jboss-vfs-2.0.0.CR1.pom', + 'milyn-smooks-javabean-1.1.pom', + 'milyn-commons-1.1.pom', + 'milyn-smooks-core-1.1.pom', + 'jbosscache-core-3.0.0.GA.pom', + 'jbosscache-core-3.2.1.GA.pom', + 'commons-lang-2.0.distribution-zip', + '1.2.12:maven-metadata.xml'] + +def run(cmd, fail=True): + print cmd + ret = os.system(cmd) + if ret != 0: + print 'Error running command' + if fail: + sys.exit(1) + +def check_and_append(opts, url_list, url): + tokens = url.split('/') + if len(tokens) < 3: + print >> sys.stderr, 'Skipping url %s, not enough tokens...' % url + return + if not url_list: + url_list.append(url) + return + last_tokens = url_list[-1].split('/') + if tokens[-3:] == last_tokens[-3:]: + # This url is the same as the previous url. A repeat download, so the + # previous download must have failed. Replace the previous URL with this one. + if url_list[-1] == url: + # exactly the same URL, so we don't need to do anything + pass + else: + if opts.debug: + print >> sys.stderr, 'Replacing: %s\n with: %s' % (url_list[-1], url) + url_list[-1] = url + else: + # New url, append it to the list + url_list.append(url) + +parser = optparse.OptionParser(usage='%prog maven-build-log') +parser.add_option('--urls', action='store_true', + help='The file passed in is a list of urls, one per line, rather than an unprocessed log file') +parser.add_option('--print-only', action='store_true', + help='Print the list of urls and exit') +parser.add_option('--print-missing', action='store_true', + help='Print artifacts that are missing from Brew') +parser.add_option('--check-tag', + help='For artifacts that are in Brew, check that they are in the given tag') +parser.add_option('--exclude', action='append', default=[], + help='groupId-artifactId prefixes to exclude from import, may be specified multiple times') +parser.add_option('--skip', type='int', help='Skip to this number in the list of urls to process') +parser.add_option('-d', '--debug', action='store_true', help='Print verbose error messages') + +opts, args = parser.parse_args() + +urls = [] + +session = koji.ClientSession(KOJI_HUB) + +SUPPORTED_EXTS = [] +for archive_type in session.getArchiveTypes(): + SUPPORTED_EXTS.extend(archive_type['extensions'].split()) + +if len(args) < 1: + parser.error('you must specify a Maven build log') + +for log in args: + fobj = file(log) + for line in fobj: + if opts.urls: + check_and_append(opts, urls, line.strip()) + else: + i = 0 + tokens = line.split() + while i < len(tokens): + if tokens[i] == 'Downloading:': + if (i + 1) < len(tokens): + url = tokens[i + 1] + baseurl = os.path.dirname(url) + baseurl = NORM_RE.sub(r'\1/', baseurl) + check_and_append(opts, urls, baseurl) + i += 1 + else: + print 'Downloading: is last token on the line: %s' % tokens + i += 1 + fobj.close() + +urls = set(urls) + +if opts.print_only: + for url in urls: + print url + sys.exit(0) +else: + print 'Processing %i urls...' % len(urls) + +if opts.check_tag: + taglist = set(opts.check_tag) + for tag in session.getFullInheritance(opts.check_tag): + taglist.add(tag['name']) + +i = 0 +for baseurl in urls: + i += 1 + if opts.skip and i <= opts.skip: + continue + sys.stderr.write('Processing url %i...\r' % i) + + try: + contents = urlgrabber.urlread(baseurl) + # print 'Downloading from %s' % baseurl + except KeyboardInterrupt: + raise + except: + if opts.debug: + print '\nCould not read %s, continuing...' % baseurl + print ' error was: ' + str(sys.exc_info()[1]) + continue + + links = LINK_RE.findall(contents) + pom_found = False + download = False + for link in links: + if link in BAD_ARTIFACTS: + continue + if link.endswith('.pom'): + if pom_found: + print '\nIgnoring directory with multiple .pom files:', baseurl + print " You'll probably need to download and import the proper version of the required artifact manually." + download = False + break + else: + pom_found = True + if '://' in link: + # absolute link + url = link + elif link.startswith('/'): + # absolute link on this host + pieces = baseurl.split('/')[:3] + url = '/'.join(pieces) + link + else: + # relative link + url = '%s/%s' % (baseurl, link) + try: + pom = urlgrabber.urlread(url) + except: + print '\nError loading pom at: %s' % url + traceback.print_exc() + continue + try: + pom_info = koji.parse_pom(contents=pom) + except KeyboardInterrupt: + raise + except: + if opts.debug: + print '\nError reading pom at: %s' % url + print ' error was: ' + str(sys.exc_info()[1]) + # traceback.print_exc() + continue + + maven_info = koji.pom_to_maven_info(pom_info) + maven_label = koji.mavenLabel(maven_info) + if '$' in maven_label: + print '\nBad GAV vector: %s at %s, skipping...' % (maven_label, baseurl) + continue + if [prefix for prefix in opts.exclude if maven_label.startswith(prefix)]: + print '\nSkipping excluded artifacts from %s...' % maven_label + continue + if 'SNAPSHOT' in maven_info['version']: + print '\nCannot handle snspshots from %s, download and import manually...' % maven_label + continue + maven_archives = session.listArchives(type='maven', typeInfo=maven_info, queryOpts={'order': 'id'}) + # print 'Maven archives for %s:' % koji.mavenLabel(maven_info), maven_archives + if not maven_archives: + maven_info['baseurl'] = baseurl + print '\n%(group_id)s:%(artifact_id)s:%(version)s from %(baseurl)s has no versions in Brew, downloading and importing' % maven_info + download = True + else: + for link in links: + link = os.path.basename(link) + if link in BAD_ARTIFACTS: + continue + if [ext for ext in SUPPORTED_EXTS if link.endswith(ext)] and link not in [a['filename'] for a in maven_archives]: + print koji.mavenLabel(maven_info), 'exists but %s is missing, importing...' % link + download = True + if opts.check_tag: + builds = set() + for archive in maven_archives: + builds.add(archive['build_id']) + for build_id in builds: + tags = set([t['name'] for t in session.listTags(build=build_id)]) + overlap = taglist.intersection(tags) + if overlap: + if opts.debug: + build = session.getBuild(build_id) + print '\n%s in available in %s (provides %s:%s:%s)' % (koji.buildLabel(build), ', '.join(overlap), + maven_info['group_id'], maven_info['artifact_id'], + maven_info['version']) + break + else: + # just take the highest numbered build + labels = [] + for build_id in sorted(builds): + labels.append(koji.buildLabel(session.getBuild(build_id))) + print '\n%s:%s:%s is missing from %s, is provided by: %s' % (maven_info['group_id'], maven_info['artifact_id'], + maven_info['version'], + opts.check_tag, + ', '.join(labels)) + + if opts.print_missing: + continue + + do_import = None + tmpdir = None + if download: + tmpdir = tempfile.mkdtemp() + for link in links: + if link.endswith('.md5') or link.endswith('.sha1') or link.endswith('.asc') or \ + link.endswith('/') or \ + link.startswith('?') or link.endswith('-zip') or link.endswith('-tgz') or \ + 'Sonatype-content.css' in link or link.endswith('.audit.json') or \ + link in BAD_ARTIFACTS: + pass + elif [ext for ext in SUPPORTED_EXTS if link.endswith(ext)]: + if '://' in link: + # absolute link + url = link + elif link.startswith('/'): + # absolute link on this host + pieces = baseurl.split('/')[:3] + url = '/'.join(pieces) + link + else: + # relative link + url = '%s/%s' % (baseurl, link) + urlgrabber.urlgrab(url, filename=os.path.join(tmpdir, os.path.basename(link)), progress_obj=METER) + if do_import is None: + do_import = True + else: + print '\nUnknown file type: %s' % link + # do_import = False + if do_import is True: + run('/mnt/redhat/devel/mikeb/mead/scripts/import-maven --tag mead-import-maven-all %s/*' % tmpdir) + elif do_import is None: + pass + else: + print '\nSkipping import of files from %s' % baseurl + + if tmpdir: + shutil.rmtree(tmpdir) + +#done +print Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-load-build-dependencies ___________________________________________________________________ Added: svn:executable + * Added: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-test-spec-fragment =================================================================== --- epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-test-spec-fragment (rev 0) +++ epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-test-spec-fragment 2011-09-09 10:06:32 UTC (rev 7343) @@ -0,0 +1,86 @@ +#!/usr/bin/python + +import koji +import Cheetah.Template +import sys +import optparse +import os +import shutil +import tempfile +import urlgrabber +import urlgrabber.progress + +DOWNLOAD_BASE = 'http://download.devel.redhat.com/brewroot/work' + +parser = optparse.OptionParser(usage="usage: %prog spec-template") +parser.add_option('--task-id', help="The taskID to retieve task data from") +parser.add_option('--nvr', help="The name-version-release to use when building the rpms") +parser.add_option('--maven-info', help="The groupId:artifactId:version to substitute into the specfile template") + +options, args = parser.parse_args() + +if len(args) != 1: + parser.error('You must specify a specfile fragment') +spec_template = args[0] + +if not options.task_id: + parser.error('You must specify a taskID with --task-id') + +session = koji.ClientSession('http://brewhub.devel.redhat.com/brewhub') + +values = {} + +if options.nvr: + nvr = koji.parse_NVR(options.nvr) + values.update(nvr) +elif options.maven_info: + groupId, artifactId, version = options.maven_info.split(':') + values['name'] = groupId + '-' + artifactId + values['version'] = version.replace('-', '_') + values['release'] = 1 + values['epoch'] = None +else: + parser.error('You must specify either --nvr or --maven-info') + +tmpdir = tempfile.mkdtemp() + +task_id = int(options.task_id) +output = session.listTaskOutput(task_id) +artifacts = {} +all_artifacts = [] +pg = urlgrabber.progress.TextMeter() +for filename in output: + if filename.endswith('.log'): + continue + base = os.path.basename(filename) + all_artifacts.append(base) + artifacts.setdefault(os.path.splitext(base)[1], []).append(base) + fileurl = DOWNLOAD_BASE + '/' + koji.pathinfo.taskrelpath(task_id) + '/' + filename + urlgrabber.grabber.urlgrab(fileurl, filename=tmpdir + '/' + base, progress_obj=pg) + +values['artifacts'] = artifacts +values['all_artifacts'] = all_artifacts + +contents = Cheetah.Template.Template(file=spec_template, + searchList=[values]).respond() +specfile = tmpdir + '/' + os.path.basename(spec_template[:-5]) +outfd = file(specfile, 'w') +outfd.write(contents) +outfd.close() + +print 'Wrote', specfile + +spec_dir = os.path.dirname(spec_template) +if not spec_dir: + spec_dir = '.' +for filename in os.listdir(spec_dir): + filepath = os.path.join(spec_dir, filename) + if not filename.startswith('.') and os.path.isfile(filepath): + shutil.copy(filepath, tmpdir) + +rpmcmd = 'rpmbuild --define "_sourcedir %(tmpdir)s" --define "_builddir %(tmpdir)s" ' \ + '--define "_srcrpmdir %(tmpdir)s" --define "_rpmdir %(tmpdir)s" ' \ + '--define "dist .test.mead" --nodeps -ba %(specfile)s' % locals() +print 'Running', rpmcmd + +os.system(rpmcmd) Property changes on: epp/portal/branches/EPP_5_2_Branch/distribution/mead-tools/src/brewscripts/mead-test-spec-fragment ___________________________________________________________________ Added: svn:executable + *