Author: nickboldt Date: 2010-12-09 15:42:44 -0500 (Thu, 09 Dec 2010) New Revision: 27300 Modified: trunk/build/aggregate/site/build.xml trunk/build/aggregate/site/pom.xml Log: remove comments and use only a single composite repo; support reading metadata from composite repos to get list of child sites; from those get zips to download Modified: trunk/build/aggregate/site/build.xml =================================================================== --- trunk/build/aggregate/site/build.xml 2010-12-09 20:42:34 UTC (rev 27299) +++ trunk/build/aggregate/site/build.xml 2010-12-09 20:42:44 UTC (rev 27300) @@ -42,6 +42,124 @@ </taskdef> </target> + <!-- = = = = = = = = = = = = = = = = = + macrodef: fetchZipsFromList - given a list of zips in zip.list.txt, fetch those zips + = = = = = = = = = = = = = = = = = --> + <macrodef name="fetchZipsFromList"> + <attribute name="repo" default="" /> + <sequential> + + <var name="location.cleaned" value="@{repo}" /> + <propertyregex defaultvalue="${location.cleaned}" input="${location.cleaned}" regexp="/all/repo/*$" replace="" casesensitive="true" override="true" property="location.cleaned" /> + <propertyregex defaultvalue="${location.cleaned}" input="${location.cleaned}" regexp="/tools/*$" replace="" casesensitive="true" override="true" property="location.cleaned" /> + + <echo level="verbose">repo = ${location.cleaned}</echo> + <!-- fetch zip.list.txt file, if available --> + <var unset="true" name="ALL_ZIPS" /> + <var unset="true" name="wget.return" /> + <!-- quieter output and return code check w/ wget instead of <get> --> + <exec executable="wget" dir="${aggegate.zips.dir}" failonerror="false" failifexecutionfails="true" resultproperty="wget.return" outputproperty="null"> + <arg line="${location.cleaned}/logs/zip.list.txt -q --no-clobber" /> + </exec> + <!--get src="${location.cleaned}/logs/zip.list.txt" dest="${aggegate.zips.dir}/zip.list.txt" ignoreerrors="true" /--> + <if> + <and> + <equals arg1="${wget.return}" arg2="0" /> + <available file="${aggegate.zips.dir}/zip.list.txt" type="file" /> + </and> + <then> + <!-- load zip.list.txt file, get ALL_ZIPS list --> + <var unset="true" name="ALL_ZIPS" /> + <property file="${aggegate.zips.dir}/zip.list.txt" /> + <if> + <isset property="ALL_ZIPS" /> + <then> + <var name="relativePath" unset="true" /> + <for list="${ALL_ZIPS}" delimiter=", " keepgoing="true" param="zipPath"> + <sequential> + <propertyregex override="true" property="relativePath" defaultvalue="" input="@{zipPath}" regexp="(.+)/([^/]+.zip)" replace="\1" /> + <mkdir dir="${aggegate.zips.dir}/${relativePath}" /> + <!-- fetch zips to local dir --> + <echo level="verbose">repoNum = @{repoNum} +relativePath = ${relativePath} +zipPath = @{zipPath} +DEST = ${aggegate.zips.dir}/@{zipPath} +</echo> + <echo> Zip URL = ${location.cleaned}/@{zipPath}</echo> + <if> + <isset property="isTest" /> + <then> + <touch file="${aggegate.zips.dir}/@{zipPath}" /> + </then> + <else> + <var name="wget.return" unset="true" /> + <exec executable="wget" dir="${aggegate.zips.dir}" failonerror="false" failifexecutionfails="false" resultproperty="wget.return" outputproperty="null"> + <arg line="${location.cleaned}/@{zipPath} -q --no-clobber" /> + </exec> + <!--get src="${location.cleaned}/@{zipPath}" dest="${aggegate.zips.dir}/@{zipPath}" /--> + <if> + <not> + <equals arg1="${wget.return}" arg2="0" /> + </not> + <then> + <echo> ** Error [${wget.return}] ** Could not wget ${location.cleaned}/@{zipPath}</echo> + </then> + </if> + </else> + </if> + </sequential> + </for> + <var name="relativePath" unset="true" /> + </then> + </if> + <var unset="true" name="ALL_ZIPS" /> + <delete file="${aggegate.zips.dir}/zip.list.txt" quiet="true" /> + </then> + <else> + <echo> ** Error [${wget.return}] ** Could not wget ${location.cleaned}/logs/zip.list.txt</echo> + </else> + </if> + <var name="location.cleaned" unset="true" /> + </sequential> + </macrodef> + + <!-- = = = = = = = = = = = = = = = = = + macrodef: fetchZipsFromMetadata - look for compositeArtifacts.xml and parse results + = = = = = = = = = = = = = = = = = --> + <macrodef name="fetchZipsFromMetadata"> + <attribute name="repo" default="" /> + <sequential> + <echo level="verbose">repo = @{repo}</echo> + <!-- if no zip.list.txt from the inputRepo, then fetch from @{repo}/compositeArtifacts.xml instead --> + <var name="wget.return" unset="true" /> + <exec executable="wget" dir="${aggegate.zips.dir}" failonerror="false" failifexecutionfails="true" resultproperty="wget.return" outputproperty="null"> + <arg line=&quot;(a){repo}/compositeArtifacts.xml -q --no-clobber" /> + </exec> + <if> + <and> + <equals arg1="${wget.return}" arg2="0" /> + <available file="${aggegate.zips.dir}/compositeArtifacts.xml" type="file" /> + </and> + <then> + <echo level="verbose">Got ${aggegate.zips.dir}/compositeArtifacts.xml</echo> + <xmlproperty file="${aggegate.zips.dir}/compositeArtifacts.xml" validate="false" collapseAttributes="true" /> + <echo level="verbose">Found ${repository.children.size} contained repos to search</echo> + <!-- foreach entry repository.children.child.location --> + <for param="location" list="${repository.children.child.location}"> + <sequential> + <echo level="error">Site URL = @{location}</echo> + <fetchZipsFromList repo="@{location}" /> + </sequential> + </for> + <delete file="${aggegate.zips.dir}/compositeArtifacts.xml" quiet="true" /> + </then> + <else> + <echo> ** Error [${wget.return}] ** Could not wget @{repo}/compositeArtifacts.xml</echo> + </else> + </if> + </sequential> + </macrodef> + <target name="add.associate.sites" if="associate.sites"> <if> <and> @@ -148,12 +266,7 @@ <isset property="BUILD_NUMBER" /> </and> <then> - <propertyregex override="true" - property="update.site.version" - defaultvalue="${JOB_NAME}" - input="${JOB_NAME}" - regexp="jbosstools-([0-9.]+.*).aggregate" - replace=": \1.${BUILD_ID}-H${BUILD_NUMBER}" /> + <propertyregex override="true" property="update.site.version" defaultvalue="${JOB_NAME}" input="${JOB_NAME}" regexp="jbosstools-([0-9.]+.*).aggregate" replace=": \1.${BUILD_ID}-H${BUILD_NUMBER}" /> </then> <else> <property name="update.site.version" value="" /> @@ -181,63 +294,64 @@ <mkdir dir="${aggegate.zips.dir}" /> <for list="${inputRepos}" delimiter=", " keepgoing="true" param="repoNum"> <sequential> - <!-- fetch zip.list.txt file, if available --> - <var unset="true" name="ALL_ZIPS" /> - <var unset="true" name="wget.return" /> - <!-- quieter output and return code check w/ wget instead of <get> --> - <exec executable="wget" dir="${aggegate.zips.dir}" failonerror="false" failifexecutionfails="true" resultproperty="wget.return"> - <arg line=&quot;${inputRepo(a){repoNum}}/logs/zip.list.txt -q --no-clobber" /> - </exec> - <!--get src=&quot;${inputRepo(a){repoNum}}/logs/zip.list.txt&quot; dest="${aggegate.zips.dir}/zip.list.txt" ignoreerrors="true" /--> + <var name="repo" value="${inputRepo@{repoNum}}" /> <if> - <and> - <equals arg1="${wget.return}" arg2="0" /> - <available file="${aggegate.zips.dir}/zip.list.txt" type="file" /> - </and> + <contains string="${repo}" substring="composite" /> <then> - <!-- load zip.list.txt file, get ALL_ZIPS list --> - <var unset="true" name="ALL_ZIPS" /> - <property file="${aggegate.zips.dir}/zip.list.txt" /> - <if> - <isset property="ALL_ZIPS" /> - <then> - <var name="relativePath" unset="true" /> - <for list="${ALL_ZIPS}" delimiter=", " keepgoing="true" param="zipPath"> - <sequential> - <propertyregex override="true" - property="relativePath" - defaultvalue="" - input="@{zipPath}" - regexp="(.+)/([^/]+.zip)" - replace="\1" /> - <mkdir dir="${aggegate.zips.dir}/${relativePath}" /> - <!-- fetch zips to local dir --> - <echo level="verbose">repoNum = @{repoNum} -relativePath = ${relativePath} -zipPath = @{zipPath} -DEST = ${aggegate.zips.dir}/@{zipPath} -</echo> - <echo>URL = ${inputRepo@{repoNum}}/@{zipPath} -</echo> - <exec executable="wget" dir="${aggegate.zips.dir}" failonerror="false" failifexecutionfails="false"> - <arg line="${inputRepo@{repoNum}}/@{zipPath} -q --no-clobber" /> - </exec> - <!--get src="${inputRepo@{repoNum}}/@{zipPath}" dest="${aggegate.zips.dir}/@{zipPath}" /--> - </sequential> - </for> - <var name="relativePath" unset="true" /> - </then> - </if> - <var unset="true" name="ALL_ZIPS" /> - <delete file="${aggegate.zips.dir}/zip.list.txt" quiet="true" /> + <fetchZipsFromMetadata repo="${repo}" /> </then> + <else> + <fetchZipsFromList repo="${repo}" /> + </else> </if> </sequential> </for> </target> - <target name="custom.build" - description="aggregate update site extras" - depends="init,check.target,add.associate.sites,add.web.content,pack.zip,collect.zips" /> + <target name="custom.build" description="aggregate update site extras" depends="init,check.target,add.associate.sites,add.web.content,pack.zip,collect.zips" /> + <target name="test.collect.zips.from.metadata" depends="init"> + <property name="isTest" value="true" /> + <antcall target="collect.zips"> + <param name="inputRepos" value="1" /> + <param name="inputRepo1" value="http://download.jboss.org/jbosstools/builds/staging/_composit... /> + </antcall> + </target> + + <target name="test.collect.zips.from.metadata.with.errors" depends="init"> + <property name="isTest" value="true" /> + <antcall target="collect.zips"> + <param name="inputRepos" value="1,2" /> + <param name="inputRepo1" value="http://download.jboss.org/jbosstools/builds/staging/_composit... /> + </antcall> + </target> + + <target name="test.collect.zips.from.list" depends="init"> + <property name="isTest" value="true" /> + <antcall target="collect.zips"> + <param name="inputRepos" value="1,2,3,4,5" /> + <param name="inputRepo1" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + <param name="inputRepo2" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + <param name="inputRepo3" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + <param name="inputRepo4" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + <param name="inputRepo5" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + </antcall> + </target> + + <target name="test.collect.zips.from.list.with.errors" depends="init"> + <property name="isTest" value="true" /> + <antcall target="collect.zips"> + <param name="inputRepos" value="1,2" /> + <param name="inputRepo2" value="http://download.jboss.org/jbosstools/builds/staging/jbosstool... /> + </antcall> + </target> + + <target name="test.collect.zips" depends="test.collect.zips.from.metadata,test.collect.zips.from.list"> + <delete dir="${aggegate.zips.dir}" quiet="true" /> + </target> + + <target name="test.collect.zips.with.error" depends="test.collect.zips.from.metadata.with.errors,test.collect.zips.from.list.with.errors"> + <delete dir="${aggegate.zips.dir}" quiet="true" /> + </target> + </project> Modified: trunk/build/aggregate/site/pom.xml =================================================================== --- trunk/build/aggregate/site/pom.xml 2010-12-09 20:42:34 UTC (rev 27299) +++ trunk/build/aggregate/site/pom.xml 2010-12-09 20:42:44 UTC (rev 27300) @@ -14,18 +14,8 @@ <packaging>eclipse-update-site</packaging> <properties> - <!-- add more inputRepo# here, then reference below; need variables so - that these can be referred to in Ant script later --> - <!-- <inputRepo1>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo2>http://download.jboss.org/jbosstools/builds/staging/pi4soa-... - <inputRepo3>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo4>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo5>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo6>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo7>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepo8>http://download.jboss.org/jbosstools/builds/staging/jbossto... - <inputRepos>1,2,3,4,5,6,7,8</inputRepos> --> - <!-- use a single composite site so adding more jobs' output is easier --> + <!-- add more inputRepos into the composite metadata at the URL below. Therefore need not edit this file and add dozens of URLs + which need to be called by Ant script later --> <inputRepo1>http://download.jboss.org/jbosstools/builds/staging/_compos... <inputRepos>1</inputRepos> </properties> @@ -44,11 +34,6 @@ <quiet>true</quiet> <tasks> <property name="inputRepo1" value="${inputRepo1}" /> - <!-- <property name="inputRepo2" value="${inputRepo2}" /> <property - name="inputRepo3" value="${inputRepo3}" /> <property name="inputRepo4" value="${inputRepo4}" - /> <property name="inputRepo5" value="${inputRepo5}" /> <property name="inputRepo6" - value="${inputRepo6}" /> <property name="inputRepo7" value="${inputRepo7}" - /> <property name="inputRepo8" value="${inputRepo8}" /> --> <property name="inputRepos" value="${inputRepos}" /> <!-- called AFTER generating update site + zip to add in extra content --> @@ -98,21 +83,6 @@ <enabled>true</enabled> </snapshots> </repository> - <!-- <repository> <id>inputRepo1</id> <url>${inputRepo1}/all/repo/</url> - <layout>p2</layout> <snapshots> <enabled>true</enabled> </snapshots> </repository> - <repository> <id>inputRepo2</id> <url>${inputRepo2}/all/repo/</url> <layout>p2</layout> - <snapshots> <enabled>true</enabled> </snapshots> </repository> <repository> - <id>inputRepo3</id> <url>${inputRepo3}/all/repo/</url> <layout>p2</layout> - <snapshots> <enabled>true</enabled> </snapshots> </repository> <repository> - <id>inputRepo4</id> <url>${inputRepo4}/all/repo/</url> <layout>p2</layout> - <snapshots> <enabled>true</enabled> </snapshots> </repository> <repository> - <id>inputRepo5</id> <url>${inputRepo5}/tools/</url> <layout>p2</layout> <snapshots> - <enabled>true</enabled> </snapshots> </repository> <repository> <id>inputRepo6</id> - <url>${inputRepo6}/tools/</url> <layout>p2</layout> <snapshots> <enabled>true</enabled> - </snapshots> </repository> <repository> <id>inputRepo7</id> <url>${inputRepo7}/tools/</url> - <layout>p2</layout> <snapshots> <enabled>true</enabled> </snapshots> </repository> - <repository> <id>inputRepo8</id> <url>${inputRepo8}/tools/</url> <layout>p2</layout> - <snapshots> <enabled>true</enabled> </snapshots> </repository> --> <repository> <id>helios</id> <url>http://download.eclipse.org/releases/helios/</url>