[infinispan-commits] Infinispan SVN: r1791 - trunk/bin.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Thu May 13 15:59:28 EDT 2010
Author: manik.surtani at jboss.com
Date: 2010-05-13 15:59:28 -0400 (Thu, 13 May 2010)
New Revision: 1791
Modified:
trunk/bin/pythonTools.py
trunk/bin/release.py
Log:
Updated release script
Modified: trunk/bin/pythonTools.py
===================================================================
--- trunk/bin/pythonTools.py 2010-05-13 14:42:05 UTC (rev 1790)
+++ trunk/bin/pythonTools.py 2010-05-13 19:59:28 UTC (rev 1791)
@@ -4,7 +4,7 @@
import subprocess
import sys
import readline
-
+import shutil
settings_file = '%s/.infinispan_dev_settings' % os.getenv('HOME')
### Known config keys
@@ -12,7 +12,32 @@
local_tags_dir_key = "local_tags_dir"
local_mvn_repo_dir_key = "local_mvn_repo_dir"
maven_pom_xml_namespace = "http://maven.apache.org/POM/4.0.0"
+default_settings = {'dry_run': False, 'multi_threaded': False, 'verbose': False}
+boolean_keys = ['dry_run', 'multi_threaded', 'verbose']
+def apply_defaults(s):
+ for e in default_settings.items():
+ if e[0] not in s:
+ s[e[0]] = e[1]
+ return s
+
+def check_mandatory_settings(s):
+ missing_keys = []
+ required_keys = [svn_base_key, local_tags_dir_key]
+ for k in required_keys:
+ if k not in s:
+ missing_keys.append(k)
+
+ if len(missing_keys) > 0:
+ print "Entries %s are missing in configuration file %s! Cannot proceed!" % (missing_keys, settings_file)
+ sys.exit(2)
+
+def to_bool(x):
+ if type(x) == bool:
+ return x
+ if type(x) == str:
+ return {'true': True, 'false': False}.get(x.strip().lower())
+
def get_settings():
"""Retrieves user-specific settings for all Infinispan tools. Returns a dict of key/value pairs, or an empty dict if the settings file doesn't exist."""
f = None
@@ -24,6 +49,10 @@
kvp = l.split("=")
if kvp and len(kvp) > 0 and kvp[0] and len(kvp) > 1:
settings[kvp[0].strip()] = kvp[1].strip()
+ settings = apply_defaults(settings)
+ check_mandatory_settings(settings)
+ for k in boolean_keys:
+ settings[k] = to_bool(settings[k])
return settings
except IOError as ioe:
return {}
@@ -50,8 +79,9 @@
s = {}
s["svn_base"] = input_with_default("Base Subversion URL to use", "https://svn.jboss.org/repos/infinispan")
s["local_tags_dir"] = input_with_default("Local tags directory to use", "%s/Code/infinispan/tags" % os.getenv("HOME"))
- s["multi_threaded"] = input_with_default("Enable multithreaded (EXPERIMENTAL!)", "False")
-
+ s["verbose"] = input_with_default("Be verbose?", False)
+ s["multi_threaded"] = input_with_default("Run multi-threaded? (Disable to debug)", True)
+ s = apply_defaults(s)
f = open(settings_file, "w")
try:
for e in s.keys():
@@ -60,7 +90,7 @@
f.close()
def require_settings_file(recursive = False):
- """Tests whether the settings file exists, and if not exits with an error message."""
+ """Tests whether the settings file exists, and if not prompts the user to create one."""
f = None
try:
f = open(settings_file)
@@ -68,13 +98,14 @@
if not recursive:
handle_release_virgin()
require_settings_file(True)
+ print "User-specific environment settings file %s created! Please start this script again!" % settings_file
+ sys.exit(4)
else:
print "User-specific environment settings file %s is missing! Cannot proceed!" % settings_file
print "Please create a file called %s with the following lines:" % settings_file
print '''
svn_base = https://svn.jboss.org/repos/infinispan
local_tags_dir = /PATH/TO/infinispan/tags
- local_mvn_repo_dir = /PATH/TO/maven2/org/infinispan
multi_threaded = False
'''
sys.exit(3)
@@ -126,46 +157,146 @@
if fnmatch.fnmatch(file, self.pattern):
return fullname
-class SvnConn(object):
+class SvnConn(object):
"""An SVN cnnection making use of the command-line SVN client. Replacement for PySVN which sucked for various reasons."""
+
+ def __init__(self):
+ if settings['verbose']:
+ self.svn_cmd = ['svn']
+ else:
+ self.svn_cmd = ['svn', '-q']
+
+ def do_svn(self, params):
+ commands = []
+ for e in self.svn_cmd:
+ commands.append(e)
+ for e in params:
+ commands.append(e)
+ subprocess.check_call(commands)
+
def tag(self, fr_url, to_url, version):
"""Tags a release."""
checkInMessage = "Infinispan Release Script: Tagging " + version
- subprocess.check_call(["svn", "cp", fr_url, to_url, "-m", checkInMessage])
+ self.do_svn(["cp", fr_url, to_url, "-m", checkInMessage])
def checkout(self, url, to_dir):
"""Checks out a URL to the given directory"""
- subprocess.check_call(["svn", "checkout", url, to_dir])
+ self.do_svn(["checkout", url, to_dir])
def checkin(self, working_dir, msg):
"""Checks in a working directory with the appropriate message"""
- subprocess.check_call(["svn", "commit", "-m", msg, working_dir])
+ self.do_svn(["commit", "-m", msg, working_dir])
def add(self, directory):
"""Adds a directory or file to SVN. Directory can either be the name of a file or dir, or a list of either."""
if directory:
- call_params = ["svn", "add"]
+ call_params = ["add"]
if isinstance(directory, str):
call_params.append(directory)
else:
for d in directory:
call_params.append(d)
- subprocess.check_call(call_params)
+ self.do_svn(call_params)
+
+
+class DryRun(object):
+ location_root = "%s/%s" % (os.getenv("HOME"), "infinispan_release_dry_run")
+ flags = "-r"
+
+ def __init__(self):
+ if settings['verbose']:
+ self.flags = "-rv"
+
+ def find_version(self, url):
+ return os.path.split(url)[1]
+ def copy(self, src, dst):
+ print " DryRun: Executing %s" % ['rsync', self.flags, src, dst]
+ try:
+ os.makedirs(dst)
+ except:
+ pass
+ subprocess.check_call(['rsync', self.flags, src, dst])
+
-def get_svn_conn():
- """Factory to create and retrieve an SvnConn instance"""
- return SvnConn()
+class DryRunSvnConn(DryRun):
+ urls = {}
+ def tag(self, fr_url, to_url, version):
+ self.urls[version] = '%s/svn/%s' % (self.location_root, version)
+ trunk_dir = settings[local_tags_dir_key].replace('/tags', '/trunk')
+ if os.path.isdir(trunk_dir) and is_in_svn(trunk_dir):
+ self.copy(trunk_dir, '%s/svn' % self.location_root)
+ os.rename('%s/svn/trunk' % self.location_root, self.urls[version])
+ else:
+ subprocess.check_call(["svn", "export", fr_url, self.urls[version]])
+
+ def checkout(self, url, to_dir):
+ ver = self.find_version(url)
+ if ver in self.urls:
+ elems = os.path.split(to_dir)
+ self.copy(self.urls[ver], elems[0])
+ else:
+ subprocess.check_call(["svn", "export", url, to_dir])
+ def checkin(self, working_dir, msg):
+ ver = self.find_version(working_dir)
+ subprocess.check_call(['rsync', working_dir, self.urls[ver]])
+ def add(self, directory):
+ print " DryRunSvnConn: Adding " + directory
+ pass
+
+class Uploader(object):
+ def __init__(self):
+ if settings['verbose']:
+ self.scp_cmd = ['scp', '-rv']
+ self.rsync_cmd = ['rsync', '-rv']
+ else:
+ self.scp_cmd = ['scp', '-r']
+ self.rsync_cmd = ['rsync', '-r']
+
+ def upload_scp(self, fr, to, flags = []):
+ self.upload(fr, to, flags, self.scp_cmd)
+
+ def upload_rsync(self, fr, to, flags = []):
+ self.upload(fr, to, flags, self.rsync_cmd)
+
+ def upload(self, fr, to, flags, cmd):
+ for e in flags:
+ cmd.append(e)
+ cmd.append(fr)
+ cmd.append(to)
+ subprocess.check_call(cmd)
+
+
+
+class DryRunUploader(DryRun):
+ def upload_scp(self, fr, to, flags = []):
+ self.upload(fr, to, "scp")
+
+ def upload_rsync(self, fr, to, flags = []):
+ self.upload(fr, to.replace(':', '____').replace('@', "__"), "rsync")
+
+ def upload(self, fr, to, type):
+ self.copy(fr, "%s/%s/%s" % (self.location_root, type, to))
+
+
def is_in_svn(directory):
return os.path.isdir(directory + "/.svn")
def maven_build_distribution():
"""Builds the distribution in the current working dir"""
- subprocess.check_call(["mvn", "install", "-Pjmxdoc", "-Dmaven.test.skip.exec=true"])
- subprocess.check_call(["mvn", "install", "-Pconfigdoc", "-Dmaven.test.skip.exec=true"])
- subprocess.check_call(["mvn", "deploy", "-Pdistribution", "-Dmaven.test.skip.exec=true"])
+ mvn_commands = [["install", "-Pjmxdoc"],["install", "-Pconfigdoc"], ["deploy", "-Pdistribution"]]
+
+ for c in mvn_commands:
+ c.append("-Dmaven.test.skip.exec=true")
+ if settings['dry_run']:
+ c.append("-Dmaven.deploy.skip=true")
+ if not settings['verbose']:
+ c.insert(0, '-q')
+ c.insert(0, 'mvn')
+ subprocess.check_call(c)
+
def get_version_pattern():
return re.compile("^([4-9]\.[0-9])\.[0-9]\.(Final|(ALPHA|BETA|CR)[1-9][0-9]?)$", re.IGNORECASE)
Modified: trunk/bin/release.py
===================================================================
--- trunk/bin/release.py 2010-05-13 14:42:05 UTC (rev 1790)
+++ trunk/bin/release.py 2010-05-13 19:59:28 UTC (rev 1791)
@@ -20,8 +20,9 @@
from pythonTools import *
-multi_threaded = 'multi_threaded' in settings and "true" == settings['multi_threaded'].strip().lower()
modules = []
+uploader = None
+svn_conn = None
def getModules(directory):
# look at the pom.xml file
@@ -60,8 +61,7 @@
helpAndExit()
def tagInSubversion(version, newVersion):
- sc = get_svn_conn()
- sc.tag("%s/trunk" % settings[svn_base_key], newVersion, version)
+ svn_conn.tag("%s/trunk" % settings[svn_base_key], newVersion, version)
def getProjectVersionTag(tree):
return tree.find("./{%s}version" % (maven_pom_xml_namespace))
@@ -83,13 +83,15 @@
finally:
in_f.close()
out_f.close()
- print " ... updated %s" % pomFile
+ if settings['verbose']:
+ print " ... updated %s" % pomFile
def patch(pomFile, version):
## Updates the version in a POM file
## We need to locate //project/parent/version, //project/version and //project/properties/project-version
## And replace the contents of these with the new version
- print "Patching %s" % pomFile
+ if settings['verbose']:
+ print "Patching %s" % pomFile
tree = ElementTree()
tree.parse(pomFile)
need_to_write = False
@@ -101,7 +103,8 @@
for tag in tags:
if tag != None:
- print "%s is %s. Setting to %s" % (str(tag), tag.text, version)
+ if settings['verbose']:
+ print "%s is %s. Setting to %s" % (str(tag), tag.text, version)
tag.text=version
need_to_write = True
@@ -109,7 +112,8 @@
# write to file again!
writePom(tree, pomFile)
else:
- print "File doesn't need updating; nothing replaced!"
+ if settings['verbose']:
+ print "File doesn't need updating; nothing replaced!"
def get_poms_to_patch(workingDir):
getModules(workingDir)
@@ -124,12 +128,8 @@
return pomsToPatch
-def updateVersions(version, workingDir, trunkDir, test = False):
- if test:
- shutil.copytree(trunkDir, workingDir)
- else:
- client = get_svn_conn()
- client.checkout(settings[svn_base_key] + "/tags/" + version, workingDir)
+def updateVersions(version, workingDir, trunkDir):
+ svn_conn.checkout(settings[svn_base_key] + "/tags/" + version, workingDir)
pomsToPatch = get_poms_to_patch(workingDir)
@@ -160,10 +160,9 @@
os.rename(version_java+".tmp", version_java)
- if not test:
- # Now make sure this goes back into SVN.
- checkInMessage = "Infinispan Release Script: Updated version numbers"
- client.checkin(workingDir, checkInMessage)
+ # Now make sure this goes back into SVN.
+ checkInMessage = "Infinispan Release Script: Updated version numbers"
+ svn_conn.checkin(workingDir, checkInMessage)
def buildAndTest(workingDir):
os.chdir(workingDir)
@@ -176,68 +175,61 @@
def uploadArtifactsToSourceforge(version):
+ shutil.rmtree(".tmp", ignore_errors = True)
os.mkdir(".tmp")
+ os.mkdir(".tmp/%s" % version)
os.chdir(".tmp")
- do_not_copy = shutil.ignore_patterns('*.xml', '*.sha1', '*.md5', '*.pom', '.svn')
- shutil.copytree("%s/%s/target/distribution" % (settings[local_tags_dir_key], version), version, ignore = do_not_copy)
- subprocess.check_call(["scp", "-r", version, "sourceforge_frs:/home/frs/project/i/in/infinispan/infinispan"])
+ dist_dir = "%s/%s/target/distribution" % (settings[local_tags_dir_key], version)
+ print "Copying from %s to %s" % (dist_dir, version)
+ for item in os.listdir(dist_dir):
+ full_name = "%s/%s" % (dist_dir, item)
+ if item.strip().lower().endswith(".zip") and os.path.isfile(full_name):
+ shutil.copy2(full_name, version)
+ uploader.upload_scp(version, "sourceforge_frs:/home/frs/project/i/in/infinispan/infinispan")
shutil.rmtree(".tmp", ignore_errors = True)
-def uploadJavadocs(base_dir, workingDir, version):
+def unzip_archive(workingDir, version):
+ os.chdir("%s/target/distribution" % workingDir)
+ ## Grab the distribution archive and un-arch it
+ shutil.rmtree("infinispan-%s" % version, ignore_errors = True)
+ if settings['verbose']:
+ subprocess.check_call(["unzip", "infinispan-%s-all.zip" % version])
+ else:
+ subprocess.check_call(["unzip", "-q", "infinispan-%s-all.zip" % version])
+
+def uploadJavadocs(workingDir, version):
"""Javadocs get rsync'ed to filemgmt.jboss.org, in the docs_htdocs/infinispan directory"""
version_short = get_version_major_minor(version)
- os.chdir("%s/target/distribution" % workingDir)
- ## Grab the distribution archive and un-arch it
- subprocess.check_call(["unzip", "infinispan-%s-all.zip" % version])
- os.chdir("infinispan-%s/doc" % version)
+ os.chdir("%s/target/distribution/infinispan-%s/doc" % (workingDir, version))
## "Fix" the docs to use the appropriate analytics tracker ID
subprocess.check_call(["%s/bin/updateTracker.sh" % workingDir])
os.mkdir(version_short)
os.rename("apidocs", "%s/apidocs" % version_short)
## rsync this stuff to filemgmt.jboss.org
- subprocess.check_call(["rsync", "-rv", "--protocol=28", version_short, "infinispan at filemgmt.jboss.org:/docs_htdocs/infinispan"])
-
- # subprocess.check_call(["tar", "zcf", "%s/apidocs-%s.tar.gz" % (base_dir, version), version_short])
- ## Upload to sourceforge
- # os.chdir(base_dir)
- # subprocess.check_call(["scp", "apidocs-%s.tar.gz" % version, "sourceforge_frs:"])
- # print """
- # API docs are in %s/apidocs-%s.tar.gz
- # They have also been uploaded to Sourceforge.
- #
- # MANUAL STEP:
- # 1) SSH to sourceforge (ssh -t SF_USERNAME,infinispan at shell.sourceforge.net create) and run '/home/groups/i/in/infinispan/install_apidocs.sh'
- # """ % (base_dir, version)
+ uploader.upload_rsync(version_short, "infinispan at filemgmt.jboss.org:/docs_htdocs/infinispan", flags = ['-rv', '--protocol=28'])
-def uploadSchema(base_dir, workingDir, version):
+def uploadSchema(workingDir, version):
"""Schema gets rsync'ed to filemgmt.jboss.org, in the docs_htdocs/infinispan/schemas directory"""
os.chdir("%s/target/distribution/infinispan-%s/etc/schema" % (workingDir, version))
## rsync this stuff to filemgmt.jboss.org
- subprocess.check_call(["rsync", "-rv", "--protocol=28", ".", "infinispan at filemgmt.jboss.org:/docs_htdocs/infinispan/schemas"])
+ uploader.upload_rsync('.', "infinispan at filemgmt.jboss.org:/docs_htdocs/infinispan/schemas", ['-rv', '--protocol=28'])
def do_task(target, args, async_processes):
- if multi_threaded:
+ if settings['multi_threaded']:
async_processes.append(Process(target = target, args = args))
else:
target(*args)
### This is the starting place for this script.
def release():
+ global settings
+ global uploader
+ global svn_conn
assert_python_minimum_version(2, 5)
require_settings_file()
-
- missing_keys = []
- expected_keys = [svn_base_key, local_tags_dir_key]
- for expected_key in expected_keys:
- if expected_key not in settings:
- missing_keys.append(expected_key)
-
- if len(missing_keys) > 0:
- print "Entries %s are missing in configuration file %s! Cannot proceed!" % (missing_keys, settings_file)
- sys.exit(2)
# We start by determining whether the version passed in is a valid one
if len(sys.argv) < 2:
@@ -248,6 +240,17 @@
print "Releasing Infinispan version " + version
print "Please stand by!"
+ ## Set up network interactive tools
+ if settings['dry_run']:
+ # Use stubs
+ print "*** This is a DRY RUN. No changes will be committed. Used to test this release script only. ***"
+ print "Your settings are %s" % settings
+ uploader = DryRunUploader()
+ svn_conn = DryRunSvnConn()
+ else:
+ uploader = Uploader()
+ svn_conn= SvnConn()
+
## Release order:
# Step 1: Tag in SVN
newVersion = "%s/tags/%s" % (settings[svn_base_key], version)
@@ -268,10 +271,13 @@
print "Step 3: Complete"
async_processes = []
+
+ ##Unzip the newly built archive now
+ unzip_archive(workingDir, version)
# Step 4: Upload javadocs to FTP
print "Step 4: Uploading Javadocs"
- do_task(uploadJavadocs, [base_dir, workingDir, version], async_processes)
+ do_task(uploadJavadocs, [workingDir, version], async_processes)
print "Step 4: Complete"
print "Step 5: Uploading to Sourceforge"
@@ -279,7 +285,7 @@
print "Step 5: Complete"
print "Step 6: Uploading to configuration XML schema"
- do_task(uploadSchema, [base_dir, workingDir, version], async_processes)
+ do_task(uploadSchema, [workingDir, version], async_processes)
print "Step 6: Complete"
## Wait for processes to finish
More information about the infinispan-commits
mailing list