Author: justi9 Date: 2008-10-01 11:13:42 -0400 (Wed, 01 Oct 2008) New Revision: 2559 Modified: mgmt/trunk/cumin/bin/cumin mgmt/trunk/cumin/bin/cumin-bench mgmt/trunk/cumin/bin/cumin-test mgmt/trunk/cumin/python/cumin/__init__.py mgmt/trunk/cumin/python/cumin/tools.py mgmt/trunk/cumin/python/cumin/util.py Log: Move commands over to the common parsley code. Remove util.Config in favor of the Config classes in parsley. Modified: mgmt/trunk/cumin/bin/cumin =================================================================== --- mgmt/trunk/cumin/bin/cumin 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/bin/cumin 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,79 +1,12 @@ #!/usr/bin/python -import sys, os, socket +import sys, os -from logging import getLogger +from cumin.tools import CuminServerTool -from cumin import * -from cumin.util import * - -log = getLogger("cumin") - -def do_main(config): - app = Cumin(config.home, config.data, config.spec) - - if config.debug: - app.enable_debug() - - try: - app.check() - except Exception, e: - log.exception(e) - sys.exit(1) - - app.init() - - server = CuminServer(app, config.addr, config.port) - - if config.ssl: - cpath = os.path.join(config.home, "etc", "cumin.crt") - kpath = os.path.join(config.home, "etc", "cumin.key") - - if os.path.isfile(cpath): - server.set_ssl_cert_path(cpath) - else: - log.error("SSL certificate file '%s' not found" % cpath) - sys.exit(1) - - if os.path.isfile(kpath): - server.set_ssl_key_path(kpath) - else: - log.error("SSL key file '%s' not found" % kpath) - sys.exit(1) - - try: - app.start() - - try: - server.start() - finally: - server.stop() - finally: - app.stop() - def main(): - config = CuminConfig() + CuminServerTool("cumin").main() - summ = ("ADDR", "Run web server at address ADDR") - config.add_param("addr", str, "localhost", summ) - - summ = ("PORT", "Run web server on port PORT") - config.add_param("port", int, 45672, summ) - - summ = "Serve web pages using SSL" - config.add_param("ssl", bool, False, summ) - - config.init() - - if "-h" in sys.argv or "--help" in sys.argv: - config.print_usage("cumin") - sys.exit(0) - - if config.debug: - config.prt() - - do_main(config) - if __name__ == "__main__": if "--profile" in sys.argv: from profile import Profile Modified: mgmt/trunk/cumin/bin/cumin-bench =================================================================== --- mgmt/trunk/cumin/bin/cumin-bench 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/bin/cumin-bench 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,69 +1,14 @@ #!/usr/bin/python import sys, os -from time import time -from wooly.devel import BenchmarkHarness -from cumin import * -from cumin.test import * -from cumin.util import * +from cumin.tools import CuminBenchTool -def do_main(home, data, spec, hits, check_xml): - app = Cumin(home, data, spec) +def do_main(): + CuminBenchTool("cumin-bench").main() - app.enable_debug() - - try: - app.check() - except Exception, e: - if hasattr(e, "message"): - print e.message - - sys.exit(1) - - app.init() - - harness = BenchmarkHarness(app, check_xml) - - try: - try: - app.start() - - harness.run(hits) - finally: - app.stop() - except KeyboardInterrupt: - pass - def main(): - config = CuminConfig() - - summ = ("COUNT", "Stop after COUNT page hits") - config.add_param("hits", int, 1000, summ) - - summ = "Enable profiling" - config.add_param("profile", bool, False, summ) - - summ = "Check that page output is well-formed XML" - config.add_param("check-xml", bool, False, summ) - - config.init() - - if "-h" in sys.argv or "--help" in sys.argv: - config.print_usage("cumin-bench") - sys.exit(0) - - if config.debug: - config.prt() - - home = config.home - data = config.data - spec = config.spec - hits = config.hits - profile = config.profile - check_xml = config.check_xml - - if config.profile: + if "--profile" in sys.argv: from profile import Profile from pstats import Stats @@ -83,9 +28,7 @@ print "Using bias %f" % prof.bias try: - statement = "do_main('%s', '%s', '%s', %i, %r)" % \ - (home, data, spec, hits, check_xml) - + statement = "do_main()" prof.run(statement) raise KeyboardInterrupt() @@ -106,7 +49,10 @@ stats.strip_dirs() else: - do_main(home, data, spec, hits, check_xml) + do_main() if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + pass Modified: mgmt/trunk/cumin/bin/cumin-test =================================================================== --- mgmt/trunk/cumin/bin/cumin-test 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/bin/cumin-test 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,68 +1,11 @@ #!/usr/bin/python import sys, os -from time import time, sleep -from cumin import * -from cumin.test import * -from cumin.util import * +from cumin.tools import CuminTestTool -def do_main(config): - app = Cumin(config.home, config.data, config.spec) - - if config.debug: - app.enable_debug() - - try: - app.check() - except Exception, e: - if hasattr(e, "message"): - print e.message - - sys.exit(1) - - usr_sbin = "/usr/sbin" - if usr_sbin not in sys.path: - sys.path.append(usr_sbin) - - app.init() - - if config.broker: - host, port = parse_broker_addr(config.broker) - else: - # XXX change this to 49152..65535 when the underlying datatype - # supports it - host, port = "localhost", randint(16384, 32767) - - broker = TestBroker("qpidd", port) - broker.start() - - env = TestEnvironment(app, host, port, config.spec) - env.init(); - - app.start() - try: - session = env.run_test(MainTest(env)) - session.report(sys.stdout) - finally: - app.stop() - -def main(): - config = CuminConfig() - - summ = ("ADDR", "Use existing broker at ADDR") - config.add_param("broker", str, None, summ) - - config.init() - - if "-h" in sys.argv or "--help" in sys.argv: - config.print_usage("cumin-test") - sys.exit(0) - - if config.debug: - config.prt() - - do_main(config) - if __name__ == "__main__": - main() + try: + CuminTestTool("cumin-test").main() + except KeyboardInterrupt: + pass Modified: mgmt/trunk/cumin/python/cumin/__init__.py =================================================================== --- mgmt/trunk/cumin/python/cumin/__init__.py 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/python/cumin/__init__.py 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,6 +1,7 @@ import sys, os, logging from random import randint +from parsley.config import Config, ConfigParameter from wooly import Application, Session, Page from wooly.pages import CssPage, JavascriptPage, ResourcePage from wooly.server import WebServer @@ -17,7 +18,6 @@ from stat import StatChartPage from action import ActionPage from user import LoginPage -from util import Config from wooly import Session @@ -172,43 +172,36 @@ hdef = os.path.normpath("/var/lib/cumin") self.home = os.environ.get("CUMIN_HOME", hdef) + sdef = os.path.normpath("/usr/share/amqp/amqp-0.10-qpid-errata.xml") + self.spec = os.environ.get("AMQP_SPEC", sdef) + if not os.path.isdir(self.home): raise Exception("Home path '%s' is not a directory") - sdef = os.path.normpath("/usr/share/amqp/amqp.0-10.xml") - spec = os.environ.get("AMQP_SPEC", sdef) + param = ConfigParameter(self, "data", str) + param.default = "postgresql://cumin@localhost/cumin" - summ = "Print this message" - self.add_param("help", bool, False, summ) + param = ConfigParameter(self, "log-file", str) + param.default = os.path.join(self.home, "log", "cumin.log") - summ = ("URI", "Connect to database at URI") - self.add_param("data", str, "postgresql://cumin@localhost/cumin", summ) + param = ConfigParameter(self, "log-level", str) + param.default = "warning" - summ = ("PATH", "Use AMQP spec file at PATH") - self.add_param("spec", str, spec, summ) + param = ConfigParameter(self, "debug", bool) + param.default = False - lpath = os.path.join(self.home, "log", "cumin.log") - summ = ("PATH", "Log to file at PATH") - self.add_param("log-file", str, lpath, summ) - - summ = ("LEVEL", "Log messages at or above LEVEL " + - "('debug', 'info', 'warning', 'error')") - self.add_param("log-level", str, "warning", summ) - - summ = "Enable debug mode; print debug logging to console" - self.add_param("debug", bool, False, summ) - def init(self): - self.load_defaults() - self.load_args(sys.argv) + super(CuminConfig, self).init() handler = logging.StreamHandler() log.addHandler(handler) self.load_file(os.path.join(self.home, "etc", "cumin.conf")) self.load_file(os.path.join(os.path.expanduser("~"), ".cumin.conf")) - self.load_args(sys.argv) log.removeHandler(handler) + self.init_logging() + + def init_logging(self): mlog = logging.getLogger("mint") level = self.get_log_level() Modified: mgmt/trunk/cumin/python/cumin/tools.py =================================================================== --- mgmt/trunk/cumin/python/cumin/tools.py 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/python/cumin/tools.py 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,12 +1,15 @@ import sys, os +from parsley.config import * from parsley.command import * +from wooly.devel import * from mint import * from getpass import getpass from psycopg2 import IntegrityError -from cumin import CuminConfig from util import * +from cumin import Cumin, CuminServer, CuminConfig +from test import * def prompt_password(): password = None @@ -55,57 +58,11 @@ def init(self): super(BaseCuminTool, self).init() - args = sys.argv[1:] - - try: - opts, remaining = self.parse_options(args) - except CommandException, e: - print "Error: %s" % e.message - e.command.print_help() - sys.exit(1) - self.config.init() - self.config.load_string_params(opts) def run(self): - if self.config.debug: - self.config.prt() + pass - args = sys.argv[1:] - - try: - opts, remaining = self.parse_options(args) - except CommandException, e: - print "Error: %s" % e.message - e.command.print_help() - sys.exit(1) - - try: - scommand = remaining[0] - except IndexError: - self.print_help() - sys.exit(1) - - try: - command = self.commands_by_name[scommand] - except KeyError: - print "Error: Command '%s' is unrecognized" % scommand - self.print_help() - sys.exit(1) - - try: - opts, args = command.parse(remaining) - - if "help" in opts: - command.print_help() - return - - command.run(opts, args) - except CommandException, e: - print "Error: %s" % e.message - e.command.print_help() - sys.exit(1) - def main(self): self.check() self.init() @@ -163,6 +120,45 @@ self.database.check() self.database.init() + def run(self): + if self.config.debug: + self.config.prt() + + args = sys.argv[1:] + + try: + opts, remaining = self.parse_options(args) + except CommandException, e: + print "Error: %s" % e.message + e.command.print_help() + sys.exit(1) + + try: + scommand = remaining[0] + except IndexError: + self.print_help() + sys.exit(1) + + try: + command = self.commands_by_name[scommand] + except KeyError: + print "Error: Command '%s' is unrecognized" % scommand + self.print_help() + sys.exit(1) + + try: + opts, args = command.parse(remaining) + + if "help" in opts: + command.print_help() + return + + command.run(opts, args) + except CommandException, e: + print "Error: %s" % e.message + e.command.print_help() + sys.exit(1) + class CreateSchema(Command): def run(self, opts, args): main = os.path.join(self.parent.config.home, "sql", "schema.sql") @@ -332,3 +328,208 @@ subject.syncUpdate() print "Password of user '%s' is changed" % subject.name + +class CuminServerTool(BaseCuminTool): + def __init__(self, name): + super(CuminServerTool, self).__init__(name) + + self.description = "Cumin web server" + + param = ConfigParameter(self.config, "addr", str) + param.default = "localhost" + + param = ConfigParameter(self.config, "port", int) + param.default = 45672 + + param = ConfigParameter(self.config, "ssl", bool) + param.default = False + + opt = CommandOption(self, "addr") + opt.argument = "ADDR" + opt.description = "Run web server at address ADDR" + + opt = CommandOption(self, "port") + opt.argument = "PORT" + opt.description = "Run web server on port PORT" + + opt = CommandOption(self, "ssl") + opt.description = "Serve web pages using SSL" + + def run(self): + try: + opts, args = self.parse(sys.argv) + except CommandException, e: + print "Error: %s" % e.message + e.command.print_help() + sys.exit(1) + + if "help" in opts: + self.print_help() + sys.exit(0) + + self.config.load_dict(opts) + + self.config.ssl = "ssl" in opts + + if self.config.debug: + self.config.prt() + + app = Cumin(self.config.home, self.config.data, self.config.spec) + + try: + app.check() + except Exception, e: + log.exception(e) + sys.exit(1) + + app.init() + + server = CuminServer(app, self.config.addr, self.config.port) + + if self.config.ssl: + cpath = os.path.join(self.config.home, "etc", "cumin.crt") + kpath = os.path.join(self.config.home, "etc", "cumin.key") + + if os.path.isfile(cpath): + server.set_ssl_cert_path(cpath) + else: + log.error("SSL certificate file '%s' not found" % cpath) + sys.exit(1) + + if os.path.isfile(kpath): + server.set_ssl_key_path(kpath) + else: + log.error("SSL key file '%s' not found" % kpath) + sys.exit(1) + + try: + app.start() + + try: + server.start() + finally: + server.stop() + finally: + app.stop() + +class CuminTestTool(BaseCuminTool): + def __init__(self, name): + super(CuminTestTool, self).__init__(name) + + self.description = "Cumin test tool" + + opt = CommandOption(self, "broker") + opt.argument = "ADDR" + opt.description = "Use existing broker at ADDR" + + def run(self): + try: + opts, args = self.parse(sys.argv) + except CommandException, e: + print "Error: %s" % e.message + e.command.print_help() + sys.exit(1) + + if "help" in opts: + self.print_help() + sys.exit(0) + + self.config.load_dict(opts) + + if self.config.debug: + self.config.prt() + + app = Cumin(self.config.home, self.config.data, self.config.spec) + + if self.config.debug: + app.enable_debug() + + try: + app.check() + except Exception, e: + if hasattr(e, "message"): + print e.message + + sys.exit(1) + + usr_sbin = "/usr/sbin" + if usr_sbin not in sys.path: + sys.path.append(usr_sbin) + + app.init() + + if "broker" in opts: + host, port = parse_broker_addr(opts["broker"]) + else: + host, port = "localhost", randint(49152, 65535) + + broker = TestBroker("qpidd", port) + broker.start() + + env = TestEnvironment(app, host, port, self.config.spec) + env.init(); + + app.start() + + try: + session = env.run_test(MainTest(env)) + session.report(sys.stdout) + finally: + app.stop() + +class CuminBenchTool(BaseCuminTool): + def __init__(self, name): + super(CuminBenchTool, self).__init__(name) + + self.description = "Cumin benchmarking tool" + + opt = CommandOption(self, "hits") + opt.argument = "COUNT" + opt.description = "Stop after COUNT page hits" + + opt = CommandOption(self, "profile") + opt.description = "Enable profiling" + + opt = CommandOption(self, "check-xml") + opt.description = "Check that page output is well-formed xml" + + def run(self): + try: + opts, args = self.parse(sys.argv) + except CommandException, e: + print "Error: %s" % e.message + e.command.print_help() + sys.exit(1) + + if "help" in opts: + self.print_help() + sys.exit(0) + + self.config.load_dict(opts) + + if self.config.debug: + self.config.prt() + + app = Cumin(self.config.home, self.config.data, self.config.spec) + + if self.config.debug: + app.enable_debug() + + try: + app.check() + except Exception, e: + if hasattr(e, "message"): + print e.message + + sys.exit(1) + + app.init() + + harness = BenchmarkHarness(app, "check-xml" in opts) + + app.start() + + try: + harness.run(int(opts.get("hits", "1000"))) + finally: + app.stop() Modified: mgmt/trunk/cumin/python/cumin/util.py =================================================================== --- mgmt/trunk/cumin/python/cumin/util.py 2008-10-01 15:11:54 UTC (rev 2558) +++ mgmt/trunk/cumin/python/cumin/util.py 2008-10-01 15:13:42 UTC (rev 2559) @@ -1,5 +1,4 @@ import sys -from ConfigParser import SafeConfigParser from datetime import datetime, timedelta from logging import getLogger from time import mktime, time, sleep @@ -67,127 +66,6 @@ ret = None return ret -class Config(object): - __log = getLogger("cumin.config") - - def __init__(self): - self.__params = list() - self.__params_by_name = dict() - - def add_param(self, name, type, default=None, summary=None): - param = self.Parameter(name, type, default, summary) - self.__params.append(param) - self.__params_by_name[name] = param - - def unmarshal(self, param, string): - if param.type is int: - value = int(string) - elif param.type is bool: - value = string is None or \ - string.lower() in ("y", "yes", "t", "true", "1") - elif param.type is str: - value = string - else: - raise Exception("Invalid type %s" % type) - - return value - - def load_defaults(self): - for param in self.__params: - name = param.name.replace("-", "_") - - if hasattr(self, name): - raise Exception("Parameter '%s' already present" % name) - - setattr(self, name, param.default) - - def load_file(self, file): - conf = SafeConfigParser() - found = conf.read(file) - - if found: - self.__log.info("Read config file '%s'" % file) - else: - self.__log.info("Config file '%s' not found" % file) - - params = dict() - - if (conf.has_section("main")): - for key, value in conf.items("main"): - params[key] = value - - self.load_string_params(params) - - def load_args(self, argv): - params = dict() - key = None - - for arg in argv: - if arg.startswith("--"): - key = arg[2:] - params[key] = None - elif key: - params[key] = arg - key = None - - self.load_string_params(params) - - def load_string_params(self, params): - for name, value in params.items(): - param = self.__params_by_name.get(name) - - if param: - setattr(self, param.name.replace("-", "_"), - self.unmarshal(param, value)) - else: - self.__log.info("Ignoring unrecognized parameter '%s'" % name) - - def print_usage(self, cmd): - print "Usage: %s OPTIONS..." % cmd - print "Options:" - - for param in self.__params: - if param.summary: - if type(param.summary) is str: - opt = "--%s" % param.name - text = param.summary - else: - opt = "--%s %s" % (param.name, param.summary[0]) - text = param.summary[1] - - main = " %-18s %s" % (opt, text) - extra = "" - - if param.default not in (None, False): - extra = "(default %s)" % param.default - - if len(main) + len(extra) > 79: - print main - print " %18s %s" % ("", extra) - else: - print main, extra - - def prt(self): - print "Configuration:" - - for param in self.__params: - value = getattr(self, param.name.replace("-", "_")) - - if value == param.default: - flag = " [default]" - else: - flag = "" - - print " %s = %s%s" % (param.name, value, flag) - - class Parameter(object): - def __init__(self, name, type, default=None, summary=None): - self.name = name - self.type = type - self.default = default - self.summary = summary - - def wait(predicate, timeout=30): start = time()