Author: nunofsantos Date: 2007-11-16 14:46:54 -0500 (Fri, 16 Nov 2007) New Revision: 1324 Added: mgmt/mint/parse Modified: mgmt/mint/config.xml mgmt/mint/python/mint/__init__.py mgmt/mint/schemaparser.py Log: new approach to code generation: generate Python from XML, and then use sqlobject-admin to generate the corresponding SQL Modified: mgmt/mint/config.xml =================================================================== --- mgmt/mint/config.xml 2007-11-16 17:50:29 UTC (rev 1323) +++ mgmt/mint/config.xml 2007-11-16 19:46:54 UTC (rev 1324) @@ -1,6 +1,5 @@ <config> <configOption name="dsn" value="postgresql://localhost/" /> <configOption name="pythonOutput" value="schema.py" /> - <configOption name="sqlOutput" value="schema.sql" /> <configOption name="schemaXML" value="xml/MgmtSchema.xml" /> </config> Added: mgmt/mint/parse =================================================================== --- mgmt/mint/parse (rev 0) +++ mgmt/mint/parse 2007-11-16 19:46:54 UTC (rev 1324) @@ -0,0 +1,24 @@ +#!/bin/bash + +# generate Python +echo "Generating Python code..." +python -c " +from schemaparser import * +parser = SchemaParser() +parser.generateCode() +" + +# generate SQL +echo "Generating SQL code..." + +SQLOUTPUT=schema.sql + +PYTHONPATH=. sqlobject-admin sql -m schema > $SQLOUTPUT +RESULT=`grep "circular reference" $SQLOUTPUT | wc -l` +if [[ $RESULT -gt 0 ]]; then + mv $SQLOUTPUT $SQLOUTPUT.1 + tail -n+3 $SQLOUTPUT.1 > $SQLOUTPUT + rm $SQLOUTPUT.1 +fi + +echo "Done." Property changes on: mgmt/mint/parse ___________________________________________________________________ Name: svn:executable + * Modified: mgmt/mint/python/mint/__init__.py =================================================================== --- mgmt/mint/python/mint/__init__.py 2007-11-16 17:50:29 UTC (rev 1323) +++ mgmt/mint/python/mint/__init__.py 2007-11-16 19:46:54 UTC (rev 1324) @@ -2,179 +2,156 @@ from datetime import * from sqlobject import * -from schema import * +import schema -class MintModel: +class OriginalIdDict: def __init__(self): - self.currentMethodId = 1 - self.outstandingMethodCalls = dict() - self.managedBrokers = dict() + self.idMap = dict() + + def set(self, idOriginal, obj): + self.idMap[idOriginal] = obj - def getQueueByOriginalId(self, id, create=False): - queue = None - try: - queue = MintQueue.selectBy(idOriginal=id)[:1][0] - except IndexError: - if (create): queue = MintQueue(idOriginal=id) - return queue + def getByOriginalId(self, objType, idOriginal, create=False, args={}): + if (idOriginal in self.idMap): + #print "\n\n=============== %s %d found\n\n" % (objType.__name__, idOriginal) + return self.idMap[idOriginal] + elif (create): + #print "\n\n=============== %s %d NOT found, creating\n\n" % (objType.__name__, idOriginal) + obj = objType.__new__(objType) + obj.__init__() + self.idMap[idOriginal] = obj + return obj + else: + #print "\n\n=============== %s %d NOT found, NOT creating\n\n" % (objType.__name__, idOriginal) + return None - def getQueueByName(self, name, vhost, create=False): - queue = None - try: - queue = MintQueue.selectBy(name=name, mintVhost=vhost)[:1][0] - except IndexError: - if (create): queue = MintQueue(name=name, mintVhost=vhost) - return queue + def getByIndexAttrib(self, objType, indexAttrib, indexValue, create=False, args={}): + ###FIX + return None + +class ConnectedBroker: + def __init__(self, managedBroker): + self.managedBroker = managedBroker + self.objs = OriginalIdDict() - def getVhostByName(self, name, broker, create=False): - vhost = None - try: - vhost = MintVhost.selectBy(name=name, mintBroker=broker)[:1][0] - except IndexError: - if (create): vhost = MintVhost(name=name, mintBroker=broker) - return vhost + def getByOriginalId(self, objType, idOriginal, create=False, args={}): + return self.objs.getByOriginalId(objType, idOriginal, create, args) - def getVhostByOriginalId(self, id, create=False): - vhost = None - try: - vhost = MintVhost.selectBy(idOriginal=id)[:1][0] - except IndexError: - if (create): vhost = MintVhost(idOriginal=id) - return vhost + def getByIndexAttrib(self, objType, indexAttrib, indexValue, parent, create=False, args={}): + return self.objs.getByIndexAttrib(objType, indexAttrib, indexValue, create, args) - def getBrokerByPort(self, port, system, create=False): - broker = None - try: - broker = MintBroker.selectBy(port=port, mintSystem=system)[:1][0] - except IndexError: - if (create): broker = MintBroker(port=port, mintSystem=system) - return broker - def getBrokerByOriginalId(self, id, create=False): - broker = None - try: - broker = MintBroker.selectBy(idOriginal=id)[:1][0] - except IndexError: - if (create): broker = MintBroker(idOriginal=id) - return broker - - def getSystemByOriginalId(self, id, create=False): - system = None - try: - system = MintSystem.selectBy(idOriginal=id)[:1][0] - except IndexError: - if (create): system = MintSystem(idOriginal=id) - return system - +class MintModel: + def __init__(self): + self.currentMethodId = 1 + self.outstandingMethodCalls = dict() + self.connectedBrokers = dict() + def sanitizeDict(self, d): for k in d.iterkeys(): - if (k.endswith("Id")): - d[self.convertKey(k)] = d.pop(k) - elif (k == "id"): - d[self.convertKey(k)] = d.pop(k) - for k in d.iterkeys(): - if (k.endswith("Ref")): - d[self.convertKey(k)] = d.pop(k) + if (k == "id"): + d[self.convertIdKey(k)] = d.pop(k) + elif (k.endswith("Ref")): + d[self.convertRefKey(k)] = d.pop(k) return d - def convertKey(self, k): - if (k == "id"): - return k + "Original" - if (k.endswith("Id")): - return k + "ent" - elif (k.endswith("Ref")): - oldK = k - k = k[0].upper() + k[1:] - return "mint" + k.replace("Ref", "ID") + def convertIdKey(self, k): + return "idOriginal" + def convertRefKey(self, k): + return k.replace("Ref", "") + def configCallback(self, broker, objectName, list, timestamps): print "\nCONFIG---------------------------------------------------" - print "broker=" + broker print objectName - print list result = None d = self.sanitizeDict(dict(list)) - d["managedBroker"] = self.managedBrokers[broker] - print d - + connectedBroker = self.connectedBrokers[broker] + d["managedBroker"] = broker d["recTime"] = datetime.fromtimestamp(timestamps[0]/1000000000) d["creationTime"] = datetime.fromtimestamp(timestamps[1]/1000000000) + print d + if (objectName == "queue"): - print "* QUEUE" - queue = self.getQueueByName(d["name"], self.getVhostByOriginalId(d.pop(self.convertKey("vhostRef"))), True) + d["vhost"] = connectedBroker.getByOriginalId(schema.Vhost, d.pop(self.convertRefKey("vhostRef"))) + queue = connectedBroker.getByOriginalId(schema.Queue, d["idOriginal"], create=True) queue.set(**d) - print "queue id = %d" % (queue.id) result = queue elif (objectName == "vhost"): - print "* VHOST" - vhost = self.getVhostByName(d["name"], self.getBrokerByOriginalId(d.pop(self.convertKey("brokerRef"))), True) + d["broker"] = connectedBroker.getByOriginalId(schema.Broker, d.pop(self.convertRefKey("brokerRef"))) + vhost = connectedBroker.getByOriginalId(schema.Vhost, d["idOriginal"], create=True) vhost.set(**d) - print "vhost id = %d" % (vhost.id) result = vhost elif (objectName == "broker"): - print "* BROKER" - d.pop(self.convertKey("systemRef")) - broker = self.getBrokerByPort(d["port"], self.getSystemByOriginalId("0"), True) +#FIX + d.pop(self.convertRefKey("systemRef")) +# d["system"] = connectedBroker.getByOriginalId(System, 0) + d["system"] = schema.System.selectBy(idOriginal=0)[:1][0] + connectedBroker.objs.set(0, d["system"]) +#FIX + broker = connectedBroker.getByOriginalId(schema.Broker, d["idOriginal"], create=True) broker.set(**d) - broker.sync() - print "broker id = %d" % (broker.id) result = broker print "END CONFIG---------------------------------------------------\n" return result def instCallback(self, broker, objectName, list, timestamps): print "\nINST---------------------------------------------------" - print "broker=" + broker print objectName - print list result = None d = self.sanitizeDict(dict(list)) + connectedBroker = self.connectedBrokers[broker] + d["recTime"] = datetime.fromtimestamp(timestamps[0]/1000000000) + print d if (objectName == "queue"): - print "* QUEUE" - queue = self.getQueueByOriginalId(d[self.convertKey("id")]) - d["mintQueue"] = queue.id - d["recTime"] = datetime.fromtimestamp(timestamps[0]/1000000000) - queueStats = MintQueueStats() + queue = connectedBroker.getByOriginalId(schema.Queue, d[self.convertIdKey("id")]) + d["queue"] = queue + queueStats = schema.QueueStats() queueStats.set(**d) d = dict() if (timestamps[2] != 0): d["deletionTime"] = datetime.fromtimestamp(timestamps[2]/1000000000) - d["mintQueueStats"] = queueStats + d["stats"] = queueStats queue.set(**d) - print queue.id result = queueStats elif (objectName == "vhost"): - print "* VHOST" + pass elif (objectName == "broker"): - print "* BROKER" - print "END INST---------------------------------------------------\n" + pass + print "END INST---------------------------------------------------\n" return result def methodCallback(self, broker, methodId, errorNo, errorText, args): print "\nMETHOD---------------------------------------------------" - print "broker=" + broker print "MethodId=%d" % (methodId) print "Error: %d %s" % (errorNo, errorText) + print "Args: " print args method = self.outstandingMethodCalls.pop(methodId) method(errorText, args) print "END METHOD---------------------------------------------------\n" - def addManagedBroker(self, host, port): + def connectToBroker(self, host, port): broker = ManagedBroker(host=host, port=port) label = "%s:%d" % (host, port) - self.managedBrokers[label] = broker + self.connectedBrokers[label] = ConnectedBroker(broker) broker.configListener(label, self.configCallback) broker.instrumentationListener (label, self.instCallback) broker.methodListener (label, self.methodCallback) broker.start() return label + def getConnectedBroker(self, label): + if (label in self.connectedBrokers): + return self.connectedBrokers[label].managedBroker + else: + return None + + # for compatibility + addManagedBroker = connectToBroker + def registerCallback(self, callback): self.currentMethodId += 1 methodId = self.currentMethodId self.outstandingMethodCalls[methodId] = callback return methodId - - def systems(self): - return MintSystem.select() Modified: mgmt/mint/schemaparser.py =================================================================== --- mgmt/mint/schemaparser.py 2007-11-16 17:50:29 UTC (rev 1323) +++ mgmt/mint/schemaparser.py 2007-11-16 19:46:54 UTC (rev 1324) @@ -1,145 +1,94 @@ import mllib -from sqlobject import connectionForURI, sqlhub, MixedCaseUnderscoreStyle +from sqlobject import * +class SchemaParser: + """parses broker XML schema""" -class SqlGenerator: - """generates SQL code from broker XML schema""" - - def __init__(self, style): - self.sqlOutput = "" - self.indexes = "" - self.currentTable = "" - self.style = style + def __init__(self): + self.options = dict() + self.parseConfigFile() + self.style = MixedCaseUnderscoreStyle() + self.pythonOutput = "from sqlobject import *\n" + self.pythonOutput += "from datetime import datetime\n" + self.pythonOutput += "conn = connectionForURI(\"%s\")\n" % (self.options["dsn"]) + self.pythonOutput += "sqlhub.processConnection = conn\n\n" + self.additional = "" + self.final = "\nclassToSchemaNameMap = dict()\n" # mapping between xml schema types and database column types # see xml/MintTypes.xml self.dataTypesMap = dict() - self.dataTypesMap["objId"] = "INT8" - self.dataTypesMap["uint8"] = self.dataTypesMap["hilo8"] = self.dataTypesMap["count8"] = "INT2" - self.dataTypesMap["uint16"] = self.dataTypesMap["hilo16"] = self.dataTypesMap["count16"] = "INT2" - self.dataTypesMap["uint32"] = self.dataTypesMap["hilo32"] = self.dataTypesMap["count32"] = "INT4" - self.dataTypesMap["uint64"] = self.dataTypesMap["hilo64"] = self.dataTypesMap["count64"] = "INT8" - self.dataTypesMap["bool"] = "BOOLEAN" - self.dataTypesMap["sstr"] = "VARCHAR(1000)" - self.dataTypesMap["lstr"] = "VARCHAR(4000)" - - def generate(self, name, elements, type): - self.startTable(name, type) - for elem in elements: - self.generateColumn(elem) - self.endTable(name, type) - - def startTable(self, name, type): - if (type == "config"): - actualName = name - else: - actualName = name + "_stats" - self.currentTable = actualName - self.sqlOutput += "\nCREATE TABLE %s (\n" % (actualName) - self.sqlOutput += " id BIGSERIAL PRIMARY KEY,\n" - self.sqlOutput += " id_original BIGINT,\n" - if (type == "config"): - self.sqlOutput += " %s_stats_id BIGINT,\n" % (name) - else: - self.sqlOutput += " %s_id BIGINT REFERENCES %s ,\n" % (name, name) - self.generateTimestampColumn("rec") - if (type == "config"): - self.generateTimestampColumn("creation") - self.generateTimestampColumn("deletion") - - def generateTimestampColumn(self, col): - self.sqlOutput += " %s_time TIMESTAMP,\n" % (col) + self.dataTypesMap["objId"] = "ForeignKey" + self.dataTypesMap["uint8"] = self.dataTypesMap["hilo8"] = self.dataTypesMap["count8"] = "SmallIntCol" + self.dataTypesMap["uint16"] = self.dataTypesMap["hilo16"] = self.dataTypesMap["count16"] = "SmallIntCol" + self.dataTypesMap["uint32"] = self.dataTypesMap["hilo32"] = self.dataTypesMap["count32"] = "IntCol" + self.dataTypesMap["uint64"] = self.dataTypesMap["hilo64"] = self.dataTypesMap["count64"] = "BigIntCol" + self.dataTypesMap["bool"] = "BoolCol" + self.dataTypesMap["sstr"] = self.dataTypesMap["lstr"] = "StringCol" - def generateIndex(self, name): - self.indexes += "\nCREATE INDEX %s_%s_index ON %s(%s);\n" % (self.currentTable, name, self.currentTable, name) - - def generateColumn(self, elem, suffix=""): - actualName = self.style.pythonAttrToDBColumn(elem["@name"] + suffix) - if (actualName.endswith("_id")): - # handle case where XML attrib ends in "Id", which confuses SqlObject into thinking it's a foreign key - actualName = actualName.replace("_id", "_ident") - if (elem[&quot;(a)type&quot;].startswith(&quot;enum&quot;)): - actualType = "enum" - params = actualName + " IN (" - for i in elem[&quot;(a)type&quot;].replace(&quot;enum(&quot;, "").replace(")", "").split(","): - params += "'" + i + "'," - params = params[:-1] + "))" - else: - actualType = elem["@type"] - params = "" - if (elem[&quot;(a)name&quot;].endswith(&quot;Ref&quot;)): - foreignKeyTable = "mint_" + actualName.replace("_ref", "") - self.sqlOutput += " %s_id BIGINT REFERENCES %s,\n" % (foreignKeyTable, foreignKeyTable) - else: - self.sqlOutput += " %s %s %s,\n" % (actualName, self.dataTypesMap[actualType], params) - if (elem[&quot;(a)type&quot;].startswith(&quot;hilo&quot;) and suffix == ""): - self.generateColumn(elem, "High") - self.generateColumn(elem, "Low") - if (elem["@index"] == "y"): - self.generateIndex(actualName) + def attrNameFromDbColumn(self, name, removeSuffix=""): + return self.style.dbColumnToPythonAttr(name.replace(removeSuffix, "")) + + def generateAttrib(self, attribName, attribType, params=""): + if (params.find("default") < 0): + if (params == ""): + params = "default=None" + else: + params += ", default=None" + self.pythonOutput += " %s = %s(%s)\n" % (attribName, attribType, params) - def endTable(self, name, type): - self.sqlOutput = self.sqlOutput[:-2] + "\n);\n" + self.indexes - if (type == "inst"): - self.sqlOutput += "\nALTER TABLE %s ADD FOREIGN KEY (%s_stats_id) REFERENCES %s_stats;\n" % (name, name, name) - self.currentTable = "" - self.indexes = "" + def generateTimestampAttrib(self, col): + self.generateAttrib(col + "Time", "TimestampCol") #, "default=datetime.min") - def getCode(self): - return self.sqlOutput - - - -class PythonGenerator: - """generates Python code from broker XML schema""" - - def __init__(self, style, dsn): - self.pythonOutput = "from sqlobject import *\n\n" - self.pythonOutput += "conn = connectionForURI(\"%s\")\n" % (dsn) - self.pythonOutput += "sqlhub.processConnection = conn\n\n" - self.additional = "" - self.style = style + def generateForeignKeyAttrib(self, name, reference): + params = "'%s', cascade='null'" % (reference) + self.generateAttrib(name, "ForeignKey", params) - def attrNameFromDbColumn(self, name, prefix="", removeSuffix=""): - name = self.style.dbColumnToPythonAttr(name.replace(removeSuffix, "")) - name = prefix + name[0].upper() + name[1:] - return name - - def generate(self, name, schemaName, schemaId, elements): - pythonName = self.attrNameFromDbColumn(name) - self.startClass(pythonName + "Stats") - self.endClass() - self.startClass(pythonName, schemaName, schemaId) - for elem in elements: - self.generateMethod(elem) - self.endClass(pythonName) + def generateHiLoAttrib(self, name, type): + self.generateAttrib(name, type) + self.generateAttrib(name + "Low", type) + self.generateAttrib(name + "High", type) - def generateAdditionalAttribs(self, name, elements): - name = self.attrNameFromDbColumn(name) + def generateClassAttribs(self, schemaName, elements): for elem in elements: - refName = elem["@name"] - if (refName.endswith("Ref")): - # generate foreign keys - refName = self.attrNameFromDbColumn(refName, "Mint", "Ref") - # add missing attribute (not added correctly with SqlObject 0.7.7; may need to be removed in later versions) - self.pythonOutput += " _SO_class_%s = %s\n" % (self.style.pythonAttrToDBColumn(refName).capitalize(), refName) - self.additional += "\n%s.sqlmeta.addJoin(MultipleJoin('%s" % (refName, name) - self.additional += "', joinMethodName='%ss'))" % (name.replace("Mint", "").lower()) + if (elem[&quot;(a)name&quot;].endswith(&quot;Ref&quot;)): + reference = self.style.dbTableToPythonClass(elem[&quot;(a)name&quot;]).replace(&quot;Ref&quot;, "") + attrib = reference[0].lower() + reference[1:] + self.generateForeignKeyAttrib(attrib, reference) + elif (elem[&quot;(a)type&quot;].startswith(&quot;hilo&quot;)): + self.generateHiLoAttrib(self.attrNameFromDbColumn(elem["@name"]), self.dataTypesMap[elem["@type"]]) + else: + args = "" + if (elem["@type"] == "sstr"): + args += "length=1000" + elif (elem["@type"] == "lstr"): + args += "length=4000" + self.generateAttrib(self.attrNameFromDbColumn(elem["@name"]), self.dataTypesMap[elem["@type"]], args) - def startClass(self, pythonName, schemaName="", schemaId=""): + def startClass(self, schemaName, stats=False): + if (stats): + pythonName = self.style.dbTableToPythonClass(schemaName + "_stats") + colPythonName = self.style.dbColumnToPythonAttr(schemaName) + keyPythonName = self.style.dbTableToPythonClass(schemaName) + else: + pythonName = self.style.dbTableToPythonClass(schemaName) + statsPythonName = self.style.dbTableToPythonClass(schemaName + "_stats") self.pythonOutput += "\nclass %s(SQLObject):\n" % (pythonName) - if (schemaName != ""): - self.pythonOutput += " schemaId = %d\n" % int(schemaId) - self.pythonOutput += " schemaName = \"%s\"\n" % schemaName - self.pythonOutput += " managedBroker = None\n" - self.pythonOutput += " class sqlmeta:\n" - self.pythonOutput += " fromDatabase = True\n" - + self.generateAttrib("idOriginal", "BigIntCol") + self.generateTimestampAttrib("rec") + if (stats): + self.generateForeignKeyAttrib(colPythonName, keyPythonName) + else: + self.generateTimestampAttrib("creation") + self.generateTimestampAttrib("deletion") + self.generateAttrib("managedBroker", "StringCol", "length=1000") + self.generateForeignKeyAttrib("stats", statsPythonName) + self.final += "classToSchemaNameMap['%s'] = '%s'\n" % (pythonName, schemaName) + def generateMethod(self, elem): if (elem["@desc"] != None): comment = ' """' + elem["@desc"] + '"""\n' else: comment = "" - formalArgs = ", " actualArgs = " actualArgs = dict()\n" for arg in elem.query["arg"]: @@ -149,36 +98,20 @@ formalArgs = formalArgs[:-2] else: formalArgs = "" - self.pythonOutput += " def %s(self, model, callbackMethod%s):\n" % (elem["@name"], formalArgs) + self.pythonOutput += "\n def %s(self, model, managedBrokerLabel, callbackMethod%s):\n" % (elem["@name"], formalArgs) self.pythonOutput += comment self.pythonOutput += actualArgs self.pythonOutput += " methodId = model.registerCallback(callbackMethod)\n" - self.pythonOutput += " self.managedBroker.method(methodId, self.idOriginal, self.schemaName, \"%s\", " % (elem["@name"]) + self.pythonOutput += " model.getConnectedBroker(managedBrokerLabel).method(methodId, self.idOriginal, classToSchemaNameMap[self.__class__.__name__], \"%s\", " % (elem["@name"]) self.pythonOutput += "args=actualArgs, packageName=\"qpid\")\n" - def endClass(self, name=""): + def endClass(self): if (self.additional != ""): self.pythonOutput += self.additional + "\n" self.additional = "" - if (name != "" and not name.endswith("Stats")): - # add missing attribute (not added correctly with SqlObject 0.7.7; may need to be removed in later versions) - self.pythonOutput += " _SO_class_%s_stats = %sStats\n" % (self.style.pythonAttrToDBColumn(name).capitalize(), name) + if (self.pythonOutput.endswith("(SQLObject):\n")): + self.pythonOutput += " pass\n" - def getCode(self): - return self.pythonOutput - - - -class SchemaParser: - """parses broker XML schema""" - - def __init__(self): - self.options = dict() - self.parseConfigFile() - self.style = MixedCaseUnderscoreStyle() - self.pythonGen = PythonGenerator(self.style, self.options["dsn"]) - self.sqlGen = SqlGenerator(self.style) - def parseConfigFile(self): config = mllib.xml_parse("config.xml") configOptions = config.query["config/configOption"] @@ -186,22 +119,20 @@ self.options[opt["@name"]] = opt["@value"] def generateCode(self): - conn = connectionForURI(self.options["dsn"]) - sqlhub.processConnection = conn - sqlFile = open(self.options["sqlOutput"], "w") - pythonFile = open(self.options["pythonOutput"], "w") + outputFile = open(self.options["pythonOutput"], "w") schema = mllib.xml_parse(self.options["schemaXML"]) classes = schema.query["schema/class"] for cls in classes: - actualName = "mint_" + cls["@name"] - self.sqlGen.generate(actualName, cls.query["configElement"], "config") - self.sqlGen.generate(actualName, cls.query["instElement"], "inst") - self.pythonGen.generate(actualName, cls["@name"], cls["@schemaId"], cls.query["method"]) - self.pythonGen.generateAdditionalAttribs(actualName, cls.query["configElement"]) - sqlFile.write(self.sqlGen.getCode()) - pythonFile.write(self.pythonGen.getCode()) - sqlFile.close() - pythonFile.close() + self.startClass(cls["@name"]) + self.generateClassAttribs(cls["@name"], cls.query["configElement"]) + for elem in cls.query["method"]: + self.generateMethod(elem) + self.endClass() + self.startClass(cls["@name"], stats=True) + self.generateClassAttribs(cls["@name"], cls.query["instElement"]) + self.endClass() + outputFile.write(self.pythonOutput + self.final) + outputFile.close() parser = SchemaParser()