Author: nunofsantos Date: 2007-10-26 13:59:32 -0400 (Fri, 26 Oct 2007) New Revision: 1168 Added: mgmt/mint/ mgmt/mint/config.xml mgmt/mint/schemaparser.py mgmt/mint/xml/ mgmt/mint/xml/MgmtSchema.xml Log: adding mint (SQL/python data model for management) Added: mgmt/mint/config.xml =================================================================== --- mgmt/mint/config.xml (rev 0) +++ mgmt/mint/config.xml 2007-10-26 17:59:32 UTC (rev 1168) @@ -0,0 +1,6 @@ +<config> + <configOption name="dsn" value="postgresql://localhost/" /> + <configOption name="pythonOutput" value="schema.out.py" /> + <configOption name="sqlOutput" value="schema.out.sql" /> + <configOption name="schemaXML" value="xml/MgmtSchema.xml" /> +</config> Added: mgmt/mint/schemaparser.py =================================================================== --- mgmt/mint/schemaparser.py (rev 0) +++ mgmt/mint/schemaparser.py 2007-10-26 17:59:32 UTC (rev 1168) @@ -0,0 +1,128 @@ +import mllib +from sqlobject import connectionForURI, sqlhub, MixedCaseUnderscoreStyle + +# mapping between xml schema types and database column types +dataTypesMap = {} +dataTypesMap["string"] = "VARCHAR(2000)" +dataTypesMap["bool"] = "BOOLEAN" +dataTypesMap["uint16"] = "INT2" +dataTypesMap["uint32"] = "INT4" +dataTypesMap["uint32_wm"] = "INT4" +dataTypesMap["uint64"] = "INT8" +dataTypesMap["ipAddress"] = "VARCHAR(100)" +dataTypesMap["enum"] = "VARCHAR(100) CHECK (" +dataTypesMap["fieldTable"] = "VARCHAR(2000)" + +# Generate Sql +def generateSql(name, elements, type): + sql = startTable(name, type) + for elem in elements: + sql += generateColumn(elem) + return endTable(sql) + +def startTable(name, type): + if (type == "config"): + actualName = name + else: + actualName = name + "_stats" + table = "CREATE TABLE " + actualName + " (\n" + table += " id BIGSERIAL PRIMARY KEY,\n" + if (type == "inst"): + table += " " + name + "_id BIGINT REFERENCES " + name + " ,\n" + table += generateTimestampColumn("rec") + if (type == "config"): + table += generateTimestampColumn("creation") + table += generateTimestampColumn("deletion") + return table + +def generateTimestampColumn(col): + column = " " + col + "_time TIMESTAMP,\n" + return column + +def generateColumn(elem, suffix=""): + actualName = style.pythonAttrToDBColumn(elem["@name"] + suffix) + if (elem[&quot;(a)type&quot;].startswith(&quot;enum&quot;)): + actualType = "enum" + params = actualName + " IN (" + for i in elem[&quot;(a)type&quot;].replace(&quot;enum(&quot;, "").replace(")", "").split(","): + params += "'" + i + "'," + params = params[:-1] + "))" + else: + actualType = elem["@type"] + params = "" + column = " " + actualName + " " + dataTypesMap[actualType] + params + ",\n" + if (elem[&quot;(a)type&quot;].endswith(&quot;_wm&quot;) and suffix == ""): + column += generateColumn(elem, "High") + column += generateColumn(elem, "Low") + return column + +def endTable(sql): + return sql[:-2] + "\n);\n\n" + + +# Generate Pyhon +def generatePython(name, elements): + name = style.dbColumnToPythonAttr(name) + name = name[0].upper() + name[1:] + python = startClass(name) + for elem in elements: + python += generateMethod(elem) + python = endClass(python) + python += startClass(name + "Stats") + python = endClass(python) + return python + +def startClass(name): + result = "class " + name + "(SQLObject):\n" + result += " class sqlmeta:\n" + result += " fromDatabase = True\n" + return result + +def generateMethod(elem): + if (elem["@desc"] != None): + comment = ' """' + elem["@desc"] + '"""\n' + else: + comment = "" + result = " def " + elem["@name"] + "():\n" + result += comment + result +=" pass\n\n" + return result + +def endClass(python): + return python + "\n" + + +# parse configuration file +config = mllib.xml_parse("config.xml") +configOptions = config.query["config/configOption"] +options = dict() +for opt in configOptions: + options[opt["@name"]] = opt["@value"] + + + +sqlFile = open(options["sqlOutput"], "w") +pythonFile = open(options["pythonOutput"], "w") + +conn = connectionForURI(options["dsn"]) +sqlhub.processConnection = conn + +schema = mllib.xml_parse(options["schemaXML"]) +objects = schema.query["schema/object"] + +style = MixedCaseUnderscoreStyle() + +pythonFile.write("from sqlobject import *\n\n" + \ + "conn = connectionForURI('" + options["dsn"] + "')\n" + \ + "sqlhub.processConnection = conn\n\n") + +for obj in objects: + actualName = "mgmt_" + obj["@name"] + sql = generateSql(actualName, obj.query["configElement"], "config") + sql += generateSql(actualName, obj.query["instElement"], "inst") + python = generatePython(actualName, obj.query["method"]) + sqlFile.write(sql) + pythonFile.write(python) + +sqlFile.close() +pythonFile.close() Added: mgmt/mint/xml/MgmtSchema.xml =================================================================== --- mgmt/mint/xml/MgmtSchema.xml (rev 0) +++ mgmt/mint/xml/MgmtSchema.xml 2007-10-26 17:59:32 UTC (rev 1168) @@ -0,0 +1,238 @@ +<schema version="0.1" date="10/22/2007"> + + <!-- Type information: + + Numeric types with "_wm" suffix are watermarked numbers. These are compound + values containing a current value, and a low and high water mark for the reporting + interval. The low and high water marks are set to the current value at the + beginning of each interval and track the minimum and maximum values of the statistic + over the interval respectively. + + Access rights for configuration elements: + + RO => Read Only + RC => Read/Create, can be set at create time only, read-only thereafter + RW => Read/Write + + If access rights are omitted for a configElement, they are assumed to be RO. + + --> + + <!-- Questions: Does C++ broker round-robin dests on queues? --> + + <!-- + =============================================================== + Server + =============================================================== + --> + <object name="server" schemaId="1"> + <configElement name="port" type="uint16" access="RO" desc="TCP Port for AMQP Service"/> + <configElement name="workerThreads" type="uint16" access="RO" desc="Thread pool size"/> + <configElement name="maxConns" type="uint16" access="RO" desc="Maximum allowed connections"/> + <configElement name="connBacklog" type="uint16" access="RO" desc="Connection backlog limit for listening socket"/> + <configElement name="stagingThreshold" type="uint32" access="RO" desc="Broker stages messages over this size to disk"/> + <configElement name="storeLib" type="string" access="RO" desc="Name of persistent storage library"/> + <configElement name="asyncStore" type="bool" access="RO" desc="Use async persistent store"/> + <configElement name="mgmtPubInterval" type="uint16" min="1" access="RW" desc="Interval for management broadcasts"/> + <configElement name="initialDiskPageSize" type="uint32" access="RO" desc="Number of disk pages allocated for storage"/> + <configElement name="initialPagesPerQueue" type="uint32" access="RO" desc="Number of disk pages allocated per queue"/> + <configElement name="clusterName" type="string" access="RO" + desc="Name of cluster this server is a member of, zero-length for standalone server"/> + + <!-- RT config/instrumentation TBD --> + + <instElement name="version" type="string" desc="Running software version"/> + + <method name="joinCluster"> + <arg name="clusterName" type="string"/> + </method> + + <method name="leaveCluster"/> + </object> + + <!-- + =============================================================== + Virtual Host + =============================================================== + --> + <object name="vhost" schemaId="2"> + <configElement name="name" type="string" access="RC" index="y"/> + </object> + + <!-- + =============================================================== + Queue + =============================================================== + --> + <object name="queue" schemaId="3"> + <configElement name="vhostRef" type="string" access="RC" index="y"/> + <configElement name="name" type="string" access="RC" index="y"/> + + <configElement name="durable" type="bool" access="RC"/> + <configElement name="autoDelete" type="bool" access="RC"/> + <configElement name="exclusive" type="bool" access="RC"/> + <configElement name="pageMemoryLimit" type="uint32" access="RO"/> + + <instElement name="diskPageSize" type="uint32"/> + <instElement name="diskPages" type="uint32"/> + <instElement name="diskAvailableSize" type="uint32"/> + + <instElement name="msgTotalEnqueues" type="uint64" desc="Total messages enqueued"/> + <instElement name="msgTotalDequeues" type="uint64" desc="Total messages dequeued"/> + <instElement name="msgTxEnqueues" type="uint64" desc="Transactional messages enqueued"/> + <instElement name="msgTxDequeues" type="uint64" desc="Transactional messages dequeued"/> + <instElement name="msgPersistEnqueues" type="uint64" desc="Persistent messages enqueued"/> + <instElement name="msgPersistDequeues" type="uint64" desc="Persistent messages dequeued"/> + <instElement name="msgDepth" type="uint32_wm" desc="Current size of queue in messages"/> + <instElement name="byteTotalEnqueues" type="uint64" desc="Total messages enqueued"/> + <instElement name="byteTotalDequeues" type="uint64" desc="Total messages dequeued"/> + <instElement name="byteTxEnqueues" type="uint64" desc="Transactional messages enqueued"/> + <instElement name="byteTxDequeues" type="uint64" desc="Transactional messages dequeued"/> + <instElement name="bytePersistEnqueues" type="uint64" desc="Persistent messages enqueued"/> + <instElement name="bytePersistDequeues" type="uint64" desc="Persistent messages dequeued"/> + <instElement name="byteDepth" type="uint32_wm" desc="Current size of queue in bytes"/> + <instElement name="enqueueTxStarts" type="uint64" desc="Total enqueue transactions started "/> + <instElement name="enqueueTxCommits" type="uint64" desc="Total enqueue transactions committed"/> + <instElement name="enqueueTxRejects" type="uint64" desc="Total enqueue transactions rejected"/> + <instElement name="enqueueTxCount" type="uint32_wm" desc="Current pending enqueue transactions"/> + <instElement name="dequeueTxStarts" type="uint64" desc="Total dequeue transactions started"/> + <instElement name="dequeueTxCommits" type="uint64" desc="Total dequeue transactions committed"/> + <instElement name="dequeueTxRejects" type="uint64" desc="Total dequeue transactions rejected"/> + <instElement name="dequeueTxCount" type="uint32_wm" desc="Current pending dequeue transactions"/> + <instElement name="consumers" type="uint32_wm" desc="Current consumers on queue"/> + <instElement name="unackedMessages" type="uint32_wm" desc="Messages consumed but not yet acked"/> + + <method name="purge" desc="Discard all messages on queue"/> + <method name="increaseDiskSize" desc="Increase number of disk pages allocated for this queue"> + <arg name="pages" type="uint32" desc="New total page allocation"/> + </method> + </object> + + <!-- + =============================================================== + Exchange + =============================================================== + --> + <object name="exchange" schemaId="4"> + <configElement name="vhostRef" type="string" access="RC" index="y"/> + <configElement name="name" type="string" access="RC" index="y"/> + <configElement name="type" type="string" access="RC"/> + + <instElement name="bindings" type="uint32_wm" desc="Current bindings"/> + <instElement name="msgReceives" type="uint64" desc="Total messages received"/> + <instElement name="msgDrops" type="uint64" desc="Total messages dropped (no matching key)"/> + <instElement name="msgRoutes" type="uint64" desc="Total routed messages"/> + <instElement name="byteReceives" type="uint64" desc="Total bytes received"/> + <instElement name="byteDrops" type="uint64" desc="Total bytes dropped (no matching key)"/> + <instElement name="byteRoutes" type="uint64" desc="Total routed bytes"/> + </object> + + <!-- + =============================================================== + Binding + =============================================================== + --> + <object name="binding" schemaId="5"> + <configElement name="queueRef" type="string" access="RC" index="y"/> + <configElement name="exchangeRef" type="string" access="RC" index="y"/> + <configElement name="bindingKey" type="string" access="RC"/> + <configElement name="arguments" type="fieldTable" access="RC"/> + + <instElement name="msgMatched" type="uint64"/> + </object> + + <!-- + =============================================================== + Client + =============================================================== + --> + <object name="client" schemaId="6"> + <configElement name="ipAddr" type="ipAddress" index="y"/> + <configElement name="port" type="uint16" index="y"/> + + <instElement name="authIdentity" type="string"/> + <instElement name="msgsProduced" type="uint64"/> + <instElement name="msgsConsumed" type="uint64"/> + <instElement name="bytesProduced" type="uint64"/> + <instElement name="bytesConsumed" type="uint64"/> + + <method name="close"/> + <method name="detach"/> + </object> + + <!-- + =============================================================== + Session + =============================================================== + --> + <object name="session" schemaId="7"> + <configElement name="vhostRef" type="string" index="y"/> + <configElement name="name" type="string" index="y"/> + <configElement name="clientRef" type="string" access="RO"/> + <configElement name="detachedLifespan" type="uint32" access="RO"/> + + <instElement name="attached" type="bool"/> + <instElement name="remainingLifespan" type="uint32"/> + <instElement name="framesOutstanding" type="uint32"/> + + <method name="solicitAck"/> + <method name="detach"/> + <method name="resetLifespan"/> + <method name="close"/> + </object> + + <!-- + =============================================================== + Destination + =============================================================== + --> + <object name="destination" schemaId="8"> + <configElement name="name" type="string" index="y"/> + + <instElement name="flowMode" type="enum(credit,window)"/> + <instElement name="maxMsgCredits" type="uint32"/> + <instElement name="maxByteCredits" type="uint32"/> + + <instElement name="msgCredits" type="uint32"/> + <instElement name="byteCredits" type="uint32"/> + + <method name="throttle" desc="Apply extra rate limiting to destination: 0 = Normal, 10 = Maximum"> + <arg name="strength" type="uint8" min="0" max="10"/> + </method> + <method name="stop"/> + <method name="start"/> + </object> + + <!-- + =============================================================== + Producer + =============================================================== + --> + <object name="producer" schemaId="9"> + <configElement name="sessionRef" type="string" index="y"/> + <configElement name="exchangeRef" type="string" index="y"/> + + <instElement name="destinationRef" type="string"/> + <instElement name="msgsProduced" type="uint64"/> + <instElement name="bytesProduced" type="uint64"/> + </object> + + <!-- + =============================================================== + Consumer + =============================================================== + --> + <object name="consumer" schemaId="10"> + <configElement name="sessionRef" type="string" index="y"/> + <configElement name="queueRef" type="string" index="y"/> + <configElement name="name" type="string" index="y"/> + + <instElement name="destinationRef" type="string"/> + <instElement name="msgsConsumed" type="uint64"/> + <instElement name="bytesConsumed" type="uint64"/> + <instElement name="unackedMessages" type="uint32_wm" desc="Messages consumed but not yet acked"/> + + <method name="close"/> + </object> +</schema> +