Author: justi9 Date: 2010-02-19 13:52:40 -0500 (Fri, 19 Feb 2010) New Revision: 3856 Added: mgmt/trunk/rosemary/xml/rosemary.xml Modified: mgmt/trunk/rosemary/python/rosemary/model.py mgmt/trunk/rosemary/python/rosemary/sqlmodel.py mgmt/trunk/rosemary/python/rosemary/sqloperation.py mgmt/trunk/rosemary/python/rosemary/sqltype.py Log: * Add deferred foreign key constraints * Add python types to the sql type mappings * Extend query support * Add __repr__s for better debugging * Store child schema elements directly on their parent classes as attributes * Load extended metadata from rosemary.xml * Keep track of inbound references to a column Modified: mgmt/trunk/rosemary/python/rosemary/model.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/model.py 2010-02-19 18:47:32 UTC (rev 3855) +++ mgmt/trunk/rosemary/python/rosemary/model.py 2010-02-19 18:52:40 UTC (rev 3856) @@ -15,12 +15,23 @@ def load_xml_dir(self, path): assert os.path.isdir(path) + extensions = os.path.join(path, "rosemary.xml") + for name in os.listdir(path): file_path = os.path.join(path, name) - if file_path.endswith(".xml") and os.path.isfile(file_path): + if not os.path.isfile(file_path): + continue + + if file_path == extensions: + continue + + if file_path.endswith(".xml"): self.load_xml_file(file_path) + if os.path.isfile(extensions): + self.load_extensions(extensions) + def load_xml_file(self, path): tree = ElementTree() file = open(path, "r") @@ -31,6 +42,16 @@ finally: file.close() + def load_extensions(self, path): + tree = ElementTree() + file = open(path, "r") + + try: + tree.parse(file) + self.extend(tree.getroot()) + finally: + file.close() + def load(self, elem): pkg = RosemaryPackage(self, elem.get("package")) pkg.load(elem) @@ -52,6 +73,11 @@ self.model.packages.append(self) self.model.packages_by_name[self.name] = self + mangled = self.name.replace(".", "_") + + if not hasattr(self.model, mangled): + setattr(self.model, mangled, self) + self.classes = list() self.classes_by_name = dict() @@ -71,6 +97,10 @@ for cls in self.classes: cls.init() + def __repr__(self): + args = (self.__class__.__name__, self.name) + return "%s(%s)" % args + class RosemaryClass(object): def __init__(self, package, name): self.package = package @@ -79,9 +109,14 @@ self.package.classes.append(self) self.package.classes_by_name[self.name] = self + if not hasattr(self.package, self.name): + setattr(self.package, self.name, self) + self.references = list() self.references_by_name = dict() + self.inbound_references = list() + self.properties = list() self.properties_by_name = dict() @@ -93,19 +128,28 @@ self.sql_table = SqlTable(self.package.sql_schema, self.name) + name = "%sStats" % self.name + + self.sql_stats_table = SqlTable(self.package.sql_schema, name) + + self.add_qmf_columns() + self.add_id_columns() + + self.id = RosemaryAttribute(self, "id") + self.id.sql_column = self.sql_table.key_column + self.sql_qmf_columns = None self.sql_select = SqlSelectItem(self.sql_table) - self.sql_select_by_object_id = SqlSelectItemByObjectId(self.sql_table) self.sql_insert = SqlInsertItem(self.sql_table) self.sql_update = SqlUpdateItem(self.sql_table) self.sql_delete = SqlDeleteItem(self.sql_table) - name = "%sStats" % self.name + self.sql_select_by_object_id = SqlSelectItemByObjectId(self.sql_table) - self.sql_stats_table = SqlTable(self.package.sql_schema, name) + def load(self, elem): + log.debug("Loading %s", self) - def load(self, elem): for child in elem.findall("property"): name = child.get("name") @@ -125,6 +169,8 @@ meth.load(child) def extend(self, elem): + log.debug("Extending %s", self) + for child in elem.findall("property"): prop = self.properties_by_name[child.get("name")] prop.extend(child) @@ -138,8 +184,7 @@ meth.extend(child) def init(self): - self.add_qmf_columns() - self.add_id_columns() + log.debug("Initializing %s", self) for ref in self.references: ref.init() @@ -169,6 +214,9 @@ parent_col = SqlColumn(self.sql_stats_table, "_parent_id", sql_int8) parent_col.foreign_key_column = id_col + name = "%s_fk" % parent_col.name + SqlForeignKeyConstraint(self.sql_stats_table, name, parent_col, id_col) + def add_qmf_columns(self): agent = SqlColumn(self.sql_table, "_qmf_agent_id", sql_text) object = SqlColumn(self.sql_table, "_qmf_object_id", sql_text) @@ -208,11 +256,18 @@ self.sql_delete.execute(cursor, (), {"id": object.id}) + def __repr__(self): + args = (self.__class__.__name__, self.package.name, self.name) + return "%s(%s,%s)" % args + class RosemaryAttribute(object): def __init__(self, cls, name): self.cls = cls self.name = name + if not hasattr(self.cls, self.name): + setattr(self.cls, self.name, self) + self.type = None self.references = None self.access = None @@ -237,6 +292,10 @@ def extend(self, elem): self.title = elem.findtext("title") + def __repr__(self): + args = (self.__class__.__name__, self.cls.name, self.name) + return "%s(%s,%s)" % args + class RosemaryReference(RosemaryAttribute): def __init__(self, cls, name): super(RosemaryReference, self).__init__(cls, name) @@ -249,18 +308,34 @@ def init(self): assert self.references + tokens = self.references.split(":", 1) + + if len(tokens) == 2: + pkg = self.cls.package.model.packages_by_name[tokens[0]] + cls = tokens[1] + else: + pkg = self.cls.package + cls = tokens[0] + try: - self.that_cls = self.cls.package.classes_by_name[self.references] + self.that_cls = pkg.classes_by_name[cls] + self.that_cls.inbound_references.append(self) except KeyError: - log.warn("Reference to '%s' invalid", self.references) + log.error("Reference to '%s' invalid", self.references) - return + raise name = "_%s_id" % self.name - self.sql_column = SqlColumn(self.cls.sql_table, name, sql_int4) - self.sql_column.foreign_key_column = self.that_cls.sql_table.key_column + col = SqlColumn(self.cls.sql_table, name, sql_int4) + col.foreign_key_column = self.that_cls.sql_table.key_column + + self.sql_column = col self.sql_column.nullable = self.is_optional + + name = "%s_fk" % col.name + + SqlForeignKeyConstraint(col.table, name, col, col.foreign_key_column) class RosemaryProperty(RosemaryAttribute): def __init__(self, cls, name): Modified: mgmt/trunk/rosemary/python/rosemary/sqlmodel.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqlmodel.py 2010-02-19 18:47:32 UTC (rev 3855) +++ mgmt/trunk/rosemary/python/rosemary/sqlmodel.py 2010-02-19 18:52:40 UTC (rev 3856) @@ -12,6 +12,10 @@ for schema in self.schemas: schema.write_create_ddl(out) + for schema in self.schemas: + for table in schema.tables: + table.write_alter_dll(out) + def write_drop_ddl(self, out): for schema in self.schemas: schema.write_drop_ddl(out) @@ -24,6 +28,11 @@ self.model.schemas.append(self) self.model.schemas_by_name[self.name] = self + mangled = self.name.replace(".", "_") + + if not hasattr(self.model, mangled): + setattr(self.model, mangled, self) + self.identifier = "\"%s\"" % self.name self.tables = list() @@ -46,6 +55,10 @@ def write_drop_ddl(self, out): out.write("drop schema %s cascade;\n" % self.identifier) + def __repr__(self): + args = (self.__class__.__name__, self.name) + return "%s(%s)" % args + class SqlTable(object): def __init__(self, schema, name): self.schema = schema @@ -54,6 +67,9 @@ self.schema.tables.append(self) self.schema.tables_by_name[self.name] = self + if not hasattr(self.schema, self.name): + setattr(self.schema, self.name, self) + self.identifier = "%s.\"%s\"" % (self.schema.identifier, self.name) self.key_column = None @@ -61,7 +77,7 @@ self.columns_by_name = dict() self.constraints = list() - self.constraints_by_name = dict() + self.deferred_constraints = list() def write_create_ddl(self, out): out.write(" create table \"%s\" (" % self.name) @@ -80,6 +96,22 @@ out.write("\n )\n") + def write_alter_dll(self, out): + if not self.deferred_constraints: + return + + out.write("alter table %s\n " % self.identifier) + + constraints = [x.get_ddl() for x in self.deferred_constraints] + + out.write(",\n ".join(constraints)) + + out.write("\n ;\n") + + def __repr__(self): + args = (self.__class__.__name__, self.schema.name, self.name) + return "%s(%s,%s)" % args + class SqlColumn(object): def __init__(self, table, name, type): self.table = table @@ -89,6 +121,9 @@ self.table.columns.append(self) self.table.columns_by_name[self.name] = self + if not hasattr(self.table, self.name): + setattr(self.table, self.name, self) + self.identifier = "\"%s\".\"%s\"" % (self.table.name, self.name) self.nullable = False @@ -103,15 +138,12 @@ if not self.nullable: tokens.append("not null") - if self.foreign_key_column: - table = self.foreign_key_column.table.identifier - col = self.foreign_key_column.name - expr = "references %s(\"%s\")" + return " ".join(tokens) - tokens.append(expr % (table, col)) + def __repr__(self): + args = (self.__class__.__name__, self.table.name, self.name) + return "%s(%s,%s)" % args - return " ".join(tokens) - class SqlTableConstraint(object): def __init__(self, table, name, columns): self.table = table @@ -119,7 +151,6 @@ self.columns = columns self.table.constraints.append(self) - self.table.constraints_by_name[self.name] = name class SqlPrimaryKeyConstraint(SqlTableConstraint): def get_ddl(self): @@ -133,6 +164,27 @@ return "constraint \"%s\" unique (%s)" % (self.name, cols) +class SqlForeignKeyConstraint(object): + def __init__(self, table, name, this_column, that_column): + self.table = table + self.name = name + self.this_column = this_column + self.that_column = that_column + + self.table.deferred_constraints.append(self) + + def get_ddl(self): + tokens = list() + + tokens.append("add constraint \"%s\"" % self.name) + tokens.append("foreign key (\"%s\")" % self.this_column.name) + tokens.append("references %s" % self.that_column.table.identifier) + tokens.append("(\"%s\")" % self.that_column.name) + + # XXX tokens.append("on delete set null;\n") + + return " ".join(tokens) + class SqlIndex(object): def __init__(self, schema, name, columns): assert len(set([x.table for x in columns])) == 1 Modified: mgmt/trunk/rosemary/python/rosemary/sqloperation.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqloperation.py 2010-02-19 18:47:32 UTC (rev 3855) +++ mgmt/trunk/rosemary/python/rosemary/sqloperation.py 2010-02-19 18:52:40 UTC (rev 3856) @@ -7,23 +7,23 @@ def __init__(self, table): self.table = table - def execute(self, cursor, columns, values): - text = self.emit(columns) + def execute(self, cursor, columns, values, options=None): + text = self.emit(columns, options) - log.debug("Sql text: %s", text) + log.debug("Sql text:\n%s", text) log.debug("Sql values: %s", values) cursor.execute(text, values) class SqlSelectItem(SqlOperation): - def emit(self, columns): + def emit(self, columns, options=None): cols = ", ".join([x.identifier for x in columns]) args = (cols, self.table.identifier) return "select %s from %s where _id = %%(id)s" % args class SqlSelectItemByObjectId(SqlOperation): - def emit(self, columns): + def emit(self, columns, options=None): cols = ", ".join([x.identifier for x in columns]) args = (cols, self.table.identifier) @@ -31,7 +31,7 @@ % args class SqlInsertItem(SqlOperation): - def emit(self, columns): + def emit(self, columns, options=None): names = [x.name for x in columns] cols = ", ".join(["\"%s\"" % x for x in names]) vals = ", ".join(["%%(%s)s" % x for x in names]) @@ -40,7 +40,7 @@ return "insert into %s (%s) values (%s)" % args class SqlUpdateItem(SqlOperation): - def emit(self, columns): + def emit(self, columns, options=None): exprs = ["\"%s\" = %%(%s)s" % (x.name, x.name) for x in columns] exprs = ", ".join(exprs) args = (self.table.identifier, exprs) @@ -48,30 +48,74 @@ return "update %s set %s where _id = %%(id)s" % args class SqlDeleteItem(SqlOperation): - def emit(self, columns): + def emit(self, columns, options=None): return "delete from %s where _id = %%(id)s" % self.table.identifier class SqlQuery(SqlOperation): def __init__(self, table): super(SqlQuery, self).__init__(table) + self.order_by = self.OrderBy() + self.limit = self.Limit() + self.joins = list() - def emit(self, columns, parameters=None): + def emit(self, columns, options): tokens = list() - cols = ",".join(["\n %s" % x.identifier for x in columns]) - tokens.append("select %s" % cols) + cols = list() + for column in columns: + if isinstance(column, SqlColumn): + cols.append(column.identifier) + else: + cols.append(str(column)) + + tokens.append("select %s" % ", ".join(cols)) + tokens.append("from %s" % self.table.identifier) for join in self.joins: tokens.append(join.emit()) + if options: + if options.sort_column: + tokens.append(self.order_by.emit(options.sort_column, + options.sort_ascending)) + + tokens.append(self.limit.emit(options.limit, options.offset)) + return "%s\n" % "\n".join(tokens) + + class OrderBy(object): + def emit(self, column, ascending): + if ascending: + direction = "asc" + else: + direction = "desc" + + return "order by %s %s" % (column.identifier, direction) + + class Limit(object): + def emit(self, limit, offset): + if limit is None: + limit = "all" + + return "limit %s offset %i" % (str(limit), offset) + +class SqlQueryOptions(object): + def __init__(self): + self.sort_column = None + self.sort_ascending = True + self.limit = None + self.offset = 0 class SqlQueryJoin(object): def __init__(self, query, this_column, that_column): + assert query + assert this_column + assert that_column + self.query = query self.this_column = this_column self.that_column = that_column Modified: mgmt/trunk/rosemary/python/rosemary/sqltype.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqltype.py 2010-02-19 18:47:32 UTC (rev 3855) +++ mgmt/trunk/rosemary/python/rosemary/sqltype.py 2010-02-19 18:52:40 UTC (rev 3856) @@ -1,11 +1,13 @@ import pickle +from datetime import datetime from psycopg2 import TimestampFromTicks from util import * class SqlType(object): - def __init__(self, literal): + def __init__(self, literal, python_type): self.literal = literal + self.python_type = python_type def adapt_value(self, value): return value @@ -15,30 +17,30 @@ if value is not None: return TimestampFromTicks(value / 1000000000) -class PickleType(SqlType): +class PickledType(SqlType): def adapt_value(self, value): if value is not None: return pickle.dumps(value) -class StringType(SqlType): +class StringIdType(SqlType): def adapt_value(self, value): if value is not None: return str(value) -sql_bool = SqlType("bool") -sql_float4 = SqlType("float4") -sql_float8 = SqlType("float8") -sql_int2 = SqlType("int2") -sql_int4 = SqlType("int4") -sql_int8 = SqlType("int8") -sql_numeric_19 = SqlType("numeric(19)") -sql_serial4 = SqlType("serial4") -sql_serial8 = SqlType("serial8") -sql_text = SqlType("text") -sql_timestamp = TimestampType("timestamp") +sql_bool = SqlType("bool", bool) +sql_float4 = SqlType("float4", float) +sql_float8 = SqlType("float8", float) +sql_int2 = SqlType("int2", long) +sql_int4 = SqlType("int4", long) +sql_int8 = SqlType("int8", long) +sql_uint8 = SqlType("numeric(19)", long) +sql_serial4 = SqlType("serial4", long) +sql_serial8 = SqlType("serial8", long) +sql_text = SqlType("text", str) +sql_timestamp = TimestampType("timestamp", datetime) -sql_pickle = PickleType("text") -sql_string = StringType("text") +sql_pickled_map = PickledType("text", dict) +sql_string_id = StringIdType("text", str) __mappings = ( (sql_bool, 11, "bool"), @@ -58,18 +60,18 @@ (sql_int8, 3, "hilo32"), (sql_int8, 3, "mma32"), (sql_int8, 3, "uint32"), - (sql_numeric_19, 4, "count64"), - (sql_numeric_19, 4, "hilo64"), - (sql_numeric_19, 4, "mma64"), - (sql_numeric_19, 4, "mmaTime"), - (sql_numeric_19, 4, "uint64"), - (sql_numeric_19, 9, "deltaTime"), - (sql_string, 10, "objId"), - (sql_string, 14, "uuid"), - (sql_pickle, 15, "map"), + (sql_pickled_map, 15, "map"), + (sql_string_id, 10, "objId"), + (sql_string_id, 14, "uuid"), (sql_text, 6, "sstr"), (sql_text, 7, "lstr"), (sql_timestamp, 8, "absTime"), + (sql_uint8, 4, "count64"), + (sql_uint8, 4, "hilo64"), + (sql_uint8, 4, "mma64"), + (sql_uint8, 4, "mmaTime"), + (sql_uint8, 4, "uint64"), + (sql_uint8, 9, "deltaTime"), ) sql_types_by_qmf_type_code = dict([(x[1], x[0]) for x in __mappings]) Added: mgmt/trunk/rosemary/xml/rosemary.xml =================================================================== --- mgmt/trunk/rosemary/xml/rosemary.xml (rev 0) +++ mgmt/trunk/rosemary/xml/rosemary.xml 2010-02-19 18:52:40 UTC (rev 3856) @@ -0,0 +1,13 @@ +<model> + <package name="org.apache.qpid.broker"> + <class name="Connection"> + <property name="remotePid"> + <title>Process ID</title> + </property> + + <property name="remoteParentPid"> + <title>Parent PID</title> + </property> + </class> + </package> +</model>