[rhmessaging-commits] rhmessaging commits: r3849 - in mgmt/trunk/rosemary: python/rosemary and 1 other directory.

Author: justi9 Date: 2010-02-11 10:25:37 -0500 (Thu, 11 Feb 2010) New Revision: 3849 Added: mgmt/trunk/rosemary/python/rosemary/sqltype.py Removed: mgmt/trunk/rosemary/python/rosemary/types.py Modified: mgmt/trunk/rosemary/bin/rosemary-test mgmt/trunk/rosemary/python/rosemary/model.py mgmt/trunk/rosemary/python/rosemary/sqlmodel.py mgmt/trunk/rosemary/python/rosemary/sqloperation.py Log: * Improve modeling of sql types * Correct the string interpolation syntax in sql operations * Add some sql debugging * On References, save the referenced class Modified: mgmt/trunk/rosemary/bin/rosemary-test =================================================================== --- mgmt/trunk/rosemary/bin/rosemary-test 2010-02-11 15:22:21 UTC (rev 3848) +++ mgmt/trunk/rosemary/bin/rosemary-test 2010-02-11 15:25:37 UTC (rev 3849) @@ -74,7 +74,7 @@ if __name__ == "__main__": model = RosemaryModel() - model.load_qmf_dir(os.path.join(os.environ["ROSEMARY_HOME"], "xml")) + model.load_xml_dir(os.path.join(os.environ["ROSEMARY_HOME"], "xml")) model.init() if len(sys.argv) == 1: Modified: mgmt/trunk/rosemary/python/rosemary/model.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/model.py 2010-02-11 15:22:21 UTC (rev 3848) +++ mgmt/trunk/rosemary/python/rosemary/model.py 2010-02-11 15:25:37 UTC (rev 3849) @@ -1,6 +1,6 @@ from sqloperation import * from sqlmodel import * -from types import * +from sqltype import * from util import * log = logging.getLogger("rosemary.model") @@ -12,16 +12,16 @@ self.sql_model = SqlModel() - def load_qmf_dir(self, dir): - assert os.path.isdir(dir) + def load_xml_dir(self, path): + assert os.path.isdir(path) - for name in os.listdir(dir): - path = os.path.join(dir, name) + for name in os.listdir(path): + file_path = os.path.join(path, name) - if path.endswith(".xml") and os.path.isfile(path): - self.load_qmf_metadata(path) + if file_path.endswith(".xml") and os.path.isfile(file_path): + self.load_xml_file(file_path) - def load_qmf_metadata(self, path): + def load_xml_file(self, path): tree = ElementTree() file = open(path, "r") @@ -93,9 +93,13 @@ self.sql_table = SqlTable(self.package.sql_schema, self.name) + self.sql_qmf_columns = None + self.sql_select = SqlSelectItem(self.sql_table) + self.sql_select_by_object_id = SqlSelectItemByObjectId(self.sql_table) self.sql_insert = SqlInsertItem(self.sql_table) self.sql_update = SqlUpdateItem(self.sql_table) + self.sql_delete = SqlDeleteItem(self.sql_table) name = "%sStats" % self.name @@ -150,39 +154,40 @@ meth.init() def add_id_columns(self): - id_col = SqlColumn(self.sql_table, "_id", "serial") + id_col = SqlColumn(self.sql_table, "_id", sql_serial8) self.sql_table.key_column = id_col name = "%s_pk" % self.name SqlPrimaryKeyConstraint(self.sql_table, name, (id_col,)) - stats_id_col = SqlColumn(self.sql_stats_table, "_id", "serial") + stats_id_col = SqlColumn(self.sql_stats_table, "_id", sql_serial8) self.sql_stats_table.key_column = stats_id_col name = "%s_pk" % self.sql_stats_table.name SqlPrimaryKeyConstraint(self.sql_stats_table, name, (stats_id_col,)) - parent_col = SqlColumn(self.sql_stats_table, "_parent_id", "int4") + parent_col = SqlColumn(self.sql_stats_table, "_parent_id", sql_int8) parent_col.foreign_key_column = id_col def add_qmf_columns(self): - agent_col = SqlColumn(self.sql_table, "_qmf_agent_id", "text") - object_col = SqlColumn(self.sql_table, "_qmf_object_id", "text") + agent = SqlColumn(self.sql_table, "_qmf_agent_id", sql_text) + object = SqlColumn(self.sql_table, "_qmf_object_id", sql_text) name = "%s_qmf_ids_uq" % self.name - SqlUniqueConstraint(self.sql_table, name, (agent_col, object_col)) + SqlUniqueConstraint(self.sql_table, name, (agent, object)) - SqlColumn(self.sql_table, "_qmf_session_id", "text") - SqlColumn(self.sql_table, "_qmf_update_time", "timestamp") - SqlColumn(self.sql_table, "_qmf_create_time", "timestamp") + session = SqlColumn(self.sql_table, "_qmf_session_id", sql_text) + session.nullable = True - col = SqlColumn(self.sql_table, "_qmf_delete_time", "timestamp") - col.nullable = True + update = SqlColumn(self.sql_table, "_qmf_update_time", sql_timestamp) + create = SqlColumn(self.sql_table, "_qmf_create_time", sql_timestamp) - update_col = SqlColumn \ - (self.sql_stats_table, "_qmf_update_time", "timestamp") + delete = SqlColumn(self.sql_table, "_qmf_delete_time", sql_timestamp) + delete.nullable = True + self.sql_qmf_columns = (agent, object, session, update, create, delete) + def load_object(self, cursor, id): self.sql_select.execute(cursor, self.sql_table.columns, {"id": id}) @@ -239,11 +244,13 @@ self.cls.references.append(self) self.cls.references_by_name[self.name] = self + self.that_cls = None + def init(self): assert self.references try: - that_cls = self.cls.package.classes_by_name[self.references] + self.that_cls = self.cls.package.classes_by_name[self.references] except KeyError: log.warn("Reference to '%s' invalid", self.references) @@ -251,8 +258,8 @@ name = "_%s_id" % self.name - self.sql_column = SqlColumn(self.cls.sql_table, name, "int4") - self.sql_column.foreign_key_column = that_cls.sql_table.key_column + self.sql_column = SqlColumn(self.cls.sql_table, name, sql_int4) + self.sql_column.foreign_key_column = self.that_cls.sql_table.key_column self.sql_column.nullable = self.is_optional class RosemaryProperty(RosemaryAttribute): @@ -265,7 +272,7 @@ def init(self): assert not self.references - type = sql_types_by_qmf_type[self.type] + type = sql_types_by_qmf_type_string[self.type] self.sql_column = SqlColumn(self.cls.sql_table, self.name, type) self.sql_column.nullable = self.is_optional @@ -280,7 +287,7 @@ def init(self): assert not self.references - type = sql_types_by_qmf_type[self.type] + type = sql_types_by_qmf_type_string[self.type] self.sql_column = SqlColumn(self.cls.sql_stats_table, self.name, type) Modified: mgmt/trunk/rosemary/python/rosemary/sqlmodel.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqlmodel.py 2010-02-11 15:22:21 UTC (rev 3848) +++ mgmt/trunk/rosemary/python/rosemary/sqlmodel.py 2010-02-11 15:25:37 UTC (rev 3849) @@ -98,15 +98,15 @@ tokens = list() tokens.append("\"%s\"" % self.name) - tokens.append(self.type) + tokens.append(self.type.literal) if not self.nullable: tokens.append("not null") if self.foreign_key_column: - table = self.foreign_key_column.table.name + table = self.foreign_key_column.table.identifier col = self.foreign_key_column.name - expr = "references \"%s\"(\"%s\")" + expr = "references %s(\"%s\")" tokens.append(expr % (table, col)) @@ -149,11 +149,3 @@ args = (self.name, self.columns[0].table.name, cols) out.write(" create index \"%s\" on \"%s\"(%s)\n" % args) - -class SqlRelation(object): - def __init__(self, model, name, columns): - self.name = name - self.columns = columns - - self.model.relations.append(self) - self.model.relations_by_name[self.name] = self Modified: mgmt/trunk/rosemary/python/rosemary/sqloperation.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqloperation.py 2010-02-11 15:22:21 UTC (rev 3848) +++ mgmt/trunk/rosemary/python/rosemary/sqloperation.py 2010-02-11 15:25:37 UTC (rev 3849) @@ -1,42 +1,55 @@ from sqlmodel import * from util import * +log = logging.getLogger("rosemary.sqloperation") + class SqlOperation(object): def __init__(self, table): self.table = table def execute(self, cursor, columns, values): - text = self.emit(names) - results = cursor.execute(text, values) - return results + text = self.emit(columns) + log.debug("Sql text: %s", text) + log.debug("Sql values: %s", values) + + cursor.execute(text, values) + class SqlSelectItem(SqlOperation): def emit(self, columns): cols = ", ".join([x.identifier for x in columns]) args = (cols, self.table.identifier) - return "select %s from %s where _id = %%(id)" % args + return "select %s from %s where _id = %%(id)s" % args +class SqlSelectItemByObjectId(SqlOperation): + def emit(self, columns): + cols = ", ".join([x.identifier for x in columns]) + args = (cols, self.table.identifier) + + return "select %s from %s where _qmf_object_id = %%(_qmf_object_id)s" \ + % args + class SqlInsertItem(SqlOperation): def emit(self, columns): names = [x.name for x in columns] cols = ", ".join(["\"%s\"" % x for x in names]) - vals = ", ".join(["%%(%s)" % x for x in names]) + vals = ", ".join(["%%(%s)s" % x for x in names]) args = (self.table.identifier, cols, vals) return "insert into %s (%s) values (%s)" % args class SqlUpdateItem(SqlOperation): def emit(self, columns): - exprs = ["\"%s\" = %%(%s)" % (x.name, x.name) for x in columns] + exprs = ["\"%s\" = %%(%s)s" % (x.name, x.name) for x in columns] exprs = ", ".join(exprs) args = (self.table.identifier, exprs) - return "update %s set %s where _id = %%(id)" % args + return "update %s set %s where _id = %%(id)s" % args class SqlDeleteItem(SqlOperation): def emit(self, columns): - return "delete from %s where _id = %%(id)" % self.table.identifier + return "delete from %s where _id = %%(id)s" % self.table.identifier class SqlQuery(SqlOperation): def __init__(self, table): Added: mgmt/trunk/rosemary/python/rosemary/sqltype.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/sqltype.py (rev 0) +++ mgmt/trunk/rosemary/python/rosemary/sqltype.py 2010-02-11 15:25:37 UTC (rev 3849) @@ -0,0 +1,76 @@ +import pickle + +from psycopg2 import TimestampFromTicks +from util import * + +class SqlType(object): + def __init__(self, literal): + self.literal = literal + + def adapt_value(self, value): + return value + +class TimestampType(SqlType): + def adapt_value(self, value): + if value is not None: + return TimestampFromTicks(value / 1000000000) + +class PickleType(SqlType): + def adapt_value(self, value): + if value is not None: + return pickle.dumps(value) + +class StringType(SqlType): + def adapt_value(self, value): + if value is not None: + return str(value) + +sql_bool = SqlType("bool") +sql_float4 = SqlType("float4") +sql_float8 = SqlType("float8") +sql_int2 = SqlType("int2") +sql_int4 = SqlType("int4") +sql_int8 = SqlType("int8") +sql_numeric_19 = SqlType("numeric(19)") +sql_serial4 = SqlType("serial4") +sql_serial8 = SqlType("serial8") +sql_text = SqlType("text") +sql_timestamp = TimestampType("timestamp") + +sql_pickle = PickleType("text") +sql_string = StringType("text") + +__mappings = ( + (sql_bool, 11, "bool"), + (sql_float4, 12, "float"), + (sql_float8, 13, "double"), + (sql_int2, 1, "count8"), + (sql_int2, 1, "hilo8"), + (sql_int2, 1, "uint8"), + (sql_int2, 16,"int8"), + (sql_int2, 17, "int16"), + (sql_int4, 18, "int32"), + (sql_int4, 2, "count16"), + (sql_int4, 2, "hilo16"), + (sql_int4, 2, "uint16"), + (sql_int8, 19, "int64"), + (sql_int8, 3, "count32"), + (sql_int8, 3, "hilo32"), + (sql_int8, 3, "mma32"), + (sql_int8, 3, "uint32"), + (sql_numeric_19, 4, "count64"), + (sql_numeric_19, 4, "hilo64"), + (sql_numeric_19, 4, "mma64"), + (sql_numeric_19, 4, "mmaTime"), + (sql_numeric_19, 4, "uint64"), + (sql_numeric_19, 9, "deltaTime"), + (sql_string, 10, "objId"), + (sql_string, 14, "uuid"), + (sql_pickle, 15, "map"), + (sql_text, 6, "sstr"), + (sql_text, 7, "lstr"), + (sql_timestamp, 8, "absTime"), + ) + +sql_types_by_qmf_type_code = dict([(x[1], x[0]) for x in __mappings]) +sql_types_by_qmf_type_string = dict([(x[2], x[0]) for x in __mappings]) Deleted: mgmt/trunk/rosemary/python/rosemary/types.py =================================================================== --- mgmt/trunk/rosemary/python/rosemary/types.py 2010-02-11 15:22:21 UTC (rev 3848) +++ mgmt/trunk/rosemary/python/rosemary/types.py 2010-02-11 15:25:37 UTC (rev 3849) @@ -1,33 +0,0 @@ -from util import * - -sql_types_by_qmf_type = { - "int8": "int2", - "int16": "int2", - "int32": "int4", - "int64": "int8", - "uint8": "int2", - "uint16": "int4", - "uint32": "int8", - "uint64": "numeric(19)", - "count8": "int2", - "count16": "int4", - "count32": "int8", - "count64": "numeric(19)", - "hilo8": "int2", - "hilo16": "int4", - "hilo32": "int8", - "hilo64": "numeric(19)", - "mma32": "int8", - "mma64": "numeric(19)", - "float": "float4", - "double": "float8", - "bool": "bool", - "sstr": "text", - "lstr": "text", - "absTime": "timestamp", - "deltaTime": "numeric(19)", - "mmaTime": "numeric(19)", - "map": "text", - "objId": "text", - "uuid": "text", - }