Author: justi9 Date: 2010-09-06 10:48:32 -0400 (Mon, 06 Sep 2010) New Revision: 4255 Modified: mgmt/newdata/cumin/python/cumin/main.py mgmt/newdata/cumin/python/cumin/model.py mgmt/newdata/cumin/python/cumin/stat.py mgmt/newdata/rosemary/python/rosemary/model.py mgmt/newdata/rosemary/python/rosemary/sqloperation.py Log: For bz623189, to provide data continuity of samples between agent restarts. Sample data is now keyed by qmf agent and object id, not by the local database ids used for objects. The latter change as agents come and go. This change requires a schema reload. Modified: mgmt/newdata/cumin/python/cumin/main.py =================================================================== --- mgmt/newdata/cumin/python/cumin/main.py 2010-09-03 20:23:44 UTC (rev 4254) +++ mgmt/newdata/cumin/python/cumin/main.py 2010-09-06 14:48:32 UTC (rev 4255) @@ -54,6 +54,8 @@ self.operator_email = None self.update_interval = 10 + # self.model.sql_logging_enabled = True + def check(self): log.info("Checking %s", self) Modified: mgmt/newdata/cumin/python/cumin/model.py =================================================================== --- mgmt/newdata/cumin/python/cumin/model.py 2010-09-03 20:23:44 UTC (rev 4254) +++ mgmt/newdata/cumin/python/cumin/model.py 2010-09-06 14:48:32 UTC (rev 4255) @@ -458,21 +458,27 @@ class SamplesSqlAdapter(SqlAdapter): qmf_update_col = '_qmf_update_time' - _parent_id_col = "_parent_id" - def __init__(self, app, table, parent_id): + def __init__(self, app, table, agent_id, object_id): super(SamplesSqlAdapter, self).__init__(app, table) - self.parent_id = parent_id + assert ";" not in agent_id + assert ";" not in object_id + col = self.table._qmf_agent_id + filter = SqlComparisonFilter(col, "'%s'" % agent_id) + self.query.add_filter(filter) + + col = self.table._qmf_object_id + filter = SqlComparisonFilter(col, "'%s'" % object_id) + self.query.add_filter(filter) + def get_sql_options(self, options): return options def avg_samples(self, stat, secs, interval, secs2): - table = self.table - stat_col = table._columns_by_name[stat.name] - updated_col = table._columns_by_name[self.qmf_update_col] - parent_col = table._columns_by_name[self._parent_id_col] + stat_col = self.table._columns_by_name[stat.name] + updated_col = self.table._columns_by_name[self.qmf_update_col] max_col = "max(%s) as interval_end" % updated_col.identifier value_col = "cast(avg(%s) as integer) as value" % stat_col.identifier @@ -484,10 +490,6 @@ columns.append(dev_col) self.columns = columns - self.query.filters = list() - filter = SqlComparisonFilter(parent_col, self.parent_id, "=") - self.query.add_filter(filter) - when = "now() - interval '%i seconds'" % int(secs + secs2) filter = SqlComparisonFilter(updated_col, when, ">=") self.query.add_filter(filter) @@ -504,25 +506,18 @@ samples = self.get_data({}, options) return samples - def samples(self, stat, secs, interval, method, secs2=0, delta=False): if method == "avg": return self.avg_samples(stat, secs, interval, secs2) - table = self.table - stat_col = table._columns_by_name[stat.name] - updated_col = table._columns_by_name[self.qmf_update_col] - parent_col = table._columns_by_name[self._parent_id_col] + stat_col = self.table._columns_by_name[stat.name] + updated_col = self.table._columns_by_name[self.qmf_update_col] columns = list() columns.append(updated_col.identifier) columns.append(stat_col.identifier) self.columns = columns - self.query.filters = list() - filter = SqlComparisonFilter(parent_col, self.parent_id, "=") - self.query.add_filter(filter) - if not delta: when = "now() - interval '%i seconds'" % int(secs + secs2) filter = SqlComparisonFilter(updated_col, when, ">=") Modified: mgmt/newdata/cumin/python/cumin/stat.py =================================================================== --- mgmt/newdata/cumin/python/cumin/stat.py 2010-09-03 20:23:44 UTC (rev 4254) +++ mgmt/newdata/cumin/python/cumin/stat.py 2010-09-06 14:48:32 UTC (rev 4255) @@ -108,7 +108,8 @@ object = self.object.get(session) params = list() - params.append("id=%i" % object._id) + params.append("agent=%s" % object._qmf_agent_id) + params.append("object=%s" % object._qmf_object_id) params.append("chart_id=%s" % self.render_id(session, None)) params.append("duration=%s" % self.duration.get(session)) params.append("rpkg=%s" % object._class._package._name) @@ -233,12 +234,13 @@ self.rosemary_package = Parameter(app, "rpkg") self.add_parameter(self.rosemary_package) - self.id = IntegerParameter(app, "id") - self.add_parameter(self.id) + self.agent_id = StringParameter(app, "agent") + self.add_parameter(self.agent_id) + self.object_id = StringParameter(app, "object") + self.add_parameter(self.object_id) + param = Parameter(app, "param") - self.add_parameter(param) - self.stats = ListParameter(app, "stat", param) self.add_parameter(self.stats) @@ -279,9 +281,16 @@ rclass = self.rosemary_class.get(session) rosemary_package = self.app.model._packages_by_name[rpackage] rosemary_class = rosemary_package._classes_by_name[rclass] - id = str(self.id.get(session)) - adapter = SamplesSqlAdapter(self.app, rosemary_class.sql_samples_table, id) + table = rosemary_class.sql_samples_table + agent_id = self.agent_id.get(session) + object_id = self.object_id.get(session) + + assert agent_id + assert object_id + + adapter = SamplesSqlAdapter(self.app, table, agent_id, object_id) + stats = [getattr(rosemary_class, x) for x in self.stats.get(session)] return (adapter, stats) Modified: mgmt/newdata/rosemary/python/rosemary/model.py =================================================================== --- mgmt/newdata/rosemary/python/rosemary/model.py 2010-09-03 20:23:44 UTC (rev 4254) +++ mgmt/newdata/rosemary/python/rosemary/model.py 2010-09-06 14:48:32 UTC (rev 4255) @@ -263,6 +263,8 @@ self._id.sql_column = self.sql_table.key_column + self.add_samples_sql() + for hdr in self._headers: hdr.init() @@ -286,8 +288,6 @@ ref.init() def add_sql_entities(self): - # Main table - name = "%s_id_seq" % self._name self.sql_sequence = SqlSequence(self._package.sql_schema, name) @@ -299,25 +299,9 @@ name = "%s_pk" % self._name SqlPrimaryKeyConstraint(self.sql_table, name, (id_col,)) - # Stats table - - name = "%s_samples" % self._name - self.sql_samples_table = SqlTable(self._package.sql_schema, name) - - name = "_parent_id" - parent_col = SqlColumn(self.sql_samples_table, name, sql_int8) - parent_col.foreign_key_column = id_col - - name = "%s_%s_idx" % (self.sql_samples_table._name, parent_col.name) - index = SqlIndex(self._package.sql_schema, name, (parent_col,)) - - name = "_qmf_update_time" - SqlColumn(self.sql_samples_table, name, sql_timestamp) - def add_sql_constraints(self): name = "%s_qmf_id_uq" % self._name cols = (self._qmf_agent_id.sql_column, self._qmf_object_id.sql_column) - SqlUniqueConstraint(self.sql_table, name, cols) def add_sql_operations(self): @@ -329,8 +313,20 @@ self.sql_update_object = SqlUpdateObject(self.sql_table) self.sql_delete_object = SqlDeleteObject(self.sql_table) - table = self.sql_samples_table + def add_samples_sql(self): + name = "%s_samples" % self._name + table = SqlTable(self._package.sql_schema, name) + self.sql_samples_table = table + + agent_col = SqlColumn(table, "_qmf_agent_id", sql_text) + object_col = SqlColumn(table, "_qmf_object_id", sql_text) + + name = "%s_qmf_id_idx" % table._name + SqlIndex(self._package.sql_schema, name, (agent_col, object_col)) + + SqlColumn(table, "_qmf_update_time", sql_timestamp) + self.sql_samples_insert = SqlInsertObjectSamples(table) self.sql_samples_delete = SqlDeleteObjectSamples(table) @@ -714,7 +710,7 @@ log.debug("Initializing %s", self) schema = self.cls._package.sql_schema - name = "%s_%s_index" % (self.cls._name, self.name) + name = "%s_%s_idx" % (self.cls._name, self.name) columns = [x.sql_column for x in self.attributes] self.sql_index = SqlIndex(schema, name, columns) Modified: mgmt/newdata/rosemary/python/rosemary/sqloperation.py =================================================================== --- mgmt/newdata/rosemary/python/rosemary/sqloperation.py 2010-09-03 20:23:44 UTC (rev 4254) +++ mgmt/newdata/rosemary/python/rosemary/sqloperation.py 2010-09-06 14:48:32 UTC (rev 4255) @@ -104,8 +104,11 @@ cols = ["\"%s\"" % x for x in names] vals = ["%%(%s)s" % x for x in names] - cols.append("\"_parent_id\"") - vals.append("%(_id)s") + cols.append("\"_qmf_agent_id\"") + cols.append("\"_qmf_object_id\"") + + vals.append("%(_qmf_agent_id)s") + vals.append("%(_qmf_object_id)s") cols = ", ".join(cols) vals = ", ".join(vals)