[rhmessaging-commits] rhmessaging commits: r4032 - mgmt/newdata/mint/python/mint.

Author: justi9 Date: 2010-06-17 09:47:29 -0400 (Thu, 17 Jun 2010) New Revision: 4032 Modified: mgmt/newdata/mint/python/mint/update.py Log: * Use a distinct sample timeout to flow control samples * Move the sample drop logic to an earlier point * Tune the queue pressure sensor to 1000 items * Count an update that results in no database writes as dropped Modified: mgmt/newdata/mint/python/mint/update.py =================================================================== --- mgmt/newdata/mint/python/mint/update.py 2010-06-16 19:08:47 UTC (rev 4031) +++ mgmt/newdata/mint/python/mint/update.py 2010-06-17 13:47:29 UTC (rev 4032) @@ -189,15 +189,27 @@ cls = self.get_class() obj = self.get_object(cls, self.object.getObjectId().objectName) - if not obj._sync_time and not self.object.getProperties(): - # This is a sample for an object we don't have yet - stats.dropped += 1; return - update_time, create_time, delete_time = self.object.getTimestamps() update_time = datetime.fromtimestamp(update_time / 1000000000) - last_update_time = obj._qmf_update_time + now = datetime.now() + if self.object.getStatistics() and not self.object.getProperties(): + # Just stats; do we want it? + + if not obj._sync_time: + # We don't have this object yet + stats.dropped += 1; return + + if stats.enqueued - stats.dequeued > 1000: + if update_time < now - minutes_ago: + # The sample is too old + stats.dropped += 1; return + + if obj._sample_time and obj._sample_time > now - seconds_ago: + # The samples are too fidelitous + stats.dropped += 1; return + obj._qmf_update_time = update_time object_columns = list() @@ -221,32 +233,17 @@ statements.append(sql) + obj._sync_time = now + if sample_columns: - keep = True + sample_columns.append(cls.sql_samples_table._qmf_update_time) - if stats.enqueued - stats.dequeued > 100: - # There's some pressure, so consider dropping samples + sql = cls.sql_samples_insert.emit(sample_columns) + statements.append(sql) - now = datetime.now() - - if update_time < now - minutes_ago: - # The sample is too old - keep = False + stats.sampled += 1 + obj._sample_time = now - if last_update_time and last_update_time > now - seconds_ago: - # The samples are too fidelitous - keep = False - - if keep: - sample_columns.append(cls.sql_samples_table._qmf_update_time) - - sql = cls.sql_samples_insert.emit(sample_columns) - stats.sampled += 1 - - statements.append(sql) - else: - stats.dropped += 1 - if statements: text = "; ".join(statements) @@ -264,9 +261,9 @@ log.info("Sql row count: %i", cursor.rowcount) raise + else: + stats.dropped += 1 - obj._sync_time = datetime.now() - def get_class(self): class_key = self.object.getClassKey()