[rhmessaging-commits] rhmessaging commits: r3894 - in mgmt/newdata/cumin/python/cumin: grid and 2 other directories.

Author: eallen Date: 2010-04-06 16:04:09 -0400 (Tue, 06 Apr 2010) New Revision: 3894 Modified: mgmt/newdata/cumin/python/cumin/grid/collector.py mgmt/newdata/cumin/python/cumin/grid/job.py mgmt/newdata/cumin/python/cumin/grid/pool.py mgmt/newdata/cumin/python/cumin/grid/scheduler.py mgmt/newdata/cumin/python/cumin/grid/slot.py mgmt/newdata/cumin/python/cumin/grid/submitter.py mgmt/newdata/cumin/python/cumin/inventory/system.py mgmt/newdata/cumin/python/cumin/messaging/broker.py mgmt/newdata/cumin/python/cumin/messaging/brokerlink.py mgmt/newdata/cumin/python/cumin/messaging/connection.py mgmt/newdata/cumin/python/cumin/messaging/exchange.py mgmt/newdata/cumin/python/cumin/messaging/exchange.strings mgmt/newdata/cumin/python/cumin/messaging/queue.py mgmt/newdata/cumin/python/cumin/messaging/queue.strings mgmt/newdata/cumin/python/cumin/model.py mgmt/newdata/cumin/python/cumin/stat.py mgmt/newdata/cumin/python/cumin/stat.strings Log: PNG and Flash charts for new data adapter. Added Queue/Exchange/Connection overview pages Modified: mgmt/newdata/cumin/python/cumin/grid/collector.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/collector.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/collector.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -75,11 +75,19 @@ self.view = CollectorView(app, "view", self.object) self.add_mode(self.view) +class CollectorGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(CollectorGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("RunningJobs", "IdleJobs", + "HostsClaimed", "HostsUnclaimed", + "HostsOwner", "HostsTotal") + class CollectorOverview(Widget): def __init__(self, app, name, collector): super(CollectorOverview, self).__init__(app, name) - self.add_child(StatSet(app, "stats", collector, "general")) + self.add_child(GeneralStatSet(app, "stats", collector)) chart = self.JobStackedChart(app, "jobs", collector) chart.stats = ("RunningJobs", "IdleJobs") Modified: mgmt/newdata/cumin/python/cumin/grid/job.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/job.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/job.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -243,12 +243,13 @@ def __init__(self, app, name): super(JobGroupStats, self).__init__(app, name) - stats = JobGroupStatSet(app, "general", "general") - self.add_child(stats) + #stats = JobGroupStatSet(app, "general", "general") + #self.add_child(stats) def render_title(self, session): return "Statistics" +""" class JobGroupStatSet(StatSet): def __init__(self, app, name, category): super(JobGroupStatSet, self).__init__(app, name, category) @@ -296,6 +297,7 @@ value = self.get_value(group, state) percent = (value*1.0) / (jobs*1.0) * 100.0 return jobs and "%2.1f" % percent or "-" +""" class JobGroupJobSet(JobTab): def __init__(self, app, name, pool): Modified: mgmt/newdata/cumin/python/cumin/grid/pool.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/pool.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/pool.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -191,7 +191,7 @@ self.grid = PoolGridAttribute(app, "grid", self.pool) self.add_attribute(self.grid) - stats = StatSet(app, "collector_stats", self.collector, "general") + stats = CollectorGeneralStatSet(app, "collector_stats", self.collector) self.add_child(stats) stats = GridStats(app, "grid_stats", self.grid) @@ -229,13 +229,27 @@ def render_title(self, session): return "Slot state" +class GridGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(GridGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("NumJobs", "SubmitsInProgress", + "SubmitsQueued", "SubmitsAllowed", + "SubmitsWanted", "RunningJobs", "IdleJobs") + + def do_render(self, session): + grid = self.object.get(session) + + if grid: + return super(GridGeneralStatSet, self).do_render(session) + class GridStats(Widget): def __init__(self, app, name, grid): super(GridStats, self).__init__(app, name) self.grid = grid - stats = self.GridStatSet(app, "stats", self.grid, "general") + stats = GridGeneralStatSet(app, "stats", self.grid) self.add_child(stats) chart = self.GridJobChart(app, "jobs_chart", self.grid) @@ -266,13 +280,6 @@ def render_title(self, session): return "Grid submits" - class GridStatSet(StatSet): - def do_render(self, session): - grid = self.object.get(session) - - if grid: - return super(GridStats.GridStatSet, self).do_render(session) - class PoolSlotMap(SlotMap): def __init__(self, app, name, pool): super(PoolSlotMap, self).__init__(app, name) Modified: mgmt/newdata/cumin/python/cumin/grid/scheduler.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/scheduler.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/scheduler.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -141,11 +141,19 @@ details = CuminDetails(app, "details", scheduler) self.tabs.add_tab(details) +class SchedulerGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(SchedulerGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("NumUsers", "TotalRunningJobs", + "TotalIdleJobs", "TotalHeldJobs", + "TotalJobAds", "TotalRemovedJobs") + class SchedulerStats(Widget): def __init__(self, app, name, scheduler): super(SchedulerStats, self).__init__(app, name) - stats = StatSet(app, "general", scheduler, "general") + stats = SchedulerGeneralStatSet(app, "general", scheduler) self.add_child(stats) chart = self.UsersChart(app, "users", scheduler) Modified: mgmt/newdata/cumin/python/cumin/grid/slot.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/slot.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/slot.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -70,11 +70,17 @@ details = CuminDetails(app, "details", slot) self.tabs.add_tab(details) +class SlotLoadStatSet(StatSet): + def __init__(self, app, name, object): + super(SlotLoadStatSet, self).__init__(app, name, object) + + self.attrs = ("CondorLoadAvg", "LoadAvg"), + class SlotStats(Widget): def __init__(self, app, name, slot): super(SlotStats, self).__init__(app, name) - stats = StatSet(app, "general", slot, "load") + stats = SlotLoadStatSet(app, "general", slot) self.add_child(stats) job_info = SlotJobInfo(app, "job_info", slot, "JobInfo") Modified: mgmt/newdata/cumin/python/cumin/grid/submitter.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/submitter.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/grid/submitter.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -68,11 +68,17 @@ details = CuminDetails(app, "details", submitter) self.tabs.add_tab(details) +class SubmitterGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(SubmitterGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("RunningJobs", "IdleJobs", "HeldJobs") + class SubmitterStats(Widget): def __init__(self, app, name, submitter): super(SubmitterStats, self).__init__(app, name) - stats = StatSet(app, "general", submitter, "general") + stats = SubmitterGeneralStatSet(app, "general", submitter) self.add_child(stats) chart = self.JobsChart(app, "jobs", submitter) Modified: mgmt/newdata/cumin/python/cumin/inventory/system.py =================================================================== --- mgmt/newdata/cumin/python/cumin/inventory/system.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/inventory/system.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -67,11 +67,20 @@ def render_content(self, session, data): return "%0.3f" % data["load_avg"] +class SystemGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(SystemGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("memFree", "swapFree", + "loadAverage1Min", "loadAverage5Min", + "loadAverage10Min", "procTotal", + "procRunning") + class SystemStats(Widget): def __init__(self, app, name, system): super(SystemStats, self).__init__(app, name) - self.add_child(StatSet(app, "stats", system, "general")) + self.add_child(SystemGeneralStatSet(app, "stats", system)) chart = StatFlashChart(app, "freemem", system) chart.stats = ("memFree",) Modified: mgmt/newdata/cumin/python/cumin/messaging/broker.py =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/broker.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/broker.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -157,6 +157,12 @@ def render_title(self, session): return "Access Control" +class AccessControlGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(AccessControlGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("aclDenyCount",) + class BrokerAccessControlView(Widget): def __init__(self, app, name, acl): super(BrokerAccessControlView, self).__init__(app, name) @@ -166,7 +172,7 @@ self.props = CuminProperties(app, "props", self.acl) self.add_child(self.props) - self.stats = StatSet(app, "stats", self.acl, "general") + self.stats = AccessControlGeneralStatSet(app, "stats", self.acl) self.add_child(self.stats) class BrokerClustering(ModeSet): @@ -198,6 +204,12 @@ def render_title(self, session): return "Clustering" +class ClusterGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(ClusterGeneralStatSet, self).__init__(app, name, object) + + self.attrs = () + class BrokerClusteringView(Widget): def __init__(self, app, name, cluster): super(BrokerClusteringView, self).__init__(app, name) @@ -205,7 +217,7 @@ props = CuminProperties(app, "props", cluster) self.add_child(props) - stats = StatSet(app, "stats", cluster, "general") + stats = ClusterGeneralStatSet(app, "stats", cluster) self.add_child(stats) class BrokerBrowser(Widget): Modified: mgmt/newdata/cumin/python/cumin/messaging/brokerlink.py =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/brokerlink.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/brokerlink.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -83,12 +83,25 @@ self.tabs.add_tab(CuminDetails(app, "details", link)) +class LinkGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(LinkGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("closing",) + +class LinkIOStatSet(StatSet): + def __init__(self, app, name, object): + super(LinkIOStatSet, self).__init__(app, name, object) + + self.attrs = ("framesFromPeer", "framesToPeer", + "bytesFromPeer", "bytesToPeer") + class LinkStats(TabbedModeSet): def __init__(self, app, name, link): super(LinkStats, self).__init__(app, name) - self.add_child(StatSet(app, "io", link, "io")) - self.add_child(StatSet(app, "general", link, "general")) + self.add_child(LinkIOStatSet(app, "io", link)) + self.add_child(LinkGeneralStatSet(app, "general", link)) chart = self.ReceiveRouteDropRateChart(app, "recvroutedrop", link) chart.stats = ("msgReceives", "msgRoutes", "msgDrops") Modified: mgmt/newdata/cumin/python/cumin/messaging/connection.py =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/connection.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/connection.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -63,6 +63,11 @@ super(ConnectionFrame, self).__init__(app, name, cls) + self.icon_href = "resource?name=client-36.png" + + self.overview = ConnectionStats(app, "overview", self.object) + self.view.add_tab(self.overview) + class ConnectionView(CuminView): def __init__(self, app, name, conn): super(ConnectionView, self).__init__(app, name, conn) @@ -77,12 +82,25 @@ self.tabs.add_tab(CuminDetails(app, "details", conn)) +class ConnectionGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(ConnectionGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("closing",) + +class ConnectionIOStatSet(StatSet): + def __init__(self, app, name, object): + super(ConnectionIOStatSet, self).__init__(app, name, object) + + self.attrs = ("bytesFromClient", "bytesToClient", + "framesFromClient", "framesToClient") + class ConnectionStats(Widget): def __init__(self, app, name, conn): super(ConnectionStats, self).__init__(app, name) - self.add_child(StatSet(app, "io", conn, "io")) - self.add_child(StatSet(app, "general", conn, "general")) + self.add_child(ConnectionIOStatSet(app, "io", conn)) + self.add_child(ConnectionGeneralStatSet(app, "general", conn)) chart = self.SendReceiveRateChart(app, "sendrecv", conn) chart.stats = ("bytesFromClient", "bytesToClient") Modified: mgmt/newdata/cumin/python/cumin/messaging/exchange.py =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/exchange.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/exchange.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -48,9 +48,8 @@ self.icon_href = "resource?name=exchange-36.png" - # XXX - #self.overview = ExchangeOverview(app, "overview", self.object) - #self.view.add_tab(self.overview) + self.overview = ExchangeOverview(app, "overview", self.object) + self.view.add_tab(self.overview) self.bindings = ExchangeBindingSelector(app, "bindings", self.object) self.view.add_tab(self.bindings) @@ -229,12 +228,26 @@ durable, sequence_numbers, initial_value) self.task.exit_with_redirect(session, vhost) +class ExchangeGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(ExchangeGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("producerCount", "bindingCount") + +class ExchangeIOStatSet(StatSet): + def __init__(self, app, name, object): + super(ExchangeIOStatSet, self).__init__(app, name, object) + + self.attrs = ("msgReceives", "msgRoutes", + "msgDrops", "byteReceives", + "byteRoutes", "byteDrops") + class ExchangeOverview(Widget): def __init__(self, app, name, exchange): super(ExchangeOverview, self).__init__(app, name) - self.add_child(StatSet(app, "io", exchange, "io")) - self.add_child(StatSet(app, "general", exchange, "general")) + self.add_child(ExchangeIOStatSet(app, "io", exchange)) + self.add_child(ExchangeGeneralStatSet(app, "general", exchange)) chart = self.ReceiveRouteDropRateChart(app, "recvroutedrop", exchange) chart.stats = ("msgReceives", "msgRoutes", "msgDrops") Modified: mgmt/newdata/cumin/python/cumin/messaging/exchange.strings =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/exchange.strings 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/exchange.strings 2010-04-06 20:04:09 UTC (rev 3894) @@ -4,7 +4,7 @@ {item_content} </div> -[ExchangeStats.html] +[ExchangeOverview.html] <table class="twocol"> <tbody> <tr> Modified: mgmt/newdata/cumin/python/cumin/messaging/queue.py =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/queue.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/queue.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -53,6 +53,9 @@ self.icon_href = "resource?name=queue-36.png" + self.overview = QueueStats(app, "overview", self.object) + self.view.add_tab(self.overview) + self.bindings = QueueBindingSelector(app, "bindings", self.object) self.view.add_tab(self.bindings) @@ -432,14 +435,29 @@ self.add_tab(QueueStatsTransactions(app, "txn", queue)) def render_title(self, session): - return "Statistics" + return "Overview" +class QueueGeneralStatSet(StatSet): + def __init__(self, app, name, object): + super(QueueGeneralStatSet, self).__init__(app, name, object) + + self.attrs = ("consumerCount", "bindingCount", + "msgDepth", "byteDepth") + +class QueueIOStatSet(StatSet): + def __init__(self, app, name, object): + super(QueueIOStatSet, self).__init__(app, name, object) + + self.attrs = ("msgTotalEnqueues", "msgTotalDequeues", + "byteTotalEnqueues", "byteTotalDequeues", + "unackedMessages", "messageLatency") + class QueueStatsGeneral(Widget): def __init__(self, app, name, queue): super(QueueStatsGeneral, self).__init__(app, name) - self.add_child(StatSet(app, "general", queue, "general")) - self.add_child(StatSet(app, "io", queue, "io")) + self.add_child(QueueGeneralStatSet(app, "general", queue)) + self.add_child(QueueIOStatSet(app, "io", queue)) chart = self.EnqueueDequeueRateChart(app, "enqdeq", queue) chart.stats = ("msgTotalEnqueues", "msgTotalDequeues") @@ -465,12 +483,19 @@ def render_title(self, session): return "Queue Depth" +class QueueIODurableStatSet(StatSet): + def __init__(self, app, name, object): + super(QueueIODurableStatSet, self).__init__(app, name, object) + + self.attrs = ("msgPersistEnqueues", "msgPersistDequeues", + "bytePersistEnqueues", "bytePersistDequeues") + class QueueStatsDurability(Widget): def __init__(self, app, name, queue): super(QueueStatsDurability, self).__init__(app, name) - self.add_child(StatSet(app, "io", queue, "io.durable")) - self.add_child(JournalStats(app, "jrnl", queue, "io.journal")) + self.add_child(QueueIODurableStatSet(app, "io", queue)) + self.add_child(JournalStats(app, "jrnl", queue)) chart = self.EnqueueDequeueRateChart(app, "enqdeq", queue) chart.stats = ("msgPersistEnqueues", "msgPersistDequeues") @@ -485,19 +510,36 @@ return "Durable messages enqueued and eequeued" class JournalStats(StatSet): - def __init__(self, app, name, queue, category): - super(JournalStats, self).__init__(app, name, queue, category) + def __init__(self, app, name, queue): + super(JournalStats, self).__init__(app, name, queue) self.journal = self.JournalAttribute(app, "journal") self.add_attribute(self.journal) + self.attrs = ("initialFileCount", "dataFileSize", + "recordDepth", "recordEnqueues", + "outstandingAIOs", "freeFileCount", + "availableFileCount", "writeWaitFailures", + "writeBusyFailures", "readRecordCount", + "readBusyFailures", "writePageCacheDepth", + "readPageCacheDepth") + class JournalAttribute(Attribute): def get(self, session): queue = self.widget.object.get(session) - for journal in Journal.selectBy(queue=queue): - return journal + cls = self.app.rosemary.com_redhat_rhm_store.Journal + conn = self.app.model.get_sql_connection() + cursor = conn.cursor() + + try: + journals = cls.select_objects(cursor, _queueRef_id=queue._id) + finally: + cursor.close() + + return len(journals) + def render_title(self, session): return "Journal" @@ -509,25 +551,19 @@ else: return "<div class=\"iblock\">%s</div>" % fmt_none() +class QueueTxnStatSet(StatSet): + def __init__(self, app, name, queue): + super(QueueTxnStatSet, self).__init__(app, name, queue) + + self.attrs = ("msgTxnEnqueues", "msgTxnDequeues", + "byteTxnEnqueues", "byteTxnDequeues") + class QueueStatsTransactions(Widget): def __init__(self, app, name, queue): super(QueueStatsTransactions, self).__init__(app, name) - self.add_child(StatSet(app, "transactions", queue, "txn")) - self.add_child(StatSet(app, "io", queue, "io.txn")) + self.add_child(QueueTxnStatSet(app, "io", queue)) - chart = self.EnqueueTransactionRateChart(app, "enqtxn", queue) - chart.stats = ("enqueueTxnStarts", "enqueueTxnCommits", - "enqueueTxnRejects") - chart.mode = "rate" - self.add_child(chart) - - chart = self.DequeueTransactionRateChart(app, "deqtxn", queue) - chart.stats = ("dequeueTxnStarts", "dequeueTxnCommits", - "dequeueTxnRejects") - chart.mode = "rate" - self.add_child(chart) - chart = self.EnqueueDequeueRateChart(app, "enqdeq", queue) chart.stats = ("msgTxnEnqueues", "msgTxnDequeues") chart.mode = "rate" Modified: mgmt/newdata/cumin/python/cumin/messaging/queue.strings =================================================================== --- mgmt/newdata/cumin/python/cumin/messaging/queue.strings 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/messaging/queue.strings 2010-04-06 20:04:09 UTC (rev 3894) @@ -72,15 +72,10 @@ <tbody> <tr> <td> - <h2>Transactions</h2> - {transactions} - <h2>Transactional Input/Output</h2> {io} </td> <td> - {enqtxn} - {deqtxn} {enqdeq} </td> </tr> Modified: mgmt/newdata/cumin/python/cumin/model.py =================================================================== --- mgmt/newdata/cumin/python/cumin/model.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/model.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -1,14 +1,17 @@ import logging from datetime import datetime, timedelta +from decimal import * from mint import Mint, MintConfig from struct import unpack, calcsize +from sqladapter import * from sqlobject import sqlhub from threading import Thread, Lock from types import * from wooly import * from wooly.parameters import * from wooly.widgets import * +from wooly.datatable import * from formats import * from parameters import * @@ -106,6 +109,11 @@ def get_ad_groups(self): return AdProperty.get_ad_groups() + def get_class_by_rosemary_object(self, rosemary_object): + for cls in self.classes: + if cls.cumin_name.lower() == rosemary_object._class._name.lower(): + return cls + def get_class_by_object(self, mint_object): for cls in self.classes: if cls.mint_class is mint_object.__class__: @@ -588,13 +596,6 @@ def init(self): pass - def recent(self, object): - stats = object.stats - - stats = stats.orderBy("-qmfUpdateTime") - if stats.count(): - return stats[0].qmfUpdateTime - def get_connection(self): return self.model.get_sql_connection() @@ -609,79 +610,6 @@ nname.append(c) return "".join(nname) - def avg_samples(self, object, secs, interval, secs2): - conn = self.get_connection() - - when = "(qmf_update_time >= now() - interval '%i seconds')" % int(secs + secs2) - when2 = "" - if secs2: - when2 = " and (qmf_update_time <= now() - interval '%i seconds')" % int(secs2) - - where = "%s and %s%s" % (object.stats.clause, when, when2) - - table_name = self.cumin_class.mint_stats_class.q.tableName - field_name = self.get_db_name() - - if conn: - cursor = conn.cursor() - - sql = """select max(qmf_update_time) as interval_end, - cast(avg(%s) as integer) as value, - stddev(%s) as dev - from %s - where %s - group by floor(extract(epoch from qmf_update_time) / %i) - order by interval_end desc;""" % (field_name, field_name, - table_name, - where, - interval) - - cursor.execute(sql) - return cursor.fetchall() - - def samples(self, object, secs, interval, method, secs2=0): - if method == "avg": - return self.avg_samples(object, secs, interval, secs2) - - stats = object.stats - - now = datetime.now() - upper_limit = now - timedelta(seconds=int(secs2)) - lower_limit = now - timedelta(seconds=int(secs + secs2)) - col = self.cumin_class.mint_stats_class.q.qmfUpdateTime - stats = stats.filter(col <= upper_limit) - - def ensure_at_least(min_count, more_time): - dt = now - timedelta(seconds=int(secs + secs2 + more_time)) - stats2 = stats.filter(col >= dt) - if stats2.count() < min_count: - more_time += 60 - if more_time <= 60 * 60: - return ensure_at_least(min_count, more_time) - return stats2 - - stats = ensure_at_least(2, 0) - stats = stats.orderBy("-qmfUpdateTime") - - samples = list() - - next_is_last = False - - for stat in stats: - if (stat.qmfUpdateTime > upper_limit): - continue - - value = getattr(stat, self.name, 0) - samples.append((stat.qmfUpdateTime, value)) - - if next_is_last: - break - - if stat.qmfUpdateTime < lower_limit: - next_is_last = True - - return samples - def value_html(self, object): text = self.value_text(object) @@ -699,7 +627,7 @@ text = "" elif self.format: text = self.format % value - elif type(value) in (int, long): + elif type(value) in (int, long, Decimal): if value == 0: text = "0" else: @@ -711,7 +639,7 @@ def value(self, object): try: - return getattr(object.statsCurr, self.name) + return getattr(object, self.name) except AttributeError: pass @@ -763,10 +691,109 @@ writer.write("<stat name=\"%s\" value=\"%s\" rate=\"%s\"/>" \ % (self.name, value, rate)) +class SamplesSqlAdapter(SqlAdapter): + qmf_update_col = '_qmf_update_time' + _parent_id_col = "_parent_id" + + def __init__(self, app, table, parent_id): + super(SamplesSqlAdapter, self).__init__(app, table) + + self.parent_id = parent_id + + def get_sql_options(self, options): + return options + + def avg_samples(self, stat, secs, interval, secs2): + table = self.table + stat_col = table._columns_by_name[stat.name] + updated_col = table._columns_by_name[self.qmf_update_col] + parent_col = table._columns_by_name[self._parent_id_col] + + max_col = "max(%s) as interval_end" % updated_col.identifier + value_col = "cast(avg(%s) as integer) as value" % stat_col.identifier + dev_col = "stddev(%s) as dev" % stat_col.identifier + + columns = list() + columns.append(max_col) + columns.append(value_col) + columns.append(dev_col) + self.columns = columns + + self.query.filters = list() + SqlComparisonFilter(self.query, parent_col, self.parent_id, "=") + + when = "now() - interval '%i seconds'" % int(secs + secs2) + SqlComparisonFilter(self.query, updated_col, when, ">=") + when2 = "" + if secs2: + when2 = "now() - interval '%i seconds'" % int(secs2) + SqlComparisonFilter(self.query, updated_col, when2, "<=") + + options = SqlQueryOptions() + options.sort_column = "interval_end" + options.sort_ascending = False + options.group_column = "floor(extract(epoch from %s) / %i)" % (updated_col.identifier, interval) + + samples = self.get_data({}, options) + return samples + + + def samples(self, stat, secs, interval, method, secs2=0, delta=False): + if method == "avg": + return self.avg_samples(stat, secs, interval, secs2) + + table = self.table + stat_col = table._columns_by_name[stat.name] + updated_col = table._columns_by_name[self.qmf_update_col] + parent_col = table._columns_by_name[self._parent_id_col] + + columns = list() + columns.append(updated_col.identifier) + columns.append(stat_col.identifier) + self.columns = columns + + self.query.filters = list() + SqlComparisonFilter(self.query, parent_col, self.parent_id, "=") + + if delta: + secs = 60*60 + when = "now() - interval '%i seconds'" % int(secs + secs2) + SqlComparisonFilter(self.query, updated_col, when, ">=") + when2 = "" + if secs2: + when2 = "now() - interval '%i seconds'" % int(secs2) + SqlComparisonFilter(self.query, updated_col, when2, "<=") + + + options = SqlQueryOptions() + options.sort_column = updated_col + options.sort_ascending = False + if delta: + options.limit = 2 + + samples = self.get_data({}, options) + return samples + + def recent(self): + updated_col = self.table._columns_by_name[self.qmf_update_col] + + columns = list() + columns.append(updated_col.identifier) + self.columns = columns + + options = SqlQueryOptions() + options.sort_column = updated_col + options.sort_ascending = False + + samples = self.get_data({}, options) + return len(samples) and samples[0][0] or None + class CuminStatistic(object): formatters = { - None: lambda x: fmt_none(), - float: lambda x: "%0.2d" % x + NoneType: lambda x: fmt_none(), + float: lambda x: "%0.2d" % x, + Decimal: lambda x: str(x), + int: lambda x: "%i" % x } def __init__(self, name): @@ -776,8 +803,12 @@ return self.name def format_value(self, session, value): + return CuminStatistic.fmt_value(value) + + @classmethod + def fmt_value(cls, value): try: - return self.formatters[type(value)](value) + return cls.formatters[type(value)](value) except KeyError: return value Modified: mgmt/newdata/cumin/python/cumin/stat.py =================================================================== --- mgmt/newdata/cumin/python/cumin/stat.py 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/stat.py 2010-04-06 20:04:09 UTC (rev 3894) @@ -10,62 +10,40 @@ from parameters import * from util import * from formats import * +from model import * from OpenFlashChart import Chart, Element strings = StringCatalog(__file__) class StatSet(ItemSet): - unit_abbrevs = dict() - unit_abbrevs["message"] = "msg" - unit_abbrevs["transaction"] = "trans" - - def __init__(self, app, name, object, category): + def __init__(self, app, name, object): super(StatSet, self).__init__(app, name) self.object = object - self.category = category - + self.attrs = list() self.update_enabled = True - def render_rate_text(self, session): - return "Per Second" + def get_fields(self, session): + return self.attrs def do_get_items(self, session): object = self.object.get(session) - cls = self.app.model.get_class_by_object(object) + stats = list() + for field in self.get_fields(session): + cls = object._class._statistics_by_name[field] + stats.append((cls, getattr(object, cls.name, None))) + return stats - return [x for x in cls.stats if x.category == self.category] - - def render_item_title(self, session, stat): + def render_item_title(self, session, item): + stat, value = item return stat.title - def render_item_name(self, session, stat): - return stat.name + def render_item_value(self, session, item): + stat, value = item + return CuminStatistic.fmt_value(value) - def render_item_value(self, session, stat): - object = self.object.get(session) - - #if stat.link_cb: - # branch = session.branch() - # stat.link_cb(self.page, branch, object) - # return fmt_link(branch.marshal(), stat.value(object)) - #else: - - return stat.value_html(object) - - def render_item_rate(self, session, stat): - object = self.object.get(session) - - #if False and stat.highlow: #XXX - # return "<small>high</small>&nbsp;<span>%i</span> " + \ - # "<small>low</small>&nbsp;<span>%i</span>" \ - # % (stat.high(object) or 0, stat.low(object) or 0) - #else: - - return stat.rate_html(object) - class NewStatSet(ItemSet): def __init__(self, app, name): super(NewStatSet, self).__init__(app, name) @@ -126,16 +104,26 @@ self.fullpageable = True - def get_href_params(self, session): + def get_object_id(self, session): object = self.object.get(session) - params = list() + return object._id - cls = self.app.model.get_class_by_object(object).cumin_name + def get_package_name(self, session): + object = self.object.get(session) + return object._class._package._name - params.append("class=%s" % cls) - params.append("id=%i" % object.id) + def get_object_class(self, session): + object = self.object.get(session) + return object._class._name + + def get_href_params(self, session): + params = list() + + params.append("id=%i" % self.get_object_id(session)) params.append("chart_id=%s" % self.render_id(session, None)) params.append("duration=%s" % self.duration.get(session)) + params.append("rpkg=%s" % self.get_package_name(session)) + params.append("rcls=%s" % self.get_object_class(session)) for stat in self.stats: params.append("stat=%s" % stat) @@ -160,12 +148,12 @@ def render_title(self, session): object = self.object.get(session) - cls = self.app.model.get_class_by_object(object) + cls = object._class return getattr(cls, self.stats[0]).title def render_stats(self, session): object = self.object.get(session) - cls = self.app.model.get_class_by_object(object) + cls = object._class writer = Writer() @@ -277,37 +265,16 @@ self.__files[name] = {"time": datetime.now(), "file": file, "cookie": args} return file -class PieChartPage(Page): - def __init__(self, app, name): - super(PieChartPage, self).__init__(app, name) - - self.cls = CuminClassParameter(app, "class") - self.add_parameter(self.cls) - - self.id = IntegerParameter(app, "id") - self.add_parameter(self.id) - - param = Parameter(app, "param") - self.add_parameter(param) - - self.stats = ListParameter(app, "stat", param) - self.add_parameter(self.stats) - - def get_cache_control(self, session): - return "no-cache" - - def get_object(self, session): - cls = self.cls.get(session).mint_class - id = self.id.get(session) - return (cls.get(id),) - class StatChartPage(Page): def __init__(self, app, name): super(StatChartPage, self).__init__(app, name) - self.cls = CuminClassParameter(app, "class") - self.add_parameter(self.cls) + self.rosemary_class = Parameter(app, "rcls") + self.add_parameter(self.rosemary_class) + self.rosemary_package = Parameter(app, "rpkg") + self.add_parameter(self.rosemary_package) + self.id = IntegerParameter(app, "id") self.add_parameter(self.id) @@ -349,14 +316,21 @@ def get_content_type(self, session): return self.samples.get(session) and "text/plain" or "image/png" + def get_adapter_stats(self, session): + rpackage = self.rosemary_package.get(session) + rclass = self.rosemary_class.get(session) + rosemary_package = self.app.rosemary._packages_by_name[rpackage] + rosemary_class = rosemary_package._classes_by_name[rclass] + id = str(self.id.get(session)) + + adapter = SamplesSqlAdapter(self.app, rosemary_class.sql_samples_table, id) + stats = [getattr(rosemary_class, x) for x in self.stats.get(session)] + + return (adapter, stats) + def get_cache_control(self, session): return "no-cache" - def get_object(self, session): - cls = self.cls.get(session).mint_class - id = self.id.get(session) - return (cls.get(id),) - def gen_filename(self, session): return session.marshal() @@ -407,19 +381,16 @@ return str(rets) def do_render(self, session): - object = self.get_object(session)[0] + adapter, stats = self.get_adapter_stats(session) colors = ((1,0,0), (0,0,1), (0,1,0)) - cls = self.cls.get(session) - stats = [getattr(cls, x) for x in self.stats.get(session)] - recent = None if len(stats): - stat = stats[0] - recent = stat.recent(object) + recent = adapter.recent() else: return + # mouseover event wants the points returned in a array if self.samples.get(session): return self.render_samples(session, recent) @@ -442,13 +413,13 @@ if mode == "rate": method = None # don't do averaging for stat in stats: - os = stat.samples(object, duration, interval, method) + os = adapter.samples(stat, duration, interval, method) ns = list() prev = None for sample in reversed(os): if prev is not None: - rate = calc_rate(sample[1], prev[1], + rate = calc_rate(float(sample[1]), float(prev[1]), secs(sample[0]), secs(prev[0])) ns.insert(0, (sample[0], rate, None)) @@ -459,7 +430,7 @@ else: method = self.method.get(session) for stat in stats: - samples[stat] = stat.samples(object, duration, interval, method) + samples[stat] = adapter.samples(stat, duration, interval, method) # take stddev into account for max and min y values deviated_values = [(nvl(x[1],0) + float(nvl(x[2],0))/2, @@ -524,12 +495,15 @@ self.updater.update_enabled = True self.add_child(self.updater) - self.cls = CuminClassParameter(app, "class") - self.add_parameter(self.cls) - self.id = IntegerParameter(app, "id") self.add_parameter(self.id) + self.rosemary_class = Parameter(app, "rcls") + self.add_parameter(self.rosemary_class) + + self.rosemary_package = Parameter(app, "rpkg") + self.add_parameter(self.rosemary_package) + self.object = self.ObjectAttribute(app, "object") self.add_attribute(self.object) @@ -564,11 +538,27 @@ class ObjectAttribute(Attribute): def get(self, session): - cls = self.widget.cls.get(session).mint_class - id = self.widget.id.get(session) - return cls.get(id) + rpackage = self.widget.rosemary_package.get(session) + rclass = self.widget.rosemary_class.get(session) + rosemary_package = self.app.rosemary._packages_by_name[rpackage] + rosemary_class = rosemary_package._classes_by_name[rclass] + return rosemary_class + class GenericChart(StatFlashChart): + def render_title(self, session): + cls = self.object.get(session) + return getattr(cls, self.stats[0]).title + + def get_object_id(self, session): + return self.parent.id.get(session) + + def get_package_name(self, session): + return self.parent.rosemary_package.get(session) + + def get_object_class(self, session): + return self.parent.rosemary_class.get(session) + def render_width(self, session): return self.parent.container_width.get(session) @@ -576,14 +566,12 @@ return self.parent.container_height.get(session) def render_img_href(self, session): - object = self.object params = self.get_href_params(session) params.append("width=%i" % self.render_width(session)) params.append("height=%i" % self.render_height(session)) return "%s?" % self.get_chart_name(session) + ";".join(params) def render_href(self, session): - object = self.object params = self.get_href_params(session) params.append("width=%i" % self.render_width(session)) params.append("height=%i" % self.render_height(session)) @@ -637,15 +625,10 @@ super(FlashPieChart, self).__init__(app, name) self.page = page - def create(self, session, object): + def create(self, session, adapter, stats): names = self.page.names.get(session) values = self.page.values.get(session) - #cls = self.page.cls.get(session) - #action = self.page.action.get(session) - #method = getattr(cls, action) - #values1 = method.do_invoke(object) - chart = Chart() chart.title.text = "" chart.bg_colour = "#FFFFFF" @@ -766,13 +749,12 @@ return None return method - def create(self, session, object): + def create(self, session, adapter, stats): # get the page parameters width = self.page.container_width.get(session) height = self.page.container_height.get(session) mode = self.page.mode.get(session) method = self.page.method.get(session) - stats = self.page.get_stats(session) time_span = self.get_time_span(session) duration = self.get_duration(session) @@ -782,7 +764,7 @@ interval = self.page.get_interval(session, duration, width) # get the most recent samples - samples = self.fetch_samples(object, duration, interval, + samples = self.fetch_samples(adapter, duration, interval, self.fix_method(method, mode), mode, delta, stats, end_seconds_ago=end_seconds_ago) @@ -797,7 +779,7 @@ max_of_axiis = max(max_value, axis_for_vals) # the most recent value(s) changed the y-axis range if axis_max != max_of_axiis: - samples = self.fetch_samples(object, time_span, interval, + samples = self.fetch_samples(adapter, time_span, interval, self.fix_method(method, mode), mode, False, stats, end_seconds_ago=end_seconds_ago) @@ -806,7 +788,7 @@ append = True # create the chart dict - chart = self.get_chart(session, object, stats, samples, time_span, max_value, min_value, append, end_seconds_ago) + chart = self.get_chart(session, adapter, stats, samples, time_span, max_value, min_value, append, end_seconds_ago) #print "**********\n"+chart.create() return chart.create() @@ -862,13 +844,13 @@ return y_axis, y_axis_right - def fetch_samples(self, object, dur, interval, method, mode, delta, stats, end_seconds_ago=0): + def fetch_samples(self, adapter, dur, interval, method, mode, delta, stats, end_seconds_ago=0): return dict() def get_max_min(self, session, stats, samples): return 0, 0 - def get_chart(self, session, object, stats, samples, duration, max_value, min_value, append, end_secs): + def get_chart(self, session, adapter, stats, samples, duration, max_value, min_value, append, end_secs): return Chart() class AreaChart(FlashLineChart): @@ -877,17 +859,17 @@ self.alpha = 0.3 - def fetch_samples(self, object, dur, interval, method, mode, delta, stats, end_seconds_ago=0): + def fetch_samples(self, adapter, dur, interval, method, mode, delta, stats, end_seconds_ago=0): samples = dict() if mode == "rate": for stat in stats: - os = stat.samples(object, dur, interval, method, secs2=end_seconds_ago) + os = adapter.samples(stat, dur, interval, method, secs2=end_seconds_ago, delta=delta) ns = list() prev = None for sample in reversed(os): if prev is not None: - rate = calc_rate(sample[1], prev[1], + rate = calc_rate(float(sample[1]), float(prev[1]), secs(sample[0]), secs(prev[0])) ns.insert(0, (sample[0], rate, None)) @@ -897,7 +879,7 @@ samples[stat] = ns else: for stat in stats: - samples[stat] = stat.samples(object, dur, interval, method, secs2=end_seconds_ago) + samples[stat] = adapter.samples(stat, dur, interval, method, secs2=end_seconds_ago, delta=delta) return samples @@ -949,7 +931,7 @@ line.values = vals chart.elements.append(line) - def get_chart(self, session, object, stats, samples, duration, max_value, min_value, append, end_secs): + def get_chart(self, session, adapter, stats, samples, duration, max_value, min_value, append, end_secs): mode = self.page.mode.get(session) width = self.page.container_width.get(session) @@ -970,7 +952,6 @@ if append: chart.append = self.get_elapsed(session)['value'] - #print "append: duration=%i elapsed=%f" % (duration, float(self.get_elapsed(session)['value'])) return chart chart.title.text = "" @@ -993,7 +974,7 @@ chart.control.x_axis = XAxis().get_x_axis(self.one_day, 0, tick_height=10) chart.control.x_axis.labels.colour = "#333333" chart.control.x_axis.grid_colour = "#FFFFFF" - samples = self.fetch_samples(object, self.one_day, 60, "avg", mode, False, stats, end_seconds_ago=0) + samples = self.fetch_samples(adapter, self.one_day, 60, "avg", mode, False, stats, end_seconds_ago=0) max_value, min_value = self.get_max_min(session, stats, samples) self.make_chart_lines(session, chart.control, "line", stats, 1, 0, 1, samples, self.one_day, 0, mode) chart.control.y_min = max(min_value, 0) @@ -1122,15 +1103,11 @@ def get_content_type(self, session): return "text/plain" - def get_stats(self, session): - cls = self.cls.get(session) - return [getattr(cls, x) for x in self.stats.get(session)] - def do_render(self, session): - object = self.get_object(session)[0] + adapter, stats = self.get_adapter_stats(session) chart = self.chart_factory(self.chart_type.get(session)) - return chart.create(session, object) + return chart.create(session, adapter, stats) def chart_factory(self, chart_type): if chart_type == "area": @@ -1145,13 +1122,10 @@ class StatStackedPage(StatChartPage): def do_render(self, session): - object = self.get_object(session)[0] - cls = self.cls.get(session) - stats = [getattr(cls, x) for x in self.stats.get(session)] + adapter, stats = self.get_adapter_stats(session) if len(stats): - stat = stats[0] - recent = stat.recent(object) + recent = adapter.recent() if recent: cached_png = self.get_cached(session, recent) if cached_png: @@ -1175,7 +1149,7 @@ collapsed = dict() values = dict() for stat in stats: - samples[stat] = stat.samples(object, duration, interval, method) + samples[stat] = adapter.samples(stat, duration, interval, method) # we sometimes get multiple samples for the same qmf_update_time for sample in samples[stat]: collapsed[sample[0]] = sample[1] Modified: mgmt/newdata/cumin/python/cumin/stat.strings =================================================================== --- mgmt/newdata/cumin/python/cumin/stat.strings 2010-04-06 16:39:42 UTC (rev 3893) +++ mgmt/newdata/cumin/python/cumin/stat.strings 2010-04-06 20:04:09 UTC (rev 3894) @@ -37,8 +37,7 @@ <thead> <tr> <th style="width: 50%; text-align: left;">Statistic</th> - <th style="width: 25%;">Value</th> - <th style="width: 25%;">{rate_text}</th> + <th style="width: 50%;">Value</th> </tr> </thead> <tbody> @@ -50,7 +49,6 @@ <tr class="item"> <th>{item_title}</th> <td class="ralign"> {item_value}</td> - <td class="ralign"> {item_rate}</td> </tr> [NewStatSet.css]