Author: justi9 Date: 2008-08-26 13:51:43 -0400 (Tue, 26 Aug 2008) New Revision: 2361 Added: mgmt/trunk/cumin/python/cumin/job.py mgmt/trunk/cumin/python/cumin/job.strings Modified: mgmt/trunk/cumin/python/cumin/pool.py mgmt/trunk/mint/python/mint/schema.py mgmt/trunk/mint/sql/schema.sql Log: Update to the latest schema files, and add rudimentary job listing Added: mgmt/trunk/cumin/python/cumin/job.py =================================================================== --- mgmt/trunk/cumin/python/cumin/job.py (rev 0) +++ mgmt/trunk/cumin/python/cumin/job.py 2008-08-26 17:51:43 UTC (rev 2361) @@ -0,0 +1,38 @@ +import logging + +from wooly import * +from wooly.widgets import * +from wooly.forms import * +from wooly.resources import * +from wooly.tables import * + +from stat import * +from widgets import * +from parameters import * +from formats import * +from util import * + +strings = StringCatalog(__file__) +log = logging.getLogger("cumin.job") + +class JobSet(CuminTable): + def __init__(self, app, name): + super(JobSet, self).__init__(app, name) + + col = self.CommandColumn(app, "name") + self.add_column(col) + + self.set_default_column(col) + + def render_title(self, session): + return "Jobs %s" % fmt_count(Job.select().count()) + + class CommandColumn(SqlTableColumn): + def render_title(self, session, data): + return "Command" + + def render_content(self, session, data): + job = Identifiable(data["id"]) + branch = session.branch() + self.frame.show_job(branch, job).show_view(branch) + return fmt_olink(branch, job, name=data["cmd"]) Added: mgmt/trunk/cumin/python/cumin/job.strings =================================================================== --- mgmt/trunk/cumin/python/cumin/job.strings (rev 0) +++ mgmt/trunk/cumin/python/cumin/job.strings 2008-08-26 17:51:43 UTC (rev 2361) @@ -0,0 +1,8 @@ +[JobSet.sql] +select + j.id, + j.cmd +from job as j + +[JobSet.count_sql] +select count(*) from pool Modified: mgmt/trunk/cumin/python/cumin/pool.py =================================================================== --- mgmt/trunk/cumin/python/cumin/pool.py 2008-08-26 15:06:01 UTC (rev 2360) +++ mgmt/trunk/cumin/python/cumin/pool.py 2008-08-26 17:51:43 UTC (rev 2361) @@ -11,6 +11,7 @@ from parameters import * from formats import * from util import * +from job import JobSet strings = StringCatalog(__file__) log = logging.getLogger("cumin.pool") @@ -48,6 +49,9 @@ self.add_mode(view) self.set_view_mode(view) + def show_job(self, session, job): + return self + class PoolView(CuminView): def __init__(self, app, name): super(PoolView, self).__init__(app, name) @@ -58,6 +62,12 @@ self.__tabs = TabbedModeSet(app, "tabs") self.add_child(self.__tabs) + jobs = self.JobsTab(app, "jobs") + self.__tabs.add_tab(jobs) + + class JobsTab(JobSet): + pass + class PoolStatus(CuminStatus): pass Modified: mgmt/trunk/mint/python/mint/schema.py =================================================================== --- mgmt/trunk/mint/python/mint/schema.py 2008-08-26 15:06:01 UTC (rev 2360) +++ mgmt/trunk/mint/python/mint/schema.py 2008-08-26 17:51:43 UTC (rev 2361) @@ -54,7 +54,6 @@ mgmtPubInterval = SmallIntCol(default=None) clusterName = StringCol(length=1000, default=None) version = StringCol(length=1000, default=None) - dataDirEnabled = BoolCol(default=None) dataDir = StringCol(length=1000, default=None) classInfos = dict() # brokerId => classInfo @@ -535,9 +534,15 @@ statsPrev = ForeignKey('StoreStats', cascade='null', default=None) broker = ForeignKey('Broker', cascade='null', default=None) location = StringCol(length=1000, default=None) - async = BoolCol(default=None) defaultInitialFileCount = SmallIntCol(default=None) defaultDataFileSize = IntCol(default=None) + tplIsInitialized = BoolCol(default=None) + tplDirectory = StringCol(length=1000, default=None) + tplWritePageSize = IntCol(default=None) + tplWritePages = IntCol(default=None) + tplInitialFileCount = SmallIntCol(default=None) + tplDataFileSize = IntCol(default=None) + tplCurrentFileCount = IntCol(default=None) classInfos = dict() # brokerId => classInfo @@ -548,6 +553,15 @@ idOriginal = BigIntCol(default=None) recTime = TimestampCol(default=None) store = ForeignKey('Store', cascade='null', default=None) + tplTransactionDepth = IntCol(default=None) + tplTransactionDepthLow = IntCol(default=None) + tplTransactionDepthHigh = IntCol(default=None) + tplTxnPrepares = BigIntCol(default=None) + tplTxnCommits = BigIntCol(default=None) + tplTxnAborts = BigIntCol(default=None) + tplOutstandingAIOs = IntCol(default=None) + tplOutstandingAIOsLow = IntCol(default=None) + tplOutstandingAIOsHigh = IntCol(default=None) classInfos = dict() # brokerId => classInfo @@ -564,14 +578,17 @@ managedBroker = StringCol(length=1000, default=None) statsCurr = ForeignKey('JournalStats', cascade='null', default=None) statsPrev = ForeignKey('JournalStats', cascade='null', default=None) + queue = ForeignKey('Queue', cascade='null', default=None) name = StringCol(length=1000, default=None) - queue = ForeignKey('Queue', cascade='null', default=None) directory = StringCol(length=1000, default=None) baseFileName = StringCol(length=1000, default=None) writePageSize = IntCol(default=None) writePages = IntCol(default=None) readPageSize = IntCol(default=None) readPages = IntCol(default=None) + initialFileCount = SmallIntCol(default=None) + dataFileSize = IntCol(default=None) + currentFileCount = IntCol(default=None) classInfos = dict() # brokerId => classInfo @@ -591,14 +608,15 @@ idOriginal = BigIntCol(default=None) recTime = TimestampCol(default=None) journal = ForeignKey('Journal', cascade='null', default=None) - initialFileCount = SmallIntCol(default=None) - dataFileSize = IntCol(default=None) - currentFileCount = IntCol(default=None) recordDepth = IntCol(default=None) recordDepthLow = IntCol(default=None) recordDepthHigh = IntCol(default=None) - recordEnqueues = BigIntCol(default=None) - recordDequeues = BigIntCol(default=None) + enqueues = BigIntCol(default=None) + dequeues = BigIntCol(default=None) + txnEnqueues = BigIntCol(default=None) + txnDequeues = BigIntCol(default=None) + txnCommits = BigIntCol(default=None) + txnAborts = BigIntCol(default=None) outstandingAIOs = IntCol(default=None) outstandingAIOsLow = IntCol(default=None) outstandingAIOsHigh = IntCol(default=None) @@ -623,6 +641,158 @@ +class Slot(SQLObject): + class sqlmeta: + lazyUpdate = True + + idOriginal = BigIntCol(default=None) + recTime = TimestampCol(default=None) + creationTime = TimestampCol(default=None) + deletionTime = TimestampCol(default=None) + managedBroker = StringCol(length=1000, default=None) + statsCurr = ForeignKey('SlotStats', cascade='null', default=None) + statsPrev = ForeignKey('SlotStats', cascade='null', default=None) + Arch = StringCol(length=1000, default=None) + CheckpointPlatform = StringCol(length=1000, default=None) + Cpus = IntCol(default=None) + Disk = IntCol(default=None) + FileSystemDomain = StringCol(length=1000, default=None) + IsValidCheckpointPlatform = StringCol(length=4000, default=None) + KFlops = IntCol(default=None) + Machine = StringCol(length=1000, default=None) + MaxJobRetirementTime = StringCol(length=4000, default=None) + Memory = IntCol(default=None) + Mips = IntCol(default=None) + MyAddress = StringCol(length=1000, default=None) + Name = StringCol(length=1000, default=None) + OpSys = StringCol(length=1000, default=None) + Requirements = StringCol(length=4000, default=None) + PublicNetworkIpAddr = StringCol(length=1000, default=None) + Rank = StringCol(length=4000, default=None) + SlotID = IntCol(default=None) + Start = StringCol(length=4000, default=None) + StarterAbilityList = StringCol(length=4000, default=None) + TotalCpus = IntCol(default=None) + TotalDisk = IntCol(default=None) + TotalMemory = IntCol(default=None) + TotalSlots = IntCol(default=None) + TotalVirtualMemory = IntCol(default=None) + UidDomain = StringCol(length=1000, default=None) + VirtualMemory = IntCol(default=None) + WindowsBuildNumber = IntCol(default=None) + WindowsMajorVersion = IntCol(default=None) + WindowsMinorVersion = IntCol(default=None) + + classInfos = dict() # brokerId => classInfo + +class SlotStats(SQLObject): + class sqlmeta: + lazyUpdate = True + + idOriginal = BigIntCol(default=None) + recTime = TimestampCol(default=None) + slot = ForeignKey('Slot', cascade='null', default=None) + AccountingGroup = StringCol(length=1000, default=None) + Activity = StringCol(length=1000, default=None) + ClientMachine = StringCol(length=1000, default=None) + ClockDay = IntCol(default=None) + ClockMin = IntCol(default=None) + CondorLoadAvg = FloatCol(default=None) + ConsoleIdle = IntCol(default=None) + CurrentRank = FloatCol(default=None) + EnteredCurrentActivity = IntCol(default=None) + EnteredCurrentState = IntCol(default=None) + GlobalJobId = StringCol(length=1000, default=None) + ImageSize = IntCol(default=None) + JobId = StringCol(length=1000, default=None) + JobStart = IntCol(default=None) + KeyboardIdle = IntCol(default=None) + LastBenchmark = IntCol(default=None) + LastFetchWorkCompleted = IntCol(default=None) + LastFetchWorkSpawned = IntCol(default=None) + LastPeriodicCheckpoint = IntCol(default=None) + LoadAvg = FloatCol(default=None) + MyCurrentTime = IntCol(default=None) + NextFetchWorkDelay = IntCol(default=None) + PreemptingOwner = StringCol(length=1000, default=None) + PreemptingUser = StringCol(length=1000, default=None) + PreemptingRank = FloatCol(default=None) + RemoteOwner = StringCol(length=1000, default=None) + RemoteUser = StringCol(length=1000, default=None) + State = StringCol(length=1000, default=None) + TimeToLive = IntCol(default=None) + TotalClaimRunTime = IntCol(default=None) + TotalClaimSuspendTime = IntCol(default=None) + TotalCondorLoadAvg = FloatCol(default=None) + TotalJobRunTime = IntCol(default=None) + TotalJobSuspendTime = IntCol(default=None) + TotalLoadAvg = FloatCol(default=None) + TotalTimeBackfillBusy = IntCol(default=None) + TotalTimeBackfillIdle = IntCol(default=None) + TotalTimeBackfillKilling = IntCol(default=None) + TotalTimeClaimedBusy = IntCol(default=None) + TotalTimeClaimedIdle = IntCol(default=None) + TotalTimeClaimedRetiring = IntCol(default=None) + TotalTimeClaimedSuspended = IntCol(default=None) + TotalTimeMatchedIdle = IntCol(default=None) + TotalTimeOwnerIdle = IntCol(default=None) + TotalTimePreemptingKilling = IntCol(default=None) + TotalTimePreemptingVacating = IntCol(default=None) + TotalTimeUnclaimedBenchmarking = IntCol(default=None) + TotalTimeUnclaimedIdle = IntCol(default=None) + + classInfos = dict() # brokerId => classInfo + + + +class Job(SQLObject): + class sqlmeta: + lazyUpdate = True + + idOriginal = BigIntCol(default=None) + recTime = TimestampCol(default=None) + creationTime = TimestampCol(default=None) + deletionTime = TimestampCol(default=None) + managedBroker = StringCol(length=1000, default=None) + statsCurr = ForeignKey('JobStats', cascade='null', default=None) + statsPrev = ForeignKey('JobStats', cascade='null', default=None) + AccountingGroup = StringCol(length=1000, default=None) + Args = StringCol(length=4000, default=None) + ClusterId = IntCol(default=None) + Cmd = StringCol(length=4000, default=None) + ConcurrencyLimits = StringCol(length=4000, default=None) + CustomGroup = StringCol(length=1000, default=None) + CustomId = StringCol(length=1000, default=None) + CustomPriority = IntCol(default=None) + GlobalJobId = StringCol(length=1000, default=None) + InRsv = StringCol(length=4000, default=None) + Iwd = StringCol(length=4000, default=None) + JobStatus = IntCol(default=None) + Note = StringCol(length=4000, default=None) + Out = StringCol(length=4000, default=None) + Owner = StringCol(length=1000, default=None) + ProcId = IntCol(default=None) + QDate = IntCol(default=None) + Requirements = StringCol(length=4000, default=None) + Scheduler = StringCol(length=1000, default=None) + Title = StringCol(length=1000, default=None) + UserLog = StringCol(length=4000, default=None) + HoldReason = StringCol(length=4000, default=None) + + classInfos = dict() # brokerId => classInfo + +class JobStats(SQLObject): + class sqlmeta: + lazyUpdate = True + + idOriginal = BigIntCol(default=None) + recTime = TimestampCol(default=None) + job = ForeignKey('Job', cascade='null', default=None) + + classInfos = dict() # brokerId => classInfo + + + classToSchemaNameMap = dict() schemaNameToClassMap = dict() classToSchemaNameMap['System'] = 'System' @@ -730,3 +900,13 @@ Journal.sqlmeta.addJoin(SQLMultipleJoin('JournalStats', joinMethodName='stats')) +classToSchemaNameMap['Slot'] = 'slot' +schemaNameToClassMap['slot'] = Slot + +Slot.sqlmeta.addJoin(SQLMultipleJoin('SlotStats', joinMethodName='stats')) + +classToSchemaNameMap['Job'] = 'Job' +schemaNameToClassMap['Job'] = Job + +Job.sqlmeta.addJoin(SQLMultipleJoin('JobStats', joinMethodName='stats')) + Modified: mgmt/trunk/mint/sql/schema.sql =================================================================== --- mgmt/trunk/mint/sql/schema.sql 2008-08-26 15:06:01 UTC (rev 2360) +++ mgmt/trunk/mint/sql/schema.sql 2008-08-26 17:51:43 UTC (rev 2361) @@ -148,7 +148,6 @@ mgmt_pub_interval SMALLINT, cluster_name VARCHAR(1000), version VARCHAR(1000), - data_dir_enabled BOOL, data_dir VARCHAR(1000), registration_id INT ); @@ -221,6 +220,46 @@ byte_routes BIGINT ); +CREATE TABLE job ( + id SERIAL PRIMARY KEY, + id_original BIGINT, + rec_time TIMESTAMP, + creation_time TIMESTAMP, + deletion_time TIMESTAMP, + managed_broker VARCHAR(1000), + stats_curr_id INT, + stats_prev_id INT, + accounting_group VARCHAR(1000), + args VARCHAR(4000), + cluster_id INT, + cmd VARCHAR(4000), + concurrency_limits VARCHAR(4000), + custom_group VARCHAR(1000), + custom_id VARCHAR(1000), + custom_priority INT, + global_job_id VARCHAR(1000), + in_rsv VARCHAR(4000), + iwd VARCHAR(4000), + job_status INT, + note VARCHAR(4000), + out VARCHAR(4000), + owner VARCHAR(1000), + proc_id INT, + q_date INT, + requirements VARCHAR(4000), + scheduler VARCHAR(1000), + title VARCHAR(1000), + user_log VARCHAR(4000), + hold_reason VARCHAR(4000) +); + +CREATE TABLE job_stats ( + id SERIAL PRIMARY KEY, + id_original BIGINT, + rec_time TIMESTAMP, + job_id INT +); + CREATE TABLE journal ( id SERIAL PRIMARY KEY, id_original BIGINT, @@ -230,14 +269,17 @@ managed_broker VARCHAR(1000), stats_curr_id INT, stats_prev_id INT, + queue_id INT, name VARCHAR(1000), - queue_id INT, directory VARCHAR(1000), base_file_name VARCHAR(1000), write_page_size INT, write_pages INT, read_page_size INT, - read_pages INT + read_pages INT, + initial_file_count SMALLINT, + data_file_size INT, + current_file_count INT ); CREATE TABLE journal_stats ( @@ -245,14 +287,15 @@ id_original BIGINT, rec_time TIMESTAMP, journal_id INT, - initial_file_count SMALLINT, - data_file_size INT, - current_file_count INT, record_depth INT, record_depth_low INT, record_depth_high INT, - record_enqueues BIGINT, - record_dequeues BIGINT, + enqueues BIGINT, + dequeues BIGINT, + txn_enqueues BIGINT, + txn_dequeues BIGINT, + txn_commits BIGINT, + txn_aborts BIGINT, outstanding_ai_os INT, outstanding_ai_os_low INT, outstanding_ai_os_high INT, @@ -384,6 +427,102 @@ frames_outstanding INT ); +CREATE TABLE slot ( + id SERIAL PRIMARY KEY, + id_original BIGINT, + rec_time TIMESTAMP, + creation_time TIMESTAMP, + deletion_time TIMESTAMP, + managed_broker VARCHAR(1000), + stats_curr_id INT, + stats_prev_id INT, + arch VARCHAR(1000), + checkpoint_platform VARCHAR(1000), + cpus INT, + disk INT, + file_system_domain VARCHAR(1000), + is_valid_checkpoint_platform VARCHAR(4000), + k_flops INT, + machine VARCHAR(1000), + max_job_retirement_time VARCHAR(4000), + memory INT, + mips INT, + my_address VARCHAR(1000), + name VARCHAR(1000), + op_sys VARCHAR(1000), + requirements VARCHAR(4000), + public_network_ip_addr VARCHAR(1000), + rank VARCHAR(4000), + slot_id INT, + start VARCHAR(4000), + starter_ability_list VARCHAR(4000), + total_cpus INT, + total_disk INT, + total_memory INT, + total_slots INT, + total_virtual_memory INT, + uid_domain VARCHAR(1000), + virtual_memory INT, + windows_build_number INT, + windows_major_version INT, + windows_minor_version INT +); + +CREATE TABLE slot_stats ( + id SERIAL PRIMARY KEY, + id_original BIGINT, + rec_time TIMESTAMP, + slot_id INT, + accounting_group VARCHAR(1000), + activity VARCHAR(1000), + client_machine VARCHAR(1000), + clock_day INT, + clock_min INT, + condor_load_avg FLOAT, + console_idle INT, + current_rank FLOAT, + entered_current_activity INT, + entered_current_state INT, + global_job_id VARCHAR(1000), + image_size INT, + job_id VARCHAR(1000), + job_start INT, + keyboard_idle INT, + last_benchmark INT, + last_fetch_work_completed INT, + last_fetch_work_spawned INT, + last_periodic_checkpoint INT, + load_avg FLOAT, + my_current_time INT, + next_fetch_work_delay INT, + preempting_owner VARCHAR(1000), + preempting_user VARCHAR(1000), + preempting_rank FLOAT, + remote_owner VARCHAR(1000), + remote_user VARCHAR(1000), + state VARCHAR(1000), + time_to_live INT, + total_claim_run_time INT, + total_claim_suspend_time INT, + total_condor_load_avg FLOAT, + total_job_run_time INT, + total_job_suspend_time INT, + total_load_avg FLOAT, + total_time_backfill_busy INT, + total_time_backfill_idle INT, + total_time_backfill_killing INT, + total_time_claimed_busy INT, + total_time_claimed_idle INT, + total_time_claimed_retiring INT, + total_time_claimed_suspended INT, + total_time_matched_idle INT, + total_time_owner_idle INT, + total_time_preempting_killing INT, + total_time_preempting_vacating INT, + total_time_unclaimed_benchmarking INT, + total_time_unclaimed_idle INT +); + CREATE TABLE store ( id SERIAL PRIMARY KEY, id_original BIGINT, @@ -395,16 +534,31 @@ stats_prev_id INT, broker_id INT, location VARCHAR(1000), - async BOOL, default_initial_file_count SMALLINT, - default_data_file_size INT + default_data_file_size INT, + tpl_is_initialized BOOL, + tpl_directory VARCHAR(1000), + tpl_write_page_size INT, + tpl_write_pages INT, + tpl_initial_file_count SMALLINT, + tpl_data_file_size INT, + tpl_current_file_count INT ); CREATE TABLE store_stats ( id SERIAL PRIMARY KEY, id_original BIGINT, rec_time TIMESTAMP, - store_id INT + store_id INT, + tpl_transaction_depth INT, + tpl_transaction_depth_low INT, + tpl_transaction_depth_high INT, + tpl_txn_prepares BIGINT, + tpl_txn_commits BIGINT, + tpl_txn_aborts BIGINT, + tpl_outstanding_ai_os INT, + tpl_outstanding_ai_os_low INT, + tpl_outstanding_ai_os_high INT ); CREATE TABLE system ( @@ -513,6 +667,12 @@ ALTER TABLE exchange_stats ADD CONSTRAINT exchange_id_exists FOREIGN KEY (exchange_id) REFERENCES exchange (id) ON DELETE SET NULL; +ALTER TABLE job ADD CONSTRAINT stats_curr_id_exists FOREIGN KEY (stats_curr_id) REFERENCES job_stats (id) ON DELETE SET NULL; + +ALTER TABLE job ADD CONSTRAINT stats_prev_id_exists FOREIGN KEY (stats_prev_id) REFERENCES job_stats (id) ON DELETE SET NULL; + +ALTER TABLE job_stats ADD CONSTRAINT job_id_exists FOREIGN KEY (job_id) REFERENCES job (id) ON DELETE SET NULL; + ALTER TABLE journal ADD CONSTRAINT stats_curr_id_exists FOREIGN KEY (stats_curr_id) REFERENCES journal_stats (id) ON DELETE SET NULL; ALTER TABLE journal ADD CONSTRAINT stats_prev_id_exists FOREIGN KEY (stats_prev_id) REFERENCES journal_stats (id) ON DELETE SET NULL; @@ -547,6 +707,12 @@ ALTER TABLE session_stats ADD CONSTRAINT session_id_exists FOREIGN KEY (session_id) REFERENCES session (id) ON DELETE SET NULL; +ALTER TABLE slot ADD CONSTRAINT stats_curr_id_exists FOREIGN KEY (stats_curr_id) REFERENCES slot_stats (id) ON DELETE SET NULL; + +ALTER TABLE slot ADD CONSTRAINT stats_prev_id_exists FOREIGN KEY (stats_prev_id) REFERENCES slot_stats (id) ON DELETE SET NULL; + +ALTER TABLE slot_stats ADD CONSTRAINT slot_id_exists FOREIGN KEY (slot_id) REFERENCES slot (id) ON DELETE SET NULL; + ALTER TABLE store ADD CONSTRAINT stats_curr_id_exists FOREIGN KEY (stats_curr_id) REFERENCES store_stats (id) ON DELETE SET NULL; ALTER TABLE store ADD CONSTRAINT stats_prev_id_exists FOREIGN KEY (stats_prev_id) REFERENCES store_stats (id) ON DELETE SET NULL;