[rhmessaging-commits] rhmessaging commits: r3588 - in mgmt/trunk/cumin/python/cumin: grid and 1 other directories.

Author: eallen Date: 2009-08-25 18:28:04 -0400 (Tue, 25 Aug 2009) New Revision: 3588 Modified: mgmt/trunk/cumin/python/cumin/grid/job.py mgmt/trunk/cumin/python/cumin/grid/job.strings mgmt/trunk/cumin/python/cumin/grid/main.py mgmt/trunk/cumin/python/cumin/grid/model.py mgmt/trunk/cumin/python/cumin/grid/slot.py mgmt/trunk/cumin/python/cumin/grid/submission.py mgmt/trunk/cumin/python/cumin/inventory/system.py mgmt/trunk/cumin/python/cumin/model.py mgmt/trunk/cumin/python/cumin/parameters.py mgmt/trunk/cumin/python/cumin/util.py mgmt/trunk/cumin/python/cumin/visualizations.py mgmt/trunk/cumin/python/cumin/visualizations.strings mgmt/trunk/cumin/python/cumin/widgets.py mgmt/trunk/cumin/python/cumin/widgets.strings Log: Change JobSet from sql based to qmf based. Change all Job qmf method calls to use scheduler instead. Modified: mgmt/trunk/cumin/python/cumin/grid/job.py =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/job.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/job.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -16,129 +16,78 @@ strings = StringCatalog(__file__) log = logging.getLogger("cumin.job") -class JobSet(CuminSelectionTable): - def __init__(self, app, name, pool): +class JobSet(CuminQMFSelectionTable): + def __init__(self, app, name, submission): item = JobParameter(app, "item") super(JobSet, self).__init__(app, name, item) - self.pool = pool + self.submission = submission - col = self.CustomIdColumn(app, "custom_id") + self.update_enabled = False + self.defer_enabled = True + + col = self.CustomIdColumn(app, "name") self.add_column(col) self.set_default_column(col) - col = self.GlobalJobIdColumn(app, "global_job_id") - col.visible = False + col = self.CommandColumn(app, "Cmd") self.add_column(col) - col = self.CommandColumn(app, "cmd") + col = self.StatusColumn(app, "JobStatus") self.add_column(col) - col = self.AccountingGroupColumn(app, "agroup") - col.visible = False - self.add_column(col) - - col = self.SchedulerColumn(app, "scheduler") - col.visible = False - self.add_column(col) - - col = self.SubmitterColumn(app, "submitter") - col.visible = False - self.add_column(col) - - col = self.ClusterIdColumn(app, "cluster_id") + col = self.ArgsColumn(app, "Args") col.alignment = "right" - col.visible = False self.add_column(col) - col = self.ConcurrencyLimitsColumn(app, "concurrency_limits") - #col.visible = False - self.add_column(col) - - col = self.CustomGroupColumn(app, "custom_group") - col.visible = False - self.add_column(col) - - col = self.CustomPriorityColumn(app, "custom_priority") - col.visible = False - self.add_column(col) - - col = self.StatusColumn(app, "job_status") - self.add_column(col) - - col = self.TitleColumn(app, "title") - col.visible = False - self.add_column(col) - - col = self.ArgsColumn(app, "args") - col.alignment = "right" - col.visible = False - self.add_column(col) - self.phase = JobStatusSwitch(app, "phase") - self.add_child(self.phase) + self.filters.add_child(self.phase) + multi_param = self.SubmissionJob(app, "subjob", self.selection, submission) + task = main.module.job_set_hold - button = TaskButton(app, "hold", task, self.selection) + button = TaskButton(app, "hold", task, multi_param) self.buttons.add_child(button) task = main.module.job_set_release - button = TaskButton(app, "release", task, self.selection) + button = TaskButton(app, "release", task, multi_param) self.buttons.add_child(button) task = main.module.job_set_remove - button = TaskButton(app, "remove", task, self.selection) + button = TaskButton(app, "remove", task, multi_param) self.buttons.add_child(button) - def get_phase_sql(self, session, phase=None): - return self.phase.get_sql_constraint(session, phase) + def do_get_items(self, session, *args): + submission = self.submission.get(session) + scheduler = submission.scheduler - def get_phase_title(self, session): + action = self.app.model.scheduler.GetJobs + jobs = action.qmfcall(scheduler, submission.Name) + return jobs + + def filter_item(self, session, item): state = self.phase.get(session) - return self.phase.get_title(state) + if state == "All": + return True - def render_sql_where(self, session, *args): - phase = len(args) > 0 and args[0] or None - elems = list() - elems.append(self.get_phase_sql(session, phase)) - elems.append("s.pool = %(pool)s") - return "where %s" % " and ".join(elems) + item_state = item["JobStatus"]["VALUE"] + item_state = JobStatusInfo.get_status_string(int(item_state)) - def render_stats_join(self, session, *args): - phase = self.phase.get(session) - return (phase == 'c') and "left outer join job_stats as c on c.id = j.stats_curr_id" or "" + return state == item_state - def get_sql_values(self, session, *args): - pool = self.pool.get(session) - return {"pool": pool.id} + def render_deferred_content(self, session, *args): + return "Loading..." - class ArgsColumn(SqlTableColumn): + def get_phase_title(self, session): + state = self.phase.get(session) + return self.phase.get_title(state) + + class ArgsColumn(QmfTableColumn): def render_title(self, session, data): return "Arguments" - class ClusterIdColumn(SqlTableColumn): + class CustomIdColumn(QmfTableColumn): def render_title(self, session, data): - return "Cluster Id" - - class ConcurrencyLimitsColumn(SqlTableColumn): - def render_title(self, session, data): - return "Limits" - - class CustomGroupColumn(SqlTableColumn): - def render_title(self, session, data): - return "Job Group" - - def render_content(self, session, data): - name = data[self.name] - - if name: - group = Identifiable(name) - href = self.page.main.grid.pool.job_group.get_href \ - (session, group) - return fmt_link(href, fmt_shorten(name, 12, 3)) - - class CustomIdColumn(SqlTableColumn): - def render_title(self, session, data): return "ID" def render_content(self, session, data): @@ -146,75 +95,34 @@ if id: job = Identifiable(data["id"]) - href = self.page.main.grid.pool.job.get_href(session, job) - return fmt_link(href, fmt_shorten(id, 12, 3)) + scheduler = self.parent.submission.get(session).scheduler + href = self.page.main.grid.pool.job.get_href(session, job, scheduler) + job_id, hash, num = id.rpartition('#') + return fmt_link(href, job_id, link_title=id) - class CustomPriorityColumn(SqlTableColumn): + class StatusColumn(QmfTableColumn): def render_title(self, session, data): - return "Custom Priority" - - class GlobalJobIdColumn(SqlTableColumn): - def render_title(self, session, data): - return "Global Job Id" - - def render_content(self, session, data): - id = data["id"] - - if id: - job = Identifiable(data["id"]) - href = self.frame.job.get_href(session, job) - content = fmt_shorten(data[self.name], 12, 3) - return fmt_link(href, content) - - class StatusColumn(SqlTableColumn): - def render_title(self, session, data): return "Status" def render_content(self, session, data): - stat = data["job_status"] - return JobStatusInfo.get_status_string(stat) + stat = data[self.name] + return JobStatusInfo.get_status_string(int(stat)) - class AccountingGroupColumn(SqlTableColumn): + class CommandColumn(QmfTableColumn): def render_title(self, session, data): - return "Accounting Group" - - class SubmitterColumn(SqlTableColumn): - def render_title(self, session, data): - return "Submitter" - - def render_content(self, session, data): - name = data["submitter"] - - if name: - sub = Identifiable(data["submitter_id"]) - href = self.page.main.grid.pool.submitter.get_href \ - (session, sub) - return fmt_link(href, fmt_shorten(name)) - - class SchedulerColumn(SqlTableColumn): - def render_title(self, session, data): - return "Scheduler" - - def render_content(self, session, data): - name = data["scheduler"] - - if name: - sched = Identifiable(data["scheduler_id"]) - href = self.page.main.grid.pool.scheduler.get_href \ - (session, sched) - return fmt_link(href, fmt_shorten(name)) - - class CommandColumn(ItemTableColumn): - def render_title(self, session, data): return "Command" def render_content(self, session, data): return fmt_shorten(data[self.name], 0, 16) - class TitleColumn(SqlTableColumn): - def render_title(self, session, data): - return "Title" + class SubmissionJob(object): + def __init__(self, app, name, job, submission): + self.job = job + self.submission = submission + def get(self, session): + return (self.job.get(session), self.submission.get(session)) + class TopJobSet(TopTable): def __init__(self, app, name): super(TopJobSet, self).__init__(app, name) @@ -469,6 +377,9 @@ self.object = JobParameter(app, "id") self.add_parameter(self.object) + self.scheduler = SchedulerParameter(app, "scheduler") + self.add_parameter(self.scheduler) + self.view = JobView(app, "view") self.add_mode(self.view) @@ -493,20 +404,23 @@ frame.set_object(session, system) return self.page.set_frame(session, frame) + def get_href(self, session, job, scheduler): + branch = session.branch() + self.show_object(branch, job) + self.scheduler.set(session, scheduler) + return branch.marshal() + class JobView(CuminView): def __init__(self, app, name): super(JobView, self).__init__(app, name) - summary = CuminSummary(app, "summary") - self.add_child(summary) - self.tabs = TabbedModeSet(app, "tabs") self.add_child(self.tabs) self.tabs.add_tab(JobAdsViewer(app, "jobads")) self.tabs.add_tab(JobOutput(app, "output")) - self.tabs.add_tab(JobSystemSet(app, "systems")) - self.tabs.add_tab(CuminDetails(app, "details")) + #self.tabs.add_tab(JobSystemSet(app, "systems")) + #self.tabs.add_tab(CuminDetails(app, "details")) class JobSystemSet(SystemSet): def get_args(self, session): @@ -534,23 +448,32 @@ def get_args(self, session): return self.frame.get_args(session) - def do_get_items(self, session, job): + def do_get_items(self, session, job, scheduler): items = self.items.get(session) if not items: - items = self.gen_items(session, job) + items = self.gen_items(session, job, scheduler) # cache the items self.items.set(session, items) return items - def get_raw_ads(self, session, job): - action = self.app.model.job.getad - return action.do_invoke(job) + def get_raw_ads(self, session, job, scheduler): + if not scheduler: + schedulers = Scheduler.select() + for sched in schedulers: + if sched.statsCurr: + if sched.statsCurr.TotalRunningJobs: + scheduler = sched + break - def gen_items(self, session, job): - job_ads = self.get_raw_ads(session, job) + if scheduler: + action = self.app.model.scheduler.GetAd + return action.qmfcall(scheduler, job.id) - cls = self.app.model.get_class_by_object(job) + def gen_items(self, session, job, scheduler): + job_ads = self.get_raw_ads(session, job, scheduler) + + cls = self.app.model.job return [self.gen_item(x, job_ads[x]["VALUE"], cls, dtype=self.types[job_ads[x]["TYPE"]]) for x in job_ads] # list of dictionaries # each disctionary has: @@ -631,7 +554,7 @@ groups = self.app.model.get_ad_groups() writer = Writer() for group in groups: - self.group_tmpl.render(writer, session, (job, group,)) + self.group_tmpl.render(writer, session, (job, group)) return writer.to_string() def render_group_name(self, session, args): @@ -676,7 +599,8 @@ group = args[1] group_items = list() if job: - all_items = super(JobAdsViewer, self).do_get_items(session, job) + scheduler = self.frame.scheduler.get(session) + all_items = super(JobAdsViewer, self).do_get_items(session, job, scheduler) for item in all_items: if "property" in item: property = item["property"] @@ -698,16 +622,20 @@ self.item_renderer = EditablePropertyRenderer(self, "property_html") + def get_args(self, session): + return tuple() + def do_get_items(self, session, args): job = args[0] group = args[1] - cls = self.app.model.get_class_by_object(job) + cls = self.app.model.job ads = self.ads.get(session) if len(ads): return [self.gen_item(x, ads[x]["value"], cls, path=self.ads.path, dtype=ads[x]["type"], error=ads[x], orig=ads[x]["orig"]) for x in ads if self.is_group(x, cls, group)] else: + scheduler = self.frame.scheduler.get(session) items = super(JobAdsEditor, self).do_get_items(session, args) for item in items: item["path"] = self.ads.path @@ -721,13 +649,13 @@ item_group = "Other" return item_group == group - def process_cancel(self, session, job): + def process_cancel(self, session): branch = session.branch() self.ads.set(branch, None) # otherwise url is too long self.frame.view.show(branch) self.page.set_redirect_url(session, branch.marshal()) - def process_submit(self, session, job): + def process_submit(self, session): ads = self.ads.get(session) errors = False @@ -765,11 +693,46 @@ just_ads[unicode(field)] = fval if not errors: + scheduler = self.frame.scheduler.get(session) + job = self.frame.get_object(session) + task = main.module.job_setattribute for field in just_ads: - action = self.app.model.job.setattribute - action.invoke(job, [field, just_ads[field]]) - self.process_cancel(session, job) + task.invoke(session, job.id, field, just_ads[field], scheduler) + self.process_cancel(session) +class OutputFile(Widget): + def __init__(self, app, name, which_file, first_last): + super(OutputFile, self).__init__(app, name) + + self.which_file = which_file + self.first_last = first_last + + self.defer_enabled = True + + def render_content(self, session): + scheduler = self.frame.scheduler.get(session) + job = self.frame.get_object(session) + file, start, end = self.get_file_args(session) + if file: + action = self.app.model.scheduler.Fetch + data = action.qmfcall(scheduler, job, file, start, end) + return escape_entity(data) + + def get_file_args(self, session): + first_last = self.first_last.get(session) + if first_last == "t": + start = -2048 + end = 0 + else: + start = 0 + end = 2048 + file = self.which_file.get_current_file_name(session) + return (file, start, end) + + def render_loading(self, session, *args): + file = self.which_file.get_current_file_name(session) + return file and "loading..." or "Output, Error, and UserLog file names are invalid." + class JobOutput(Form): def __init__(self, app, name): super(JobOutput, self).__init__(app, name) @@ -783,29 +746,42 @@ self.__fetch = self.FetchButton(app, "refresh") self.add_child(self.__fetch) - self.output = self.OutputFile(app, "job_output") + self.output = OutputFile(app, "job_output", self.which_file, self.first_last) self.add_child(self.output) + self.ads = JobAdsSet(app, "ads") + def get_args(self, session): return self.frame.get_args(session) def render_title(self, session, *args): return "Output" - def do_render(self, session, *args): + def render_is_tail(self, session, *args): + tail = self.first_last.get(session) + return tail == "t" and "1" or "0" + + def render_out_time(self, session, *args): + now = datetime.now() + return fmt_datetime(now, sec=True) + + def do_process(self, session, *args): job = args[0] - out_file = job.Out - user_file = job.UserLog - # get err_file from JobAd + + out_file = None + user_file = None err_file = None - if job.Ad: - try: - if "Err" in job.Ad: - err_file = job.Ad["Err"]["VALUE"] - err_file = strip_string_quotes(err_file) - except Exception, e: - pass + scheduler = self.frame.scheduler.get(session) + ads = self.ads.do_get_items(session, job, scheduler) + for ad in ads: + if ad['name'] == "Out": + out_file = ad['value'] + elif ad['name'] == "UserLog": + user_file = ad['value'] + elif ad['name'] == "Err": + err_file = ad['value'] + # set title for radiotab so mouseover will display file name self.which_file.set_file_name(session, "o", out_file) self.which_file.set_file_name(session,"e", err_file) @@ -818,37 +794,8 @@ if self.which_file.is_bad(user_file): self.which_file.disable(session, "u") - return super(JobOutput, self).do_render(session, *args) + return super(JobOutput, self).do_process(session, *args) - def render_loading(self, session, *args): - file = self.which_file.get_current_file_name(session) - return file and "loading..." or "Output, Error, and UserLog file names are invalid." - - def get_file_args(self, session): - first_last = self.first_last.get(session) - if first_last == "t": - start = -2048 - end = 0 - else: - start = 0 - end = 2048 - file = self.which_file.get_current_file_name(session) - return (file, start, end) - - class OutputFile(AjaxField): - def get_url(self, session): - job = self.parent.frame.get_args(session)[0] - if job: - file, start, end = self.parent.get_file_args(session) - if file: - fl = self.parent.first_last.get(session) - args = "%s&%i&%i&%s" % (file, start, end, fl) - args = escape_amp(args) - return "call.xml?class=job;id=%i;method=fetch;xargs=%s" % (job.id, args) - - def do_render(self, session): - return self.render_script(session) - class FetchButton(FormButton): def render_content(self, session): return "Refresh" @@ -999,6 +946,9 @@ self.reason = self.ReasonField(app, "reason") self.add_field(self.reason) + self.scheduler = SchedulerParameter(app, "scheduler") + self.add_parameter(self.scheduler) + def init(self): assert self.object super(JobReasonForm, self).init() @@ -1010,23 +960,26 @@ return "Cancel" def process_submit(self, session): - object = self.object.get(session) + job = self.object.get(session) reason = self.get_reason(session, self.verb) + scheduler = self.scheduler.get(session) if not reason: error = FormError("Reason is required") self.errors.add(session, error) if not self.errors.get(session): - self.task.invoke(session, object, reason) - self.task.exit_with_redirect(session, object) + self.task.invoke(session, job, reason, scheduler) + self.task.exit_with_redirect(session, job) def get_reason(self, session, verb): """ returns <verb> by username[: <user input reason>] """ - reason = [self.reason.get(session)] - verb_by = "%s by %s" % (verb, session.user_session.subject.name) - reason.insert(0, verb_by) - return ": ".join(reason) + reason = self.reason.get(session) + if reason: + reason = [reason] + verb_by = "%s by %s" % (verb, session.user_session.subject.name) + reason.insert(0, verb_by) + return ": ".join(reason) class ReasonField(StringField): def render_title(self, session): @@ -1060,12 +1013,12 @@ def __init__(self, app, name): super(JobStatusSwitch, self).__init__(app, name) - self.add_state("a", "All") - self.add_state("r", "Running") - self.add_state("i", "Idle") - self.add_state("h", "Held") - self.add_state("c", "Completed") - self.add_state("d", "Removed") + self.add_state("All", "All") + self.add_state("Running", "Running") + self.add_state("Idle", "Idle") + self.add_state("Held", "Held") + self.add_state("Completed", "Completed") + self.add_state("Removed", "Removed") def get_sql_constraint(self, session, phase=None): if not phase: @@ -1098,30 +1051,3 @@ return sql -class JobStatusInfo(object): - stat_strings = ["Unexpanded", "Idle", "Running", "Removed", "Completed", "Held", "Submission Error"] - stat_colors = ["red", "clear", "green", "black", "blue", "yellow", "red"] - @classmethod - def get_status_string(cls, stat): - try: - return cls.stat_strings[stat] - except: - return "" - - @classmethod - def get_status_color(cls, stat): - try: - return cls.stat_colors[stat] - except: - return "red" - - @classmethod - def get_status_int(cls, stat): - try: - return cls.stat_strings.index(stat) - except: - return -1 - - @classmethod - def get_zipped_colors(cls): - return zip(cls.stat_strings, cls.stat_colors) Modified: mgmt/trunk/cumin/python/cumin/grid/job.strings =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/job.strings 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/job.strings 2009-08-25 22:28:04 UTC (rev 3588) @@ -226,7 +226,7 @@ </form> [JobOutput.css] -textarea#job_output { +textarea.job_output { height: 25em; width: 100%; border: 1px solid #EAEAEA; @@ -253,12 +253,11 @@ } } -function outputEnd() { - var tarea = document.getElementById("job_output"); +function outputEnd(id) { + var tarea = $(id).getElement("textarea"); if (tarea) { scrollToEnd(tarea); - setTimeout("get_job_output()", 5000); } } @@ -266,48 +265,38 @@ <form id="{id}" style="width:100%; border:0px;" class="mform" method="post" action="?"> <div class="out_actions"> <div class="rfloat">{first_last}</div> - {file} </div> - <div class="sactions refresh_info">{refresh} <h2>Last refresh was at <span id="out_time"></span></h2></div> + <div class="sactions refresh_info">{refresh} <h2>Last refresh was at {out_time}</h2></div> <div> - <textarea name="job_output" id="job_output" rows="20" cols="80"> - {loading}{job_output} - </textarea> - + {job_output} {hidden_inputs} </div> </form> - -[OutTime.javascript] -function got_out_time(obj, id) { - var elem = document.getElementById(id); - - if (elem) { - var str = obj.time.value; - elem.innerHTML = str; +<script type="text/javascript"> +<![CDATA[ +(function() { + var scrolled = false; + if ({is_tail}) { + wooly.addPageUpdateListener( function () { + if (!scrolled) { + outputEnd('{id}'); + scrolled = true; + } + } ); } -} +}()) +]]> +</script> -[OutputFile.javascript] -function got_job_output(obj, id) { - var elem = document.getElementById(id); +[OutputFile.html] +<textarea id="{id}" name="job_output" class="job_output" rows="20" cols="80"> + {content} +</textarea> - if (elem) { - var str = obj.fetch.output; - elem.value = str; - - if (obj.fetch.tail == "t") { - outputEnd(); - } - - var d = new Date(); - var out_time = document.getElementById("out_time"); - - if (out_time) { - out_time.innerHTML = d.toLocaleString(); - } - } -} +[OutputFile.deferred_html] +<textarea id="{id}" name="job_output" class="job_output" rows="20" cols="80"> + {loading} +</textarea> Modified: mgmt/trunk/cumin/python/cumin/grid/main.py =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/main.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/main.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -20,6 +20,7 @@ self.job_set_release = JobSetReleaseTask(app, job) self.job_remove = JobRemoveTask(app, job) self.job_set_remove = JobSetRemoveTask(app, job) + self.job_setattribute = JobSetattributeTask(app, job) scheduler = app.model.scheduler self.scheduler_start = SchedulerStartTask(app, scheduler) Modified: mgmt/trunk/cumin/python/cumin/grid/model.py =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/model.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/model.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -277,29 +277,41 @@ self.form = form self.verb = verb + def get_href(self, session, object): + fsession = self.enter(session, object) + #XXX fix this. what happens if this is called from the overview or inventory tab? + scheduler = self.cls.frame.scheduler.get(session) + self.form.scheduler.set(fsession, scheduler) + return fsession.marshal() + def do_enter(self, session, job): self.form.object.set(session, job) def do_exit(self, session, job): - self.app.main_page.main.grid.job.view.show(session) + self.app.main_page.main.grid.pool.job.view.show(session) def get_title(self, session): return self.verb class JobSetBaseTask(SetTask): - def do_enter(self, session, jobs): - self.form.object.set(session, jobs) + def get_href(self, session, object): + job, submission = object + fsession = self.enter(session, job) + self.form.scheduler.set(fsession, submission.scheduler) + return fsession.marshal() + def do_enter(self, session, job): + self.form.object.set(session, job) + class JobHoldTask(JobBaseTask): def __init__(self, app, cls): form = JobTaskForm(app, "job_hold", self, "Hold") super(JobHoldTask, self).__init__(app, cls, form, "Hold") - def do_invoke(self, completion, session, job, reason): - assert isinstance(job, Job) + def do_invoke(self, completion, session, job, reason, scheduler): + assert isinstance(scheduler, Scheduler) + scheduler.Hold(self.app.model.mint.model, completion, job, reason) - job.Hold(self.app.model.mint.model, completion, reason) - class JobSetHoldTask(JobSetBaseTask): def __init__(self, app, cls): super(JobSetHoldTask, self).__init__(app, cls) @@ -312,11 +324,10 @@ form = JobTaskForm(app, "job_release", self, "Release") super(JobReleaseTask, self).__init__(app, cls, form, "Release") - def do_invoke(self, completion, session, job, reason): - assert isinstance(job, Job) + def do_invoke(self, completion, session, job, reason, scheduler): + assert isinstance(scheduler, Scheduler) + scheduler.Release(self.app.model.mint.model, completion, job, reason) - job.Release(self.app.model.mint.model, completion, reason) - class JobSetReleaseTask(JobSetBaseTask): def __init__(self, app, cls): super(JobSetReleaseTask, self).__init__(app, cls) @@ -329,11 +340,10 @@ form = JobTaskForm(app, "job_remove", self, "Remove") super(JobRemoveTask, self).__init__(app, cls, form, "Remove") - def do_invoke(self, completion, session, job, reason): - assert isinstance(job, Job) + def do_invoke(self, completion, session, job, reason, scheduler): + assert isinstance(scheduler, Scheduler) + scheduler.Remove(self.app.model.mint.model, completion, job, reason) - job.Remove(self.app.model.mint.model, completion, reason) - class JobSetRemoveTask(JobSetBaseTask): def __init__(self, app, cls): super(JobSetRemoveTask, self).__init__(app, cls) @@ -341,6 +351,18 @@ self.form = JobSetTaskForm(app, "job_set_remove", self, "Remove") self.item_task = JobRemoveTask(app, cls) +class JobSetattributeTask(QmfTask): + def __init__(self, app, cls): + super(JobSetattributeTask, self).__init__(app, cls) + self.navigable = False + + def do_invoke(self, completion, session, job, name, value, scheduler): + assert isinstance(scheduler, Scheduler) + scheduler.SetAttribute(self.app.model.mint.model, completion, job, name, str(value)) + + def get_title(self, session): + return "Set Job Attribute" + class NegotiatorGroupTask(QmfTask): def __init__(self, app, cls): super(NegotiatorGroupTask, self).__init__(app, cls) Modified: mgmt/trunk/cumin/python/cumin/grid/slot.py =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/slot.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/slot.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -41,7 +41,7 @@ col = self.LoadAvgColumn(app, "load_avg") self.add_column(col) - col = self.JobColumn(app, "jid") + col = self.JobColumn(app, "job_id") self.add_column(col) class NameColumn(SqlTableColumn): @@ -61,26 +61,27 @@ def render_title(self, session, data): return "State" - class StateColumn(SqlTableColumn): - def render_title(self, session, data): - return "State" - class LoadAvgColumn(SqlTableColumn): def render_title(self, session, data): return "Load" def render_value(self, session, value): - return "%2.02f" % value + if value: + return "%2.02f" % value + else: + return fmt_none_brief() class JobColumn(SqlTableColumn): def render_title(self, session, data): return "Current Job" def render_content(self, session, data): - if data["jid"]: - job = Identifiable(data["jid"]) - href = self.page.main.grid.pool.job.get_href(session, job) - return fmt_link(href, data["job_id"]) + if data[self.name]: + job = Identifiable(data[self.name]) + pool = self.frame.get_object(session) + scheduler = Scheduler.select("pool='%s'" % pool.id)[0] + href = self.page.main.grid.pool.job.get_href(session, job, scheduler) + return fmt_link(href, data[self.name]) def render_items(self, session, *args): """ overridden because a slotset query is expensive. Modified: mgmt/trunk/cumin/python/cumin/grid/submission.py =================================================================== --- mgmt/trunk/cumin/python/cumin/grid/submission.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/grid/submission.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -2,6 +2,7 @@ from cumin.widgets import * from cumin.util import * +from job import JobSet import main @@ -108,12 +109,7 @@ self.jobs = SubmissionJobSet(app, "jobs", submission) self.tabs.add_tab(self.jobs) -class SubmissionJobSet(Widget): - def __init__(self, app, name, submission): - super(SubmissionJobSet, self).__init__(app, name) - - self.submission = submission - +class SubmissionJobSet(JobSet): def render_title(self, session): return "Jobs" Modified: mgmt/trunk/cumin/python/cumin/inventory/system.py =================================================================== --- mgmt/trunk/cumin/python/cumin/inventory/system.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/inventory/system.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -145,7 +145,7 @@ def render_slot_job_url(self, session): job = Identifiable("XXX") - return self.page.main.grid.pool.job.get_href(session, job) + return self.page.main.grid.pool.job.get_href(session, job, None) class SystemSlotMap(SlotMap): def get_title_name(self, session, sysimage): @@ -165,7 +165,7 @@ self.add_child(self.__tabs) self.__tabs.add_tab(SystemStats(app, "stats")) - self.__tabs.add_tab(SystemJobSet(app, "jobs", None)) + #self.__tabs.add_tab(SystemJobSet(app, "jobs", submission)) self.__tabs.add_tab(SystemSlotSet(app, "slots")) self.__tabs.add_tab(SystemServices(app, "services")) self.__tabs.add_tab(CuminDetails(app, "details")) @@ -246,7 +246,8 @@ brokers = Broker.select() for broker in brokers: if broker.system.nodeName == system.nodeName: - daemons.append(BrokerRegistration.get(broker.registrationID)) + if broker.registrationID: + daemons.append(BrokerRegistration.get(broker.registrationID)) return daemons @@ -258,7 +259,7 @@ else: pool = model.Pool.get(item.Pool) self.page.main.grid.pool.object.set(session, pool) - self.page.main.show_grid_tab(session) + #self.page.main.show_grid_tab(session) daemon = Identifiable(item.id) if isinstance(item, Collector): self.page.main.grid.pool.view.set_collector_tab(session) Modified: mgmt/trunk/cumin/python/cumin/model.py =================================================================== --- mgmt/trunk/cumin/python/cumin/model.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/model.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -384,7 +384,7 @@ return value class AdProperty(CuminProperty): - groups = ["Main", "Command Info", "Job Status Info", "Condor Info", "Other"] + groups = ["Main", "Command Info", "Job Status Info", "Condor Info", "Dates", "Other"] def __init__(self, cls, name): # don't call super since we don't want to call add_property @@ -1899,9 +1899,19 @@ def get_icon_href(self, session): return "resource?name=group-36.png" -class CuminJob(RemoteClass): +class Job(object): + def __init__(self, id): + + self.id = id + self.name = id + + def get(cls, id): + return Job(id) + get = classmethod(get) + +class CuminJob(CuminClass): def __init__(self, model): - super(CuminJob, self).__init__(model, "job", Job, JobStats) + super(CuminJob, self).__init__(model, "job", Job) ### Main Group prop = AdProperty(self, "Owner") @@ -1979,47 +1989,60 @@ prop = DateAdProperty(self, "QDate") prop.description = "When the job was submitted" + prop.group = "Dates" prop.writable = False + prop = DateAdProperty(self, "LastVacateTime") + prop.group = "Dates" + prop.writable = False + prop = DateAdProperty(self, "JobStartDate") prop.group = "Main" prop.writable = False prop = DateAdProperty(self, "JobCurrentStartDate") + prop.group = "Dates" prop.writable = False + prop = DateAdProperty(self, "JobLastStartDate") + prop.group = "Dates" + prop.writable = False + prop = DateAdProperty(self, "LastSuspensionTime") + prop.group = "Dates" prop.writable = False prop = DateAdProperty(self, "ShadowBday") + prop.group = "Dates" prop.writable = False prop = DateAdProperty(self, "LastJobLeaseRenewal") + prop.group = "Dates" prop.writable = False prop = DateAdProperty(self, "EnteredCurrentStatus") + prop.group = "Dates" prop.writable = False prop = DateAdProperty(self, "CommittedTime") + prop.group = "Dates" prop.writable = False prop = DateAdProperty(self, "LastMatchTime") + prop.group = "Dates" prop.writable = False ######## Properties - prop = self.GroupProperty(self, "CustomGroup") + prop = CuminProperty(self, "CustomGroup") prop.title = "Job Group" - prop.summary = True prop.escape = False - prop = self.SchedulerProperty(self, "scheduler") + prop = CuminProperty(self, "scheduler") prop.title = "Scheduler" - prop.summary = True prop.escape = False - prop = self.SubmitterProperty(self, "submitter") + prop = CuminProperty(self, "submitter") prop.title = "Submitter" - prop.summary = True prop.escape = False prop = CuminProperty(self, "AccountingGroup") @@ -2070,15 +2093,6 @@ prop = CuminProperty(self, "HoldReason") prop.title = "Hold Reason" - action = self.SetAttribute(self, "setattribute") - action.navigable = False - - action = self.GetAd(self, "getad") - action.navigable = False - - action = self.Fetch(self, "fetch") - action.navigable = False - def init(self): super(CuminJob, self).init() @@ -2088,7 +2102,10 @@ return "Job" def get_object_name(self, job): - return job.CustomId + if isinstance(job, basestring): + return job + else: + return job.name def get_icon_href(self, session): return "resource?name=job-36.png" @@ -2097,102 +2114,6 @@ def render_status(self, session, status): return JobStatusInfo.get_status_string(status) - class GroupProperty(CuminProperty): - def value(self, session, job): - group = JobGroup(job.CustomGroup) - href = self.model.frame.grid.pool.job_group.get_href \ - (session, group) - content = fmt_shorten(group.id, 12, 3) - return fmt_link(href, group.get_id()) - - class SubmitterProperty(CuminProperty): - def value(self, session, job): - sub = job.submitter - href = self.model.frame.grid.pool.submitter.get_href(session, sub) - return fmt_link(href, sub.Name) - - class SchedulerProperty(CuminProperty): - def value(self, session, job): - sched = job.scheduler - href = self.model.frame.grid.pool.scheduler.get_href \ - (session, sched) - return fmt_link(href, sched.Name) - - class GetAd(CuminAction): - def do_invoke(self, job): - self.job_ads = {"":{"VALUE": "", "TYPE": 0}} - self.got_data = False - - def completion(status, job_ads): - try: - self.job_ads = job_ads["JobAd"] - self.got_data = True - except: - pass - - def predicate(): - return self.got_data - - try: - job.GetAd(self.mint.model, completion, None) - except Exception, e: - return self.job_ads - - # wait for up to 20 seconds for completion to be called - wait(predicate, timeout=20) - return self.job_ads - - class Fetch(CuminAction): - def get_xml_response(self, session, job, *args): - file, start, end, tail = args - self.job_output = None - self.got_data = False - self.reason = None - - def completion(status, job_output): - if "Data" in job_output: - raw = job_output["Data"] - #remove the last partial line from the buffer - lindex = -1 - ord_nl = ord('\n') - while ord(raw[-lindex:][0]) != ord_nl: - lindex = lindex + 1 - self.job_output = raw[:-lindex] - else: - self.reason = status - - self.got_data = True - - def predicate(): - return self.got_data - - try: - data = None - job.Fetch(self.mint.model, completion, file, start, end, data) - # wait for up to 20 seconds for completion to be called - wait(predicate, timeout=20) - if not self.got_data: - self.reason = "time out" - except Exception, e: - self.reason = e - - if self.reason: - self.job_output = "Unable to get file at %s. Reason: %s" % (file, self.reason) - - return self.job_output and "<fetch><output>%s</output><tail>%s</tail></fetch>" % (escape_entity(self.job_output), tail) - - class SetAttribute(CuminAction): - def show(self, session, job): - pass - - def get_title(self, session): - return "Save Ad for" - - def do_invoke(self, job, args, completion): - Name = args[0] - Value = args[1] - job.SetAttribute(self.mint.model, completion, Name, str(Value)) - class GetStartedAction(CuminAction): def get_xml_response(self, session, object, *args): updateTime = object.statsCurr and object.statsCurr.qmfUpdateTime \ @@ -2203,6 +2124,27 @@ return "%s%s" % (conf, rect) +class QMFCaller(object): + def __init__(self, default): + self.data = default + self.got_data = False + self.error = False + self.status = None + + def get_completion(self): + def completion(status, data): + self.status = status + if (status == 0) or (status == "OK"): + self.data = data + self.got_data = True + else: + self.error = True + + return completion + + def done(self): + return self.got_data or self.error + class CuminScheduler(RemoteClass): def __init__(self, model): super(CuminScheduler, self).__init__(model, "scheduler", @@ -2251,6 +2193,15 @@ action = GetStartedAction(self, "GetStarted") action.navigable = False + action = self.GetJobs(self, "GetJobs") + action.navigable = False + + action = self.GetAd(self, "GetAd") + action.navigable = False + + action = self.Fetch(self, "Fetch") + action.navigable = False + def init(self): super(CuminScheduler, self).init() @@ -2262,6 +2213,35 @@ def get_object_name(self, sched): return sched.Name + class Fetch(CuminAction): + def qmfcall(self, scheduler, job, file, start, end): + default = {'Data': ""} + qmf = QMFCaller(default) + scheduler.Fetch(self.mint.model, qmf.get_completion(), job.id, file, start, end, None) + wait(qmf.done, timeout=10) + + return qmf.data['Data'] + + class GetAd(CuminAction): + def qmfcall(self, scheduler, job): + + default = {'JobAd': {"":{"VALUE":"", "TYPE":0}}} + qmf = QMFCaller(default) + scheduler.GetAd(self.mint.model, qmf.get_completion(), job, None) + wait(qmf.done, timeout=10) + + return qmf.data['JobAd'] + + class GetJobs(CuminAction): + def qmfcall(self, scheduler, submission): + + default = {'Jobs': {}} + qmf = QMFCaller(default) + scheduler.GetJobs(self.mint.model, qmf.get_completion(), submission, None) + wait(qmf.done, timeout=10) + + return qmf.data['Jobs'] + class CuminSubmission(RemoteClass): def __init__(self, model): super(CuminSubmission, self).__init__(model, "submission", @@ -2433,29 +2413,15 @@ class GetRawConfig(CuminAction): def do_invoke(self, negotiator, config_name, timeout=5): - self.data = None - self.got_data = False - self.error = False + default = {'Value': ""} + qmf = QMFCaller(default) + negotiator.GetRawConfig(self.mint.model, qmf.get_completion(), config_name, None) + wait(qmf.done, timeout=timeout) + return qmf.data - def completion(status, data): - self.data = data - self.got_data = True - - def predicate(): - return self.got_data or self.error - - try: - negotiator.GetRawConfig(self.mint.model, completion, config_name, None) - except Exception, e: - self.error = True - - wait(predicate, timeout=timeout) - return self.data - class GetConfigSet(CuminAction): - """ send a set or qmf requests at the same time and wait for + """ send a set of qmf requests at the same time and wait for them all to complete or timeout """ - def do_invoke(self, negotiator, groups, prepend="", method="GetStats", timeout=5): def predicate(calls): done = 0 @@ -2465,70 +2431,31 @@ return done == len(calls) calls = list() + default = {'Value': 0} for group in groups: - call = self.StatGetter() + call = QMFCaller(default) calls.append(call) - call.invoke(self.mint.model, negotiator, group, prepend, method, {'Value': 0}) + if method == "GetStats": + negotiator.GetStats(self.mint.model, call.get_completion(), group, None) + elif method == "GetRawConfig": + negotiator.GetRawConfig(self.mint.model, call.get_completion(), prepend+group, None) wait(predicate, timeout=timeout, args=calls) - return [(call.group, call.data, (call.status, call.error, call.got_data)) - for call in calls] + return [(group, call.data, (call.status, call.error, call.got_data)) + for call, group in zip(calls, groups)] - class StatGetter(object): - def invoke(self, model, negotiator, group, prepend, method, default): - self.data = default - self.group = group - self.got_data = False - self.error = False - self.status = None - - def completion(status, data): - self.status = status - if (status == 0) or (status == "OK"): - self.data = data - self.got_data = True - else: - self.error = True - - try: - if method == "GetStats": - negotiator.GetStats(model, completion, group, None) - elif method == "GetRawConfig": - negotiator.GetRawConfig(model, completion, prepend+group, None) - except Exception, e: - self.error = True - - def done(self): - return self.got_data or self.error - class GetLimits(CuminAction): def do_invoke(self, negotiator): - self.lim = dict() - self.got_data = False - self.error = False + default = {'Limits': {}} + qmf = QMFCaller(default) + negotiator.GetLimits(self.mint.model, qmf.get_completion(), None) + wait(qmf.done, timeout=2) - def completion(status, data): - try: - self.lim = data["Limits"] - self.got_data = True - except KeyError: - self.error = True + if not qmf.got_data: + return {'timeout': True} + return qmf.data['Limits'] - def predicate(): - return self.got_data or self.error - - try: - negotiator.GetLimits(self.mint.model, completion, None) - except Exception, e: - return self.lim - - # wait a couple of seconds for completion to be called - wait(predicate, timeout=2) - if not self.got_data: - self.lim["timeout"] = True - return self.lim - class CuminSubject(CuminClass): def __init__(self, model): super(CuminSubject, self).__init__(model, "subject", Subject) Modified: mgmt/trunk/cumin/python/cumin/parameters.py =================================================================== --- mgmt/trunk/cumin/python/cumin/parameters.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/parameters.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -78,10 +78,12 @@ class JobParameter(Parameter): def do_unmarshal(self, string): - return Job.get(int(string)) + return model.Job(string) def do_marshal(self, job): - return str(job.id) + if isinstance(job, basestring): + return job + return job.id class JobGroupParameter(Parameter): def do_unmarshal(self, string): Modified: mgmt/trunk/cumin/python/cumin/util.py =================================================================== --- mgmt/trunk/cumin/python/cumin/util.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/util.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -145,3 +145,31 @@ if value[:1] == "\"" and value[-1:] == "\"": dvalue = value[1:-1] return dvalue + +class JobStatusInfo(object): + stat_strings = ["Unexpanded", "Idle", "Running", "Removed", "Completed", "Held", "Submission Error"] + stat_colors = ["red", "clear", "green", "black", "blue", "yellow", "red"] + @classmethod + def get_status_string(cls, stat): + try: + return cls.stat_strings[stat] + except: + return "" + + @classmethod + def get_status_color(cls, stat): + try: + return cls.stat_colors[stat] + except: + return "red" + + @classmethod + def get_status_int(cls, stat): + try: + return cls.stat_strings.index(stat) + except: + return -1 + + @classmethod + def get_zipped_colors(cls): + return zip(cls.stat_strings, cls.stat_colors) Modified: mgmt/trunk/cumin/python/cumin/visualizations.py =================================================================== --- mgmt/trunk/cumin/python/cumin/visualizations.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/visualizations.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -49,7 +49,15 @@ def render_slot_job_url(self, session, *args): job = Identifiable("XXX") - return self.page.main.grid.pool.job.get_href(session, job) + object = args[0] + try: + if isinstance(object, Sysimage): + scheduler = Scheduler.select("system='%s'" % object.nodeName)[0] + else: + scheduler = Scheduler.select("pool='%s'" % object.id)[0] + return self.page.main.grid.pool.job.get_href(session, job, scheduler) + except: + pass def render_slot_clip_size(self, session, *args): return 400 @@ -150,7 +158,8 @@ return item[1] def render_job_id(self, session, item): - return item[0] == "jid" and "id='job_id'" or "" + #return item[0] == "jid" and "id='job_id'" or "" + return item[0] == "Job ID" and "id='job_id'" or "" def render_row_class(self, session, item): return item[0] == "jid" and "class='hidden_row'" or "" Modified: mgmt/trunk/cumin/python/cumin/visualizations.strings =================================================================== --- mgmt/trunk/cumin/python/cumin/visualizations.strings 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/visualizations.strings 2009-08-25 22:28:04 UTC (rev 3588) @@ -917,6 +917,7 @@ div#slot_visualization { position: relative; display: none; + float: left; } div.zoom_node { @@ -1015,6 +1016,7 @@ <div id="slot_pandown" onclick="vis.pan_down()"></div> {slot_legend} </div> +<div style="clear: left"><!-- --></div> {slot_info} <script type="text/javascript"> var slot_current_id = "{id}"; Modified: mgmt/trunk/cumin/python/cumin/widgets.py =================================================================== --- mgmt/trunk/cumin/python/cumin/widgets.py 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/widgets.py 2009-08-25 22:28:04 UTC (rev 3588) @@ -682,6 +682,14 @@ cls = super(ClientTruncateColumn, self).get_class_attr(session) return "width=\"%i%%\" %s" % (self.col_percent, cls) +class QmfTableColumn(ItemTableColumn): + def render_content(self, session, data): + key = self.get_column_key(session) + return self.render_value(session, data[key]) + + def render_value(self, session, value): + return str(value) + class TopTableColumn(SqlTableColumn): def __init__(self, app, name): super(TopTableColumn, self).__init__(app, name) @@ -830,6 +838,57 @@ self.links.html_class = "actions" # XXX fix this self.add_child(self.links) +class CuminQMFSelectionTable(CuminSelectionTable): + def filter_item(self, session, item): + return True + + def get_item_count(self, session, *args): + items = self.items.get(session) + if not items: + items = self.do_get_items(session, *args) + + self.items.set(session, items) + + count = 0 + for item in items: + if self.filter_item(session, items[item]): + count += 1 + return count + + def do_get_items(self, session, *args): + raise Exception("not implemented") + + def render_items(self, session, *args): + items = self.get_items(session, *args) + rows = list() + for item in items: + data = dict() + # filter them + if self.filter_item(session, items[item]): + data['id'] = item + data['name'] = item + for datum in items[item]: + data[datum] = items[item][datum]['VALUE'] + rows.append(data) + + # sort them + scol = self.get_selected_column(session) + rows = sorted(rows, key=lambda x: x[scol.name], reverse=self.reversed.get(session)) + + # limit them + page = self.paginator.page_index.get(session) + items_per_page = self.paginator.page_size + + start = page * items_per_page + stop = start + items_per_page + rows = rows[start:stop] + + # render them + writer = Writer() + for row in rows: + self.item_tmpl.render(writer, session, row) + return writer.to_string() + class NullSortColumn(SqlTableColumn): def get_order_by_sql(self, session): key = self.get_column_key(session) Modified: mgmt/trunk/cumin/python/cumin/widgets.strings =================================================================== --- mgmt/trunk/cumin/python/cumin/widgets.strings 2009-08-25 15:20:11 UTC (rev 3587) +++ mgmt/trunk/cumin/python/cumin/widgets.strings 2009-08-25 22:28:04 UTC (rev 3588) @@ -484,6 +484,10 @@ display: inline; } +div.CuminTable form { + clear: both; +} + table.CuminTable td { text-overflow:ellipsis; } @@ -595,11 +599,12 @@ [CheckboxColumnHeader.javascript] function checkAll(control_id, form_id, elem_name) { - control = document.getElementById(control_id) - form = document.getElementById(form_id); + var control = document.getElementById(control_id) + var div = document.getElementById(form_id); + var form = $(div).getElement("form"); - for (i = 0; i < form.elements.length; i++) { - elem = form.elements[i]; + for (var i = 0; i < form.elements.length; i++) { + var elem = form.elements[i]; if (elem.name == elem_name) { if (elem.checked != control.checked)