Author: eallen Date: 2010-07-09 16:15:59 -0400 (Fri, 09 Jul 2010) New Revision: 4096 Modified: mgmt/newdata/cumin/python/cumin/grid/negotiator.py mgmt/newdata/cumin/python/cumin/grid/negotiator.strings mgmt/newdata/cumin/python/cumin/model.py Log: Rework negotiator configs to better handle hundreds of groups/users Modified: mgmt/newdata/cumin/python/cumin/grid/negotiator.py =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/negotiator.py 2010-07-09 15:39:44 UTC (rev 4095) +++ mgmt/newdata/cumin/python/cumin/grid/negotiator.py 2010-07-09 20:15:59 UTC (rev 4096) @@ -25,14 +25,15 @@ super(NegotiatorFrame, self).__init__(app, name, cls) self.group_add = NegotiatorGroupAdd(app, self) - self.edit_dynamic_quota = NegotiatorEditDynamicQuota(app, self) + """ self.edit_static_quota = NegotiatorEditStaticQuota(app, self) self.edit_prio_factor = NegotiatorEditPrioFactor(app, self) self.edit_regroup = NegotiatorEditRegroup(app, self) self.user_prio_factor = NegotiatorUserPrioFactor(app, self) self.user_regroup = NegotiatorUserRegroup(app, self) - + """ + overview = NegotiatorOverview(app, "overview", self.object, self) self.view.add_tab(overview) @@ -64,7 +65,7 @@ self.title = None self.getter = getter - self.align = "right" + #self.align = "right" self.user = False self.negotiator = negotiator self.task = task @@ -79,21 +80,10 @@ if x[0] == group: return self.render_data(session, x) - if self.user: - data = self.getter(session, [group], True) - for x in data: - if x[0] == group: - return self.render_data(session, x) - return "" def render_data(self, session, data): - # is user - if "." in data[0] and self.user_task: - href = "%s" % self.user_task.get_href \ - (session, (self.negotiator.get(session))) - else: - href = self.task.get_href(session) + href = self.task.get_href(session) content = data[2][2] and str(data[1]) or "NOT SET" return fmt_link(href, content, "", "", self.fmt_hover(data[0])) @@ -110,19 +100,12 @@ self.group_helper = GroupHelper(app, "groups", negotiator) self.add_child(self.group_helper) - self.expand = self.ExpandSwitch(app, "expand") - self.add_child(self.expand) + self.expand = Parameter(app, "expand") + self.add_parameter(self.expand) - self.buttons = WidgetSet(app, "buttons") - self.buttons.html_class = "buttons" - self.add_child(self.buttons) - - task = frame.group_add - button = EditButton(app, "add_group_button", task, negotiator) - self.buttons.add_child(button) - - col = self.ExpandColumn(app, "expand_col") + col = self.ExpandColumn(app, "expand_col", self.expand) col.header_class = ItemTableColumnHeader + col.width = "20px" self.add_column(col) col = self.GroupColumn(app, "group") @@ -130,10 +113,11 @@ self.set_default_column(col) task = frame.edit_dynamic_quota - col = self.DynamicColumn(app, "dynamic", self.group_helper.get_dyn_quota, negotiator, task) + col = self.DynamicColumn(app, "dynamic", None, negotiator, task) col.title = "Dynamic Quota" self.add_column(col) + """ task = frame.edit_static_quota col = QmfGroupColumn(app, "static", self.group_helper.get_static_quota, negotiator, task) col.title = "Static Quota" @@ -152,7 +136,8 @@ col.user = True col.user_task = frame.user_regroup self.add_column(col) - + """ + def render_title(self, session): return "Group Configuration" @@ -163,30 +148,49 @@ return "Loading..." def do_get_items(self, session): - groups = self.group_helper.get_group_names(session) - users = self.group_helper.get_users(session, groups) - self.expand.initialize(session, groups, users) + info = self.group_helper.get_config_info(session) - names = list(groups) + names = list(info) + names = sorted(names) if self.reversed.get(session): names = reversed(names) - # if current expanded group has users expanded = self.expand.get(session) - if expanded in users: - self.group_helper.get_priority_factor(session, users[expanded], True) - self.group_helper.get_regroups(session, users[expanded], True) - # insert the users into the list + # make a list of groups to show based on what is expanded + parent = expanded and self.group_helper.get_parent(session, expanded) or None items = list() for name in names: - items.append(name) - if name == expanded and expanded in users: - items.extend(users[expanded]) + # always show top level groups + if "." not in name: + items.append(name) + # add the current expanded item + elif expanded == name: + items.append(name) + # add the ancestors + elif expanded and expanded.startswith(name+"."): + items.append(name) + # add the direct children of expanded item + elif expanded and name.startswith(expanded+"."): + sub_name = name[len(expanded) + 1:] + if "." not in sub_name: + items.append(name) + # and direct children of expanded item's parent + elif parent and name.startswith(parent+"."): + sub_name = name[len(parent) + 1:] + if "." not in sub_name: + items.append(name) + self.group_helper.get_config_for_groups(session, "GROUP_QUOTA_DYNAMIC", items) + return items class ExpandColumn(ItemTableColumn): + def __init__(self, app, name, expand): + super(NegotiatorOverview.ExpandColumn, self).__init__(app, name) + + self.expand = expand + def render_title(self, session, *args): return "" @@ -194,11 +198,30 @@ return ["expand"] def render_content(self, session, group): - if self.parent.expand.get_title(session, group): - return self.parent.expand.render_item_link(session, group) - else: + if not self.parent.group_helper.has_child(session, group): return "" + info = self.parent.group_helper.get_config_info(session) + # return a " or - depending on current expand + expand = self.expand.get(session) + if expand and expand.startswith(group): + state = "-" + else: + state = "+" + + parent = self.parent.group_helper.get_parent(session, group) + return self.render_item_link(session, group, parent, state) + + def render_item_link(self, session, group, parent, state): + branch = session.branch() + + next_expand = state == "+" and group or parent + self.expand.set(branch, next_expand) + + hover = state == "-" and "Collapse" or "Expand" + + return fmt_link(branch.marshal(), state, link_title=hover) + class GroupColumn(ItemTableColumn): def render_title(self, session, *args): return "Group" @@ -206,40 +229,19 @@ def render_content(self, session, group): # if a user if "." in group: - (g, sep, user) = group.partition(".") - return "<span style='padding-left: 2em;'>%s</span>" % user + (g, sep, user) = group.rpartition(".") + return "<span style='padding-left: 1em;'>%s</span>" % user return group class DynamicColumn(QmfGroupColumn): - def render_data(self, session, data): + def render_content(self, session, group): + value = self.parent.group_helper.get_config_value(session, group, "GROUP_QUOTA_DYNAMIC") + self.task.form.group_leader.set(session, group) href = self.task.get_href(session) - content = "%s%%" % str(round(float(data[1]) * 100.0, 2)) - return fmt_link(href, content, "", "", self.fmt_hover(data[0])) + content = "%s%%" % str(round(float(value) * 100.0, 2)) + return fmt_link(href, content, "", "", self.fmt_hover("")) - class ExpandSwitch(DynamicSwitch): - def render_item_link(self, session, state): - branch = session.branch() - expanded = self.get(session) == state - - if expanded: - self.set(branch, "") - else: - self.set(branch, state) - - content = expanded and "[-]" or "[+]" - verb = expanded and "Hide" or "Show" - hover = self.get_hover(session, state) % verb - - return fmt_link(branch.marshal(), content, "", link_title=hover) - - def initialize(self, session, groups, users): - if len(self.get_items(session)) == 0: - for group in groups: - if group in users: - self.add_state(session, group, "+", hover="%%s users for %s" % group) - self.param.default = None - class GroupHelper(Widget): def __init__(self, app, name, negotiator): super(GroupHelper, self).__init__(app, name) @@ -273,37 +275,106 @@ self.users = self.Users(app, "users") self.add_attribute(self.users) - def get_group_names(self, session): - groups = self.groups.get(session) - if len(groups) == 0: - negotiator = self.negotiator.get(session) + def get_config_info(self, session): + negotiator = self.negotiator.get(session) + try: + info = self.app.model.configs_by_negotiator \ + [negotiator._qmf_agent_id] + except KeyError: + info = dict() + + if len(info) == 0: default = {'Value': None} action = QmfCall(self.app, default=default, timeout=10) results = action.execute(negotiator, "GetRawConfig", "GROUP_NAMES") - # TODO: remove this temp workaround - # XXX temp work around for qmf call - if results.error: - results.data = {'Value': 'msg, grid, mgmt, rt'} groups = results.data try: groups = self.split_group_names(groups['Value']) except Exception, e: groups = [] - if len(groups): - self.groups.set(session, groups) - return groups + info = dict() + for group in groups: + info[group] = dict() + + self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] = info + + return info + + def has_child(self, session, group): + negotiator = self.negotiator.get(session) + info = self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] + + try: + return info[group]['has_child'] + except KeyError: + info[group]['has_child'] = False + for key in info: + if key.startswith(group+"."): + info[group]['has_child'] = True + break + + self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] = info + return info[group]['has_child'] + + def get_parent(self, session, group): + negotiator = self.negotiator.get(session) + info = self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] + + try: + return info[group]['parent'] + except KeyError: + parent = "" + for key in info: + if key != group and group.startswith(key): + if len(key) > len(parent): + parent = key + + info[group]['parent'] = parent + + self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] = info + return info[group]['parent'] + + def get_siblings(self, session, node): + negotiator = self.negotiator.get(session) + info = self.app.model.configs_by_negotiator[negotiator._qmf_agent_id] + + siblings = list() + (ng, s, nn) = node.rpartition(".") + for group in info: + (g, s, n) = group.rpartition(".") + if g == ng: + siblings.append(group) + + return siblings + + def get_group_names(self, session): + info = self.get_config_info(session) + return list(info) + def split_group_names(self, group_string): groups = [] - gsplit = group_string.split() - if len(gsplit) < 2: - gsplit = group_string.split(",") - for group in gsplit: - group = group.replace(",", "") - group = group.replace(" ", "") - groups.append(group) - return groups + g_string = group_string.replace(", ", ",") + return g_string.split(",") + def get_config_for_groups(self, session, config, groups): + info = self.get_config_info(session) + + needed_groups = [x for x in groups if not config in info[x]] + + if len(needed_groups) > 0: + negotiator = self.negotiator.get(session) + action = FetchRawConfigSet(self.app) + raw_configs = action.execute(negotiator, needed_groups, config+"_") + + for group in raw_configs: + res = raw_configs[group] + info[group][config] = res.data['Value'] + + def get_config_value(self, session, group, config): + info = self.get_config_info(session) + return info[group][config] + def get_group_raw_config(self, session, config, param, groups=None): configs = param.get(session) @@ -350,55 +421,29 @@ "GROUP_AUTOREGROUP", self.autoregroup, [""]) - def get_users(self, session, groups): - try: - users = self.users.get_users(session, groups, self.negotiator) - except: - users = dict() - return users - - def append_unclaimed_dyn_quota(self, session, quotas, force=False): + def get_unclaimed_dyn_quota(self, session, groups): + info = self.get_config_info(session) total = 0.0 - for [group, value, status] in quotas: + for group in groups: + value = info[group]["GROUP_QUOTA_DYNAMIC"] try: total = total + float(value) except: pass - if (total < 1.0) or force: - val = 1.0 - total - appended = list(quotas) - appended.append(['Unclaimed', str(val), ("OK", False, True)]) - return appended - else: - return quotas + val = 1.0 - total + val = max(0, val) + val = min(1.0, val) + return val + class GroupAttribute(Attribute): def get_default(self, session): return list() class Users(Attribute): - def get_users(self, session, groups, negotiator): - #returns {group: [user, user], group: [user, user], ...} - users = self.get(session) - if not users: - neg = negotiator.get(session) - pool = neg.Pool - cls = self.app.model.com_redhat_grid.Scheduler - scheduler = cls.get_object(session.cursor, Pool=pool) - cls = self.app.model.com_redhat_grid.Submitter - submitters = cls.get_selection(session.cursor, _schedulerRef_id=scheduler._id) - user_names = [x.Name for x in submitters] - users = dict() - for group in groups: - for user_name in user_names: - if user_name.startswith("%s." % group): - if not group in users: - users[group] = list() - users[group].append(user_name) + def get_default(self, session): + return dict() - self.set(session, users) - return users - class GroupAddForm(ObjectTaskForm): def __init__(self, app, name, task): super(GroupAddForm, self).__init__(app, name, task) @@ -420,11 +465,15 @@ original_groups = self.group_helper.get_group_names(session) if self.is_valid(group_name): - original_groups.append(group_name) - new_groups = ", ".join(original_groups) - self.task.invoke(session, negotiator, "GROUP_NAMES", new_groups) - self.task.reconfig(negotiator) + if group_name not in original_groups: + original_groups.append(group_name) + new_groups = ", ".join(original_groups) + self.task.invoke(session, negotiator, "GROUP_NAMES", new_groups) + self.task.reconfig(negotiator) + info = self.group_helper.get_config_info(session) + info[group_name] = dict() + self.task.exit_with_redirect(session) def is_valid(self, group): @@ -458,7 +507,7 @@ self.add_parameter(self.original_values) def render_group_name(self, session, group): - return group[0] + return group def render_group_name_path(self, session, group): return self.group_names.path @@ -583,8 +632,11 @@ self.quotas = ListParameter(app, "quotas", quota) self.add_parameter(self.quotas) + self.group_leader = Parameter(app, "group_leader") + self.add_parameter(self.group_leader) + self.chart = PriorityPieChart \ - (app, "chart", self.object, self.group_helper) + (app, "chart", self.object, self.group_helper, self.group_leader) self.add_child(self.chart) def render_title(self, session): @@ -597,28 +649,34 @@ return "priorityForm" def render_data_col_header(self, session): - return "Percent of Slots" + return "Percent" def render_groups(self, session): writer = Writer() - groups = self.group_helper.get_dyn_quota(session) - groups = self.group_helper.append_unclaimed_dyn_quota \ - (session, groups, force=True) - for group in groups: - if group[0] == "Unclaimed": - self.unclaimed_tmpl.render(writer, session, group) - else: - self.field_tmpl.render(writer, session, group) + group_leader = self.group_leader.get(session) + groups = self.group_helper.get_siblings(session, group_leader) + for group in sorted(groups): + self.field_tmpl.render(writer, session, group) + self.unclaimed_tmpl.render(writer, session, "Unclaimed") + return writer.to_string() def render_quota_name(self, session, group): return self.quotas.path def render_quota_value(self, session, group): - return round(float(group[1]) * 100.0, 2) + value = self.group_helper.get_config_value(session, group, "GROUP_QUOTA_DYNAMIC") + return round(float(value) * 100.0, 2) + def render_unclaimed_value(self, session, group): + group_leader = self.group_leader.get(session) + groups = self.group_helper.get_siblings(session, group_leader) + unclaimed = self.group_helper.get_unclaimed_dyn_quota \ + (session, groups) + return round(float(unclaimed) * 100.0, 2) + def render_chart_id(self, session): return self.chart.render_id(session) @@ -627,6 +685,7 @@ quotas = self.quotas.get(session) group_names = self.group_names.get(session) original_values = self.original_values.get(session) + info = self.group_helper.get_config_info(session) changed = False for group, new_value, original_value in zip(group_names, quotas, original_values): @@ -636,7 +695,9 @@ if quota: self.task.invoke(session, negotiator, "GROUP_QUOTA_DYNAMIC_" + group, quota) + info[group]["GROUP_QUOTA_DYNAMIC"] = quota changed = True + if changed: self.task.reconfig(negotiator) self.task.exit_with_redirect(session) @@ -829,7 +890,7 @@ self.task.exit_with_redirect(session) class PriorityPieChart(StatFlashChart): - def __init__(self, app, name, negotiator, groups): + def __init__(self, app, name, negotiator, group_helper, group_leader): super(PriorityPieChart, self).__init__(app, name, negotiator) self.negotiator = negotiator @@ -838,7 +899,8 @@ self.fullpageable = False self.update_enabled = False - self.groups = groups + self.group_helper = group_helper + self.group_leader = group_leader def render_title(self, session): pass @@ -859,12 +921,18 @@ # send the group names and values to the chart page so we # don't have to get them again - values = self.groups.get_dyn_quota(session) - values = self.groups.append_unclaimed_dyn_quota(session, values) - names = ["name=%s" % x[0] for x in values] + group_leader = self.group_leader.get(session) + groups = self.group_helper.get_siblings(session, group_leader) + names = ["name=%s" % x for x in groups] params.extend(names) - vals = ["value=%s" % str(x[1]) for x in values] + params.append("name=Unclaimed") + + info = self.group_helper.get_config_info(session) + vals = ["value=%s" % info[x]["GROUP_QUOTA_DYNAMIC"] for x in groups] params.extend(vals) + unclaimed = self.group_helper.get_unclaimed_dyn_quota(session, groups) + params.append("value=%s" % str(unclaimed)) + return params class EditButton(ActionSet): @@ -927,6 +995,9 @@ self.form = GroupAddForm(app, self.name, self) def get_title(self, session): + return "" + + def get_description(self, session): return "Add group" class NegotiatorEditRegroup(NegotiatorGroupTask): @@ -954,8 +1025,15 @@ self.form = EditDynamicQuotaForm(app, self.name, self) def get_title(self, session): + return "" + + def get_description(self, session): return "Edit dynamic quota" + def do_enter(self, session, osession): + group_leader = self.form.group_leader.get(osession) + self.form.group_leader.set(session, group_leader) + class NegotiatorEditStaticQuota(NegotiatorGroupTask): def __init__(self, app, frame): super(NegotiatorEditStaticQuota, self).__init__(app, frame) Modified: mgmt/newdata/cumin/python/cumin/grid/negotiator.strings =================================================================== --- mgmt/newdata/cumin/python/cumin/grid/negotiator.strings 2010-07-09 15:39:44 UTC (rev 4095) +++ mgmt/newdata/cumin/python/cumin/grid/negotiator.strings 2010-07-09 20:15:59 UTC (rev 4096) @@ -6,6 +6,14 @@ padding: 0.75em; } +[NegotiatorOverview.css] +th.ExpandColumn { + width: 2em; +} +th.GroupColumn { + width: 50%; +} + [NegotiatorOverview.html] <div id="{id}" class="CuminTable GroupTable"> <table {class}> @@ -181,11 +189,11 @@ {group_name} </td> <td> - <input id="{group_name}" class="disabled" disabled="disabled" type="text" name="{quota_name}" value="{quota_value}" + <input id="{group_name}" class="disabled" disabled="disabled" type="text" name="{quota_name}" value="{unclaimed_value}" size="6" tabindex="100"/> <!-- these are here so we don't need to reget the groups and values after a submit --> <input type="hidden" name="{group_name_path}" value="{group_name}" /> - <input type="hidden" name="{original_value_path}" value="{quota_value}" /> + <input type="hidden" name="{original_value_path}" value="{unclaimed_value}" /> </td> </tr> @@ -219,7 +227,7 @@ //alert('failed'); }, evaluateOnSubmit: false, - evaluateFieldsOnBlur: false + evaluateFieldsOnBlur: true }); myFormValidator.add('IsPercent', { errorMsg: 'Values must be between 0 and 100.', Modified: mgmt/newdata/cumin/python/cumin/model.py =================================================================== --- mgmt/newdata/cumin/python/cumin/model.py 2010-07-09 15:39:44 UTC (rev 4095) +++ mgmt/newdata/cumin/python/cumin/model.py 2010-07-09 20:15:59 UTC (rev 4096) @@ -30,6 +30,7 @@ self.limits_by_negotiator = dict() self.job_summaries_by_submission = dict() + self.configs_by_negotiator = dict() self.lock = Lock()