Author: nunofsantos Date: 2008-11-20 18:07:47 -0500 (Thu, 20 Nov 2008) New Revision: 2857 Added: mgmt/trunk/mint/python/mint/cache.py Modified: mgmt/trunk/mint/python/mint/__init__.py mgmt/trunk/mint/python/mint/update.py Log: use commit/rollback-able cache to store mapping between qmf ids and database ids Modified: mgmt/trunk/mint/python/mint/__init__.py =================================================================== --- mgmt/trunk/mint/python/mint/__init__.py 2008-11-20 22:21:56 UTC (rev 2856) +++ mgmt/trunk/mint/python/mint/__init__.py 2008-11-20 23:07:47 UTC (rev 2857) @@ -11,6 +11,7 @@ from qpid.datatypes import UUID from mint.schema import * from mint import update +from mint.cache import MintCache log = logging.getLogger("mint") @@ -315,7 +316,9 @@ self.outstandingMethodCalls = dict() self.managedBrokers = dict() - self.qmfIdToDbIdMap = dict() + # cache contains mapping between qmf ids and database ids + # (idFirst, idSecond) -> dbId + self.cache = MintCache() if self.debug: log.setLevel(logging.DEBUG) Added: mgmt/trunk/mint/python/mint/cache.py =================================================================== --- mgmt/trunk/mint/python/mint/cache.py (rev 0) +++ mgmt/trunk/mint/python/mint/cache.py 2008-11-20 23:07:47 UTC (rev 2857) @@ -0,0 +1,47 @@ +from threading import RLock + +class MintCache: + staticInstance = None + + def __init__(self): + assert MintCache.staticInstance is None + MintCache.staticInstance = self + + self.__lock = RLock() + self.__cache = dict() + self.__pending = dict() + self.__dirty = False + + def get(self, key, usePending=True): + result = None + self.__lock.acquire() + if key in self.__cache: + result = self.__cache[key] + elif usePending and key in self.__pending: + result = self.__pending[key] + self.__lock.release() + return result + + def set(self, key, value): + self.__lock.acquire() + self.__pending[key] = value + self.__dirty = True + self.__lock.release() + + def commit(self): + self.__lock.acquire() + self.__cache.update(self.__pending) + self.__pending.clear() + self.__dirty = False + self.__lock.release() + + def rollback(self): + self.__lock.acquire() + self.__pending.clear() + self.__dirty = False + self.__lock.release() + + def isDirty(self): + self.__lock.acquire() + return self.__dirty + self.__lock.release() Property changes on: mgmt/trunk/mint/python/mint/cache.py ___________________________________________________________________ Name: svn:eol-style + native Modified: mgmt/trunk/mint/python/mint/update.py =================================================================== --- mgmt/trunk/mint/python/mint/update.py 2008-11-20 22:21:56 UTC (rev 2856) +++ mgmt/trunk/mint/python/mint/update.py 2008-11-20 23:07:47 UTC (rev 2857) @@ -56,9 +56,12 @@ try: update.process(self.model, conn) - conn.commit() + if self.dequeueCount % 100 == 0: + conn.commit() + self.model.cache.commit() except: conn.rollback() + self.model.cache.rollback() log.exception("Update failed") pass @@ -95,8 +98,9 @@ foreignKey = name + "_id" valueFirst = value.first valueSecond = value.second - if (valueFirst, valueSecond) in model.qmfIdToDbIdMap: - results[foreignKey] = model.qmfIdToDbIdMap[(valueFirst, valueSecond)] + cachedId = model.cache.get((valueFirst, valueSecond)) + if cachedId != None: + results[foreignKey] = cachedId else: model.addOrphanObject(className, valueFirst, valueSecond, self) orphan = True @@ -173,7 +177,8 @@ cursor.execute(sql) log.debug("%s(%s) created", cls, id) - if (idFirst, idSecond) not in model.qmfIdToDbIdMap: + dbId = model.cache.get((idFirst, idSecond)) + if dbId is None: if isUpdate: sql = model.generateSQLSelect(sqlCls, id) else: @@ -182,9 +187,7 @@ cursor.execute(sql) rec = cursor.fetchone() dbId = rec[0] - model.qmfIdToDbIdMap[(idFirst, idSecond)] = dbId - else: - dbId = model.qmfIdToDbIdMap[(idFirst, idSecond)] + model.cache.set((idFirst, idSecond), dbId) if cls == "Broker" and str(self.broker.getBrokerId()) in model.managedBrokers: broker, dbObjId = model.managedBrokers[str(self.broker.getBrokerId())] @@ -241,8 +244,9 @@ return attrs["recTime"] = time_unwarp(datetime.fromtimestamp(timestamps[0]/1000000000)) - if (idFirst, idSecond) in model.qmfIdToDbIdMap: - attrs[sqlCls + "_id"] = model.qmfIdToDbIdMap[(idFirst, idSecond)] + cachedId = model.cache.get((idFirst, idSecond)) + if cachedId != None: + attrs[sqlCls + "_id"] = cachedId else: model.addOrphanObject(cls, idFirst, idSecond, self)