Mercurial > public > src > rhodecode
changeset 2454:90e06f53af8c beta
Implemented cache-map on main page to save executing select
statements that checks if cache should be invalidated.
It reduces number of executed queries from N which is number of
repos to 1 which is needed to fetch all keys from database.
On pages with large number of repos this could reduce load time by half
| author | Marcin Kuzminski <marcin@python-works.com> |
|---|---|
| date | Mon, 28 May 2012 16:26:47 +0200 |
| parents | f55b8142bf6c |
| children | ef71d7e63806 |
| files | rhodecode/model/db.py rhodecode/model/scm.py |
| diffstat | 2 files changed, 56 insertions(+), 7 deletions(-) [+] |
line wrap: on
line diff
--- a/rhodecode/model/db.py Mon May 28 14:42:07 2012 +0200 +++ b/rhodecode/model/db.py Mon May 28 16:26:47 2012 +0200 @@ -693,18 +693,27 @@ def scm_instance(self): return self.__get_instance() - @property - def scm_instance_cached(self): + def scm_instance_cached(self, cache_map=None): @cache_region('long_term') def _c(repo_name): return self.__get_instance() rn = self.repo_name log.debug('Getting cached instance of repo') - inv = self.invalidate - if inv is not None: + + if cache_map: + # get using prefilled cache_map + invalidate_repo = cache_map[self.repo_name] + if invalidate_repo: + invalidate_repo = (None if invalidate_repo.cache_active + else invalidate_repo) + else: + # get from invalidate + invalidate_repo = self.invalidate + + if invalidate_repo is not None: region_invalidate(_c, None, rn) # update our cache - CacheInvalidation.set_valid(inv.cache_key) + CacheInvalidation.set_valid(invalidate_repo.cache_key) return _c(rn) def __get_instance(self): @@ -1072,6 +1081,7 @@ __tablename__ = 'cache_invalidation' __table_args__ = ( UniqueConstraint('cache_key'), + Index('key_idx', 'cache_key'), {'extend_existing': True, 'mysql_engine':'InnoDB', 'mysql_charset': 'utf8'}, ) @@ -1088,6 +1098,7 @@ def __unicode__(self): return u"<%s('%s:%s')>" % (self.__class__.__name__, self.cache_id, self.cache_key) + @classmethod def clear_cache(cls): cls.query().delete() @@ -1117,7 +1128,7 @@ try: inv_obj = CacheInvalidation(key, org_key) Session.add(inv_obj) - Session.commit() + #Session.commit() except Exception: log.error(traceback.format_exc()) Session.rollback() @@ -1174,6 +1185,40 @@ Session.add(inv_obj) Session.commit() + @classmethod + def get_cache_map(cls): + + class cachemapdict(dict): + + def __init__(self, *args, **kwargs): + fixkey = kwargs.get('fixkey') + if fixkey: + del kwargs['fixkey'] + self.fixkey = fixkey + super(cachemapdict, self).__init__(*args, **kwargs) + + def __getattr__(self, name): + key = name + if self.fixkey: + key, _prefix, _org_key = cls._get_key(key) + if key in self.__dict__: + return self.__dict__[key] + else: + return self[key] + + def __getitem__(self, key): + if self.fixkey: + key, _prefix, _org_key = cls._get_key(key) + try: + return super(cachemapdict, self).__getitem__(key) + except KeyError: + return + + cache_map = cachemapdict(fixkey=True) + for obj in cls.query().all(): + cache_map[obj.cache_key] = cachemapdict(obj.get_dict()) + return cache_map + class ChangesetComment(Base, BaseModel): __tablename__ = 'changeset_comments'
--- a/rhodecode/model/scm.py Mon May 28 14:42:07 2012 +0200 +++ b/rhodecode/model/scm.py Mon May 28 16:26:47 2012 +0200 @@ -77,8 +77,12 @@ return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) def __iter__(self): + # pre-propagated cache_map to save executing select statements + # for each repo + cache_map = CacheInvalidation.get_cache_map() + for dbr in self.db_repo_list: - scmr = dbr.scm_instance_cached + scmr = dbr.scm_instance_cached(cache_map) # check permission at this level if not HasRepoPermissionAny( 'repository.read', 'repository.write', 'repository.admin'