Mercurial > public > src > rhodecode

comparison pylons_app/lib/celerylib/tasks.py @ 551:d5efb83590ef

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions! added tredning languages stats
author Marcin Kuzminski <marcin@python-works.com>
date Mon, 27 Sep 2010 02:17:03 +0200
parents d280aa1c85c6
children b50e79b4257a
comparison
equal deleted inserted replaced
550:5aba7adff196 551:d5efb83590ef
1 from celery.decorators import task 1 from celery.decorators import task
2 from celery.task.sets import subtask 2 from celery.task.sets import subtask
3 from celeryconfig import PYLONS_CONFIG as config 3 from celeryconfig import PYLONS_CONFIG as config
4 from operator import itemgetter
4 from pylons.i18n.translation import _ 5 from pylons.i18n.translation import _
5 from pylons_app.lib.celerylib import run_task, locked_task 6 from pylons_app.lib.celerylib import run_task, locked_task
6 from pylons_app.lib.helpers import person 7 from pylons_app.lib.helpers import person
7 from pylons_app.lib.smtp_mailer import SmtpMailer 8 from pylons_app.lib.smtp_mailer import SmtpMailer
8 from pylons_app.lib.utils import OrderedDict 9 from pylons_app.lib.utils import OrderedDict
9 from operator import itemgetter 10 from time import mktime
10 from vcs.backends.hg import MercurialRepository 11 from vcs.backends.hg import MercurialRepository
11 from time import mktime 12 import json
12 import traceback 13 import traceback
13 import json
14 14
15 __all__ = ['whoosh_index', 'get_commits_stats', 15 __all__ = ['whoosh_index', 'get_commits_stats',
16 'reset_user_password', 'send_email'] 16 'reset_user_password', 'send_email']
17 17
18 def get_session(): 18 def get_session():
73 WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index) 73 WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index)
74 74
75 @task 75 @task
76 @locked_task 76 @locked_task
77 def get_commits_stats(repo_name, ts_min_y, ts_max_y): 77 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
78 author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
79
80 from pylons_app.model.db import Statistics, Repository 78 from pylons_app.model.db import Statistics, Repository
81 log = get_commits_stats.get_logger() 79 log = get_commits_stats.get_logger()
80 author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
81
82 commits_by_day_author_aggregate = {} 82 commits_by_day_author_aggregate = {}
83 commits_by_day_aggregate = {} 83 commits_by_day_aggregate = {}
84 repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') 84 repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
85 repo = MercurialRepository(repos_path + repo_name) 85 repo = MercurialRepository(repos_path + repo_name)
86 86
97 cur_stats = sa.query(Statistics)\ 97 cur_stats = sa.query(Statistics)\
98 .filter(Statistics.repository == dbrepo).scalar() 98 .filter(Statistics.repository == dbrepo).scalar()
99 if cur_stats: 99 if cur_stats:
100 last_rev = cur_stats.stat_on_revision 100 last_rev = cur_stats.stat_on_revision
101 101
102 if last_rev == repo.revisions[-1]: 102 if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
103 #pass silently without any work 103 #pass silently without any work
104 return True 104 return True
105 105
106 if cur_stats: 106 if cur_stats:
107 commits_by_day_aggregate = OrderedDict( 107 commits_by_day_aggregate = OrderedDict(
108 json.loads( 108 json.loads(
109 cur_stats.commit_activity_combined)) 109 cur_stats.commit_activity_combined))
110 commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) 110 commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
111 111
112 log.debug('starting parsing %s', parse_limit)
112 for cnt, rev in enumerate(repo.revisions[last_rev:]): 113 for cnt, rev in enumerate(repo.revisions[last_rev:]):
113 last_cs = cs = repo.get_changeset(rev) 114 last_cs = cs = repo.get_changeset(rev)
114 k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1], 115 k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
115 cs.date.timetuple()[2]) 116 cs.date.timetuple()[2])
116 timetupple = [int(x) for x in k.split('-')] 117 timetupple = [int(x) for x in k.split('-')]
185 } 186 }
186 187
187 stats = cur_stats if cur_stats else Statistics() 188 stats = cur_stats if cur_stats else Statistics()
188 stats.commit_activity = json.dumps(commits_by_day_author_aggregate) 189 stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
189 stats.commit_activity_combined = json.dumps(overview_data) 190 stats.commit_activity_combined = json.dumps(overview_data)
191
192 log.debug('last revison %s', last_rev)
193 leftovers = len(repo.revisions[last_rev:])
194 log.debug('revisions to parse %s', leftovers)
195
196 if last_rev == 0 or leftovers < parse_limit:
197 stats.languages = json.dumps(__get_codes_stats(repo_name))
198
190 stats.repository = dbrepo 199 stats.repository = dbrepo
191 stats.stat_on_revision = last_cs.revision 200 stats.stat_on_revision = last_cs.revision
192 stats.languages = json.dumps({'_TOTAL_':0, '':0})
193 201
194 try: 202 try:
195 sa.add(stats) 203 sa.add(stats)
196 sa.commit() 204 sa.commit()
197 except: 205 except:
198 log.error(traceback.format_exc()) 206 log.error(traceback.format_exc())
199 sa.rollback() 207 sa.rollback()
200 return False 208 return False
201 209 if len(repo.revisions) > 1:
202 run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) 210 run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
203 211
204 return True 212 return True
205 213
206 @task 214 @task
207 def reset_user_password(user_email): 215 def reset_user_password(user_email):
257 except: 265 except:
258 log.error('Mail sending failed') 266 log.error('Mail sending failed')
259 log.error(traceback.format_exc()) 267 log.error(traceback.format_exc())
260 return False 268 return False
261 return True 269 return True
270
271 def __get_codes_stats(repo_name):
272 LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
273 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl',
274 'h', 'java', 'js', 'jsp', 'jspx', 'lisp',
275 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
276 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh',
277 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
278 'yaws']
279 repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
280 repo = MercurialRepository(repos_path + repo_name)
281
282 code_stats = {}
283 for topnode, dirs, files in repo.walk('/', 'tip'):
284 for f in files:
285 k = f.mimetype
286 if f.extension in LANGUAGES_EXTENSIONS:
287 if code_stats.has_key(k):
288 code_stats[k] += 1
289 else:
290 code_stats[k] = 1
291
292 return code_stats or {}
293
294
295
296
297