Mercurial > public > src > rhodecode

annotate pylons_app/lib/celerylib/tasks.py @ 510:3fc3ce53659b celery

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
starting celery branch
author Marcin Kuzminski <marcin@python-works.com>
date Sat, 11 Sep 2010 01:55:46 +0200
parents
children a3d9d24acbec
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
1 from celery.decorators import task
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
2 from datetime import datetime, timedelta
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
3 from pylons_app.lib.helpers import person
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
4 from pylons_app.lib.utils import OrderedDict
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
5 from time import mktime
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
6 import calendar
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
7 import logging
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
8 from vcs.backends.hg import MercurialRepository
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
9
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
10 log = logging.getLogger(__name__)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
11
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
12 @task()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
13 def whoosh_index(repo_location,full_index):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
14 from pylons_app.lib.indexers import DaemonLock
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
15 from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon,LockHeld
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
16 try:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
17 l = DaemonLock()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
18 WhooshIndexingDaemon(repo_location=repo_location)\
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
19 .run(full_index=full_index)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
20 l.release()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
21 return 'Done'
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
22 except LockHeld:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
23 log.info('LockHeld')
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
24 return 'LockHeld'
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
25
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
26 @task()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
27 def get_commits_stats(repo):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
28 aggregate = OrderedDict()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
29 repo = MercurialRepository('/home/marcink/hg_repos/'+repo)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
30 #graph range
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
31 td = datetime.today() + timedelta(days=1)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
32 y, m, d = td.year, td.month, td.day
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
33 ts_min = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
34 d, 0, 0, 0, 0, 0, 0,))
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
35 ts_max = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
36
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
37 def author_key_cleaner(k):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
38 k = person(k)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
39 k = k.replace('"', "'") #for js data compatibilty
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
40 return k
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
41
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
42 for cs in repo[:200]:#added limit 200 until fix #29 is made
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
43 k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
44 cs.date.timetuple()[2])
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
45 timetupple = [int(x) for x in k.split('-')]
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
46 timetupple.extend([0 for _ in xrange(6)])
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
47 k = mktime(timetupple)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
48 if aggregate.has_key(author_key_cleaner(cs.author)):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
49 if aggregate[author_key_cleaner(cs.author)].has_key(k):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
50 aggregate[author_key_cleaner(cs.author)][k]["commits"] += 1
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
51 aggregate[author_key_cleaner(cs.author)][k]["added"] += len(cs.added)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
52 aggregate[author_key_cleaner(cs.author)][k]["changed"] += len(cs.changed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
53 aggregate[author_key_cleaner(cs.author)][k]["removed"] += len(cs.removed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
54
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
55 else:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
56 #aggregate[author_key_cleaner(cs.author)].update(dates_range)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
57 if k >= ts_min and k <= ts_max:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
58 aggregate[author_key_cleaner(cs.author)][k] = {}
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
59 aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
60 aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
61 aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
62 aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
63
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
64 else:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
65 if k >= ts_min and k <= ts_max:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
66 aggregate[author_key_cleaner(cs.author)] = OrderedDict()
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
67 #aggregate[author_key_cleaner(cs.author)].update(dates_range)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
68 aggregate[author_key_cleaner(cs.author)][k] = {}
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
69 aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
70 aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
71 aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
72 aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
73
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
74 d = ''
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
75 tmpl0 = u""""%s":%s"""
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
76 tmpl1 = u"""{label:"%s",data:%s,schema:["commits"]},"""
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
77 for author in aggregate:
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
78
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
79 d += tmpl0 % (author,
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
80 tmpl1 \
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
81 % (author,
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
82 [{"time":x,
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
83 "commits":aggregate[author][x]['commits'],
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
84 "added":aggregate[author][x]['added'],
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
85 "changed":aggregate[author][x]['changed'],
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
86 "removed":aggregate[author][x]['removed'],
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
87 } for x in aggregate[author]]))
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
88 if d == '':
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
89 d = '"%s":{label:"%s",data:[[0,1],]}' \
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
90 % (author_key_cleaner(repo.contact),
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
91 author_key_cleaner(repo.contact))
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
92 return (ts_min, ts_max, d)