Mercurial > public > src > rhodecode

annotate pylons_app/lib/celerylib/__init__.py @ 547:ac32a026c306

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
simplified task locking, and fixed some bugs for keyworded arguments
author Marcin Kuzminski <marcin@python-works.com>
date Thu, 23 Sep 2010 21:25:30 +0200
parents fb0c3af6031b
children d5efb83590ef
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
1 from pylons_app.lib.pidlock import DaemonLock, LockHeld
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
2 from vcs.utils.lazy import LazyProperty
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
3 from decorator import decorator
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
4 import logging
518
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 510
diff changeset
5 import os
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 510
diff changeset
6 import sys
a3d9d24acbec Implemented password reset(forms/models/ tasks) and mailing tasks.
Marcin Kuzminski <marcin@python-works.com>
parents: 510
diff changeset
7 import traceback
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
8 from hashlib import md5
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
9 log = logging.getLogger(__name__)
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
10
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
11 class ResultWrapper(object):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
12 def __init__(self, task):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
13 self.task = task
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
14
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
15 @LazyProperty
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
16 def result(self):
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
17 return self.task
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
18
531
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
19 def run_task(task, *args, **kwargs):
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
20 try:
531
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
21 t = task.delay(*args, **kwargs)
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
22 log.info('running task %s', t.task_id)
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
23 return t
531
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
24 except Exception, e:
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
25 print e
531
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
26 if e.errno == 111:
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
27 log.debug('Unnable to connect. Sync execution')
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
28 else:
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
29 log.error(traceback.format_exc())
510
3fc3ce53659b starting celery branch
Marcin Kuzminski <marcin@python-works.com>
parents:
diff changeset
30 #pure sync version
531
b12ea84fb906 Some fixes to summary, and total rewrite of summary graphs implemented more interactive graph.
Marcin Kuzminski <marcin@python-works.com>
parents: 518
diff changeset
31 return ResultWrapper(task(*args, **kwargs))
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
32
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
33
547
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
34 def locked_task(func):
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
35 def __wrapper(func, *fargs, **fkwargs):
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
36 params = list(fargs)
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
37 params.extend(['%s-%s' % ar for ar in fkwargs.items()])
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
38
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
39 lockkey = 'task_%s' % \
547
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
40 md5(str(func.__name__) + '-' + \
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
41 '-'.join(map(str, params))).hexdigest()
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
42 log.info('running task with lockkey %s', lockkey)
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
43 try:
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
44 l = DaemonLock(lockkey)
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
45 return func(*fargs, **fkwargs)
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
46 l.release()
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
47 except LockHeld:
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
48 log.info('LockHeld')
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
49 return 'Task with key %s already running' % lockkey
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
50
547
ac32a026c306 simplified task locking, and fixed some bugs for keyworded arguments
Marcin Kuzminski <marcin@python-works.com>
parents: 541
diff changeset
51 return decorator(__wrapper, func)
541
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
52
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
53
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
54
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
55
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
56
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
57
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
58
fb0c3af6031b Implemented locking for task, to prevent for running the same tasks,
Marcin Kuzminski <marcin@python-works.com>
parents: 531
diff changeset
59