equal
deleted
inserted
replaced
114 _STARTUP_COST = 1e30 |
114 _STARTUP_COST = 1e30 |
115 _DISALLOW_THREAD_UNSAFE = False |
115 _DISALLOW_THREAD_UNSAFE = False |
116 |
116 |
117 |
117 |
118 def worthwhile(ui, costperop, nops, threadsafe=True): |
118 def worthwhile(ui, costperop, nops, threadsafe=True): |
119 '''try to determine whether the benefit of multiple processes can |
119 """try to determine whether the benefit of multiple processes can |
120 outweigh the cost of starting them''' |
120 outweigh the cost of starting them""" |
121 |
121 |
122 if not threadsafe and _DISALLOW_THREAD_UNSAFE: |
122 if not threadsafe and _DISALLOW_THREAD_UNSAFE: |
123 return False |
123 return False |
124 |
124 |
125 linear = costperop * nops |
125 linear = costperop * nops |
129 |
129 |
130 |
130 |
131 def worker( |
131 def worker( |
132 ui, costperarg, func, staticargs, args, hasretval=False, threadsafe=True |
132 ui, costperarg, func, staticargs, args, hasretval=False, threadsafe=True |
133 ): |
133 ): |
134 '''run a function, possibly in parallel in multiple worker |
134 """run a function, possibly in parallel in multiple worker |
135 processes. |
135 processes. |
136 |
136 |
137 returns a progress iterator |
137 returns a progress iterator |
138 |
138 |
139 costperarg - cost of a single task |
139 costperarg - cost of a single task |
151 overlapping keys are a bad idea. |
151 overlapping keys are a bad idea. |
152 |
152 |
153 threadsafe - whether work items are thread safe and can be executed using |
153 threadsafe - whether work items are thread safe and can be executed using |
154 a thread-based worker. Should be disabled for CPU heavy tasks that don't |
154 a thread-based worker. Should be disabled for CPU heavy tasks that don't |
155 release the GIL. |
155 release the GIL. |
156 ''' |
156 """ |
157 enabled = ui.configbool(b'worker', b'enabled') |
157 enabled = ui.configbool(b'worker', b'enabled') |
158 if enabled and worthwhile(ui, costperarg, len(args), threadsafe=threadsafe): |
158 if enabled and worthwhile(ui, costperarg, len(args), threadsafe=threadsafe): |
159 return _platformworker(ui, func, staticargs, args, hasretval) |
159 return _platformworker(ui, func, staticargs, args, hasretval) |
160 return func(*staticargs + (args,)) |
160 return func(*staticargs + (args,)) |
161 |
161 |
304 if hasretval: |
304 if hasretval: |
305 yield True, retval |
305 yield True, retval |
306 |
306 |
307 |
307 |
308 def _posixexitstatus(code): |
308 def _posixexitstatus(code): |
309 '''convert a posix exit status into the same form returned by |
309 """convert a posix exit status into the same form returned by |
310 os.spawnv |
310 os.spawnv |
311 |
311 |
312 returns None if the process was stopped instead of exiting''' |
312 returns None if the process was stopped instead of exiting""" |
313 if os.WIFEXITED(code): |
313 if os.WIFEXITED(code): |
314 return os.WEXITSTATUS(code) |
314 return os.WEXITSTATUS(code) |
315 elif os.WIFSIGNALED(code): |
315 elif os.WIFSIGNALED(code): |
316 return -(os.WTERMSIG(code)) |
316 return -(os.WTERMSIG(code)) |
317 |
317 |
421 _platformworker = _posixworker |
421 _platformworker = _posixworker |
422 _exitstatus = _posixexitstatus |
422 _exitstatus = _posixexitstatus |
423 |
423 |
424 |
424 |
425 def partition(lst, nslices): |
425 def partition(lst, nslices): |
426 '''partition a list into N slices of roughly equal size |
426 """partition a list into N slices of roughly equal size |
427 |
427 |
428 The current strategy takes every Nth element from the input. If |
428 The current strategy takes every Nth element from the input. If |
429 we ever write workers that need to preserve grouping in input |
429 we ever write workers that need to preserve grouping in input |
430 we should consider allowing callers to specify a partition strategy. |
430 we should consider allowing callers to specify a partition strategy. |
431 |
431 |
448 the workload switched from streaming to random I/O. |
448 the workload switched from streaming to random I/O. |
449 |
449 |
450 What we should really be doing is have workers read filenames from a |
450 What we should really be doing is have workers read filenames from a |
451 ordered queue. This preserves locality and also keeps any worker from |
451 ordered queue. This preserves locality and also keeps any worker from |
452 getting more than one file out of balance. |
452 getting more than one file out of balance. |
453 ''' |
453 """ |
454 for i in range(nslices): |
454 for i in range(nslices): |
455 yield lst[i::nslices] |
455 yield lst[i::nslices] |