Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/reposetup.py @ 15250:f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
author | Na'Tosha Bard <natosha@unity3d.com> |
---|---|
date | Thu, 13 Oct 2011 12:11:25 +0200 |
parents | 7c604d8c7e83 |
children | 6e809bb4f969 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
9 '''setup for largefiles repositories: reposetup''' | |
10 import copy | |
11 import types | |
12 import os | |
13 import re | |
14 | |
15 from mercurial import context, error, manifest, match as match_, \ | |
16 node, util | |
17 from mercurial.i18n import _ | |
18 | |
19 import lfcommands | |
20 import proto | |
21 import lfutil | |
22 | |
23 def reposetup(ui, repo): | |
24 # wire repositories should be given new wireproto functions but not the | |
25 # other largefiles modifications | |
26 if not repo.local(): | |
27 return proto.wirereposetup(ui, repo) | |
28 | |
29 for name in ('status', 'commitctx', 'commit', 'push'): | |
30 method = getattr(repo, name) | |
31 #if not (isinstance(method, types.MethodType) and | |
32 # method.im_func is repo.__class__.commitctx.im_func): | |
33 if isinstance(method, types.FunctionType) and method.func_name == \ | |
34 'wrap': | |
35 ui.warn(_('largefiles: repo method %r appears to have already been' | |
36 ' wrapped by another extension: ' | |
37 'largefiles may behave incorrectly\n') | |
38 % name) | |
39 | |
40 class lfiles_repo(repo.__class__): | |
41 lfstatus = False | |
42 def status_nolfiles(self, *args, **kwargs): | |
43 return super(lfiles_repo, self).status(*args, **kwargs) | |
44 | |
45 # When lfstatus is set, return a context that gives the names of lfiles | |
46 # instead of their corresponding standins and identifies the lfiles as | |
47 # always binary, regardless of their actual contents. | |
48 def __getitem__(self, changeid): | |
49 ctx = super(lfiles_repo, self).__getitem__(changeid) | |
50 if self.lfstatus: | |
51 class lfiles_manifestdict(manifest.manifestdict): | |
52 def __contains__(self, filename): | |
53 if super(lfiles_manifestdict, | |
54 self).__contains__(filename): | |
55 return True | |
56 return super(lfiles_manifestdict, | |
57 self).__contains__(lfutil.shortname+'/' + filename) | |
58 class lfiles_ctx(ctx.__class__): | |
59 def files(self): | |
60 filenames = super(lfiles_ctx, self).files() | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
61 return [re.sub('^\\'+lfutil.shortname+'/', '', |
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
62 filename) for filename in filenames] |
15168 | 63 def manifest(self): |
64 man1 = super(lfiles_ctx, self).manifest() | |
65 man1.__class__ = lfiles_manifestdict | |
66 return man1 | |
67 def filectx(self, path, fileid=None, filelog=None): | |
68 try: | |
69 result = super(lfiles_ctx, self).filectx(path, | |
70 fileid, filelog) | |
71 except error.LookupError: | |
72 # Adding a null character will cause Mercurial to | |
73 # identify this as a binary file. | |
74 result = super(lfiles_ctx, self).filectx( | |
75 lfutil.shortname + '/' + path, fileid, | |
76 filelog) | |
77 olddata = result.data | |
78 result.data = lambda: olddata() + '\0' | |
79 return result | |
80 ctx.__class__ = lfiles_ctx | |
81 return ctx | |
82 | |
83 # Figure out the status of big files and insert them into the | |
84 # appropriate list in the result. Also removes standin files from | |
85 # the listing. This function reverts to the original status if | |
86 # self.lfstatus is False | |
87 def status(self, node1='.', node2=None, match=None, ignored=False, | |
88 clean=False, unknown=False, listsubrepos=False): | |
89 listignored, listclean, listunknown = ignored, clean, unknown | |
90 if not self.lfstatus: | |
91 try: | |
92 return super(lfiles_repo, self).status(node1, node2, match, | |
93 listignored, listclean, listunknown, listsubrepos) | |
94 except TypeError: | |
95 return super(lfiles_repo, self).status(node1, node2, match, | |
96 listignored, listclean, listunknown) | |
97 else: | |
98 # some calls in this function rely on the old version of status | |
99 self.lfstatus = False | |
100 if isinstance(node1, context.changectx): | |
101 ctx1 = node1 | |
102 else: | |
103 ctx1 = repo[node1] | |
104 if isinstance(node2, context.changectx): | |
105 ctx2 = node2 | |
106 else: | |
107 ctx2 = repo[node2] | |
108 working = ctx2.rev() is None | |
109 parentworking = working and ctx1 == self['.'] | |
110 | |
111 def inctx(file, ctx): | |
112 try: | |
113 if ctx.rev() is None: | |
114 return file in ctx.manifest() | |
115 ctx[file] | |
116 return True | |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
117 except KeyError: |
15168 | 118 return False |
119 | |
120 # create a copy of match that matches standins instead of | |
121 # lfiles if matcher not set then it is the always matcher so | |
122 # overwrite that | |
123 if match is None: | |
124 match = match_.always(self.root, self.getcwd()) | |
125 | |
126 def tostandin(file): | |
127 if inctx(lfutil.standin(file), ctx2): | |
128 return lfutil.standin(file) | |
129 return file | |
130 | |
131 m = copy.copy(match) | |
132 m._files = [tostandin(f) for f in m._files] | |
133 | |
134 # get ignored clean and unknown but remove them later if they | |
135 # were not asked for | |
136 try: | |
137 result = super(lfiles_repo, self).status(node1, node2, m, | |
138 True, True, True, listsubrepos) | |
139 except TypeError: | |
140 result = super(lfiles_repo, self).status(node1, node2, m, | |
141 True, True, True) | |
142 if working: | |
143 # Hold the wlock while we read lfiles and update the | |
144 # lfdirstate | |
145 wlock = repo.wlock() | |
146 try: | |
147 # Any non lfiles that were explicitly listed must be | |
148 # taken out or lfdirstate.status will report an error. | |
149 # The status of these files was already computed using | |
150 # super's status. | |
151 lfdirstate = lfutil.openlfdirstate(ui, self) | |
152 match._files = [f for f in match._files if f in | |
153 lfdirstate] | |
154 s = lfdirstate.status(match, [], listignored, | |
155 listclean, listunknown) | |
156 (unsure, modified, added, removed, missing, unknown, | |
157 ignored, clean) = s | |
158 if parentworking: | |
159 for lfile in unsure: | |
160 if ctx1[lfutil.standin(lfile)].data().strip() \ | |
161 != lfutil.hashfile(self.wjoin(lfile)): | |
162 modified.append(lfile) | |
163 else: | |
164 clean.append(lfile) | |
165 lfdirstate.normal(lfile) | |
166 lfdirstate.write() | |
167 else: | |
168 tocheck = unsure + modified + added + clean | |
169 modified, added, clean = [], [], [] | |
170 | |
171 for lfile in tocheck: | |
172 standin = lfutil.standin(lfile) | |
173 if inctx(standin, ctx1): | |
174 if ctx1[standin].data().strip() != \ | |
175 lfutil.hashfile(self.wjoin(lfile)): | |
176 modified.append(lfile) | |
177 else: | |
178 clean.append(lfile) | |
179 else: | |
180 added.append(lfile) | |
181 finally: | |
182 wlock.release() | |
183 | |
184 for standin in ctx1.manifest(): | |
185 if not lfutil.isstandin(standin): | |
186 continue | |
187 lfile = lfutil.splitstandin(standin) | |
188 if not match(lfile): | |
189 continue | |
190 if lfile not in lfdirstate: | |
191 removed.append(lfile) | |
192 # Handle unknown and ignored differently | |
193 lfiles = (modified, added, removed, missing, [], [], clean) | |
194 result = list(result) | |
195 # Unknown files | |
196 result[4] = [f for f in unknown if repo.dirstate[f] == '?'\ | |
197 and not lfutil.isstandin(f)] | |
198 # Ignored files must be ignored by both the dirstate and | |
199 # lfdirstate | |
200 result[5] = set(ignored).intersection(set(result[5])) | |
201 # combine normal files and lfiles | |
202 normals = [[fn for fn in filelist if not \ | |
203 lfutil.isstandin(fn)] for filelist in result] | |
204 result = [sorted(list1 + list2) for (list1, list2) in \ | |
205 zip(normals, lfiles)] | |
206 else: | |
207 def toname(f): | |
208 if lfutil.isstandin(f): | |
209 return lfutil.splitstandin(f) | |
210 return f | |
211 result = [[toname(f) for f in items] for items in result] | |
212 | |
213 if not listunknown: | |
214 result[4] = [] | |
215 if not listignored: | |
216 result[5] = [] | |
217 if not listclean: | |
218 result[6] = [] | |
219 self.lfstatus = True | |
220 return result | |
221 | |
222 # This call happens after a commit has occurred. Copy all of the lfiles | |
223 # into the cache | |
224 def commitctx(self, *args, **kwargs): | |
225 node = super(lfiles_repo, self).commitctx(*args, **kwargs) | |
226 ctx = self[node] | |
227 for filename in ctx.files(): | |
228 if lfutil.isstandin(filename) and filename in ctx.manifest(): | |
229 realfile = lfutil.splitstandin(filename) | |
230 lfutil.copytocache(self, ctx.node(), realfile) | |
231 | |
232 return node | |
233 | |
234 # This call happens before a commit has occurred. The lfile standins | |
235 # have not had their contents updated (to reflect the hash of their | |
236 # lfile). Do that here. | |
237 def commit(self, text="", user=None, date=None, match=None, | |
238 force=False, editor=False, extra={}): | |
239 orig = super(lfiles_repo, self).commit | |
240 | |
241 wlock = repo.wlock() | |
242 try: | |
243 if getattr(repo, "_isrebasing", False): | |
244 # We have to take the time to pull down the new lfiles now. | |
245 # Otherwise if we are rebasing, any lfiles that were | |
246 # modified in the changesets we are rebasing on top of get | |
247 # overwritten either by the rebase or in the first commit | |
248 # after the rebase. | |
249 lfcommands.updatelfiles(repo.ui, repo) | |
250 # Case 1: user calls commit with no specific files or | |
15250
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
251 # include/exclude patterns: refresh and commit all files that |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
252 # are "dirty". |
15168 | 253 if (match is None) or (not match.anypats() and not \ |
254 match.files()): | |
15250
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
255 # Spend a bit of time here to get a list of files we know |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
256 # are modified so we can compare only against those. |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
257 # It can cost a lot of time (several seconds) |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
258 # otherwise to update all standins if the largefiles are |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
259 # large. |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
260 lfdirstate = lfutil.openlfdirstate(ui, self) |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
261 dirtymatch = match_.always(repo.root, repo.getcwd()) |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
262 s = lfdirstate.status(dirtymatch, [], False, False, False) |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
263 modifiedfiles = [] |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
264 for i in s: |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
265 modifiedfiles.extend(i) |
15168 | 266 lfiles = lfutil.listlfiles(self) |
267 # this only loops through lfiles that exist (not | |
268 # removed/renamed) | |
269 for lfile in lfiles: | |
15250
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
270 if lfile in modifiedfiles: |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
271 if os.path.exists(self.wjoin(lfutil.standin(lfile))): |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
272 # this handles the case where a rebase is being |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
273 # performed and the working copy is not updated |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
274 # yet. |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
275 if os.path.exists(self.wjoin(lfile)): |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
276 lfutil.updatestandin(self, |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
277 lfutil.standin(lfile)) |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
278 lfdirstate.normal(lfile) |
15168 | 279 for lfile in lfdirstate: |
15250
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
280 if lfile in modifiedfiles: |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
281 if not os.path.exists( |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
282 repo.wjoin(lfutil.standin(lfile))): |
f172292cd416
largefiles: speed up commit by only rewriting standins for modified largefiles
Na'Tosha Bard <natosha@unity3d.com>
parents:
15224
diff
changeset
|
283 lfdirstate.drop(lfile) |
15168 | 284 lfdirstate.write() |
285 | |
286 return orig(text=text, user=user, date=date, match=match, | |
287 force=force, editor=editor, extra=extra) | |
288 | |
289 for file in match.files(): | |
290 if lfutil.isstandin(file): | |
291 raise util.Abort( | |
292 "Don't commit largefile standin. Commit largefile.") | |
293 | |
294 # Case 2: user calls commit with specified patterns: refresh | |
295 # any matching big files. | |
296 smatcher = lfutil.composestandinmatcher(self, match) | |
297 standins = lfutil.dirstate_walk(self.dirstate, smatcher) | |
298 | |
299 # No matching big files: get out of the way and pass control to | |
300 # the usual commit() method. | |
301 if not standins: | |
302 return orig(text=text, user=user, date=date, match=match, | |
303 force=force, editor=editor, extra=extra) | |
304 | |
305 # Refresh all matching big files. It's possible that the | |
306 # commit will end up failing, in which case the big files will | |
307 # stay refreshed. No harm done: the user modified them and | |
308 # asked to commit them, so sooner or later we're going to | |
309 # refresh the standins. Might as well leave them refreshed. | |
310 lfdirstate = lfutil.openlfdirstate(ui, self) | |
311 for standin in standins: | |
312 lfile = lfutil.splitstandin(standin) | |
313 if lfdirstate[lfile] <> 'r': | |
314 lfutil.updatestandin(self, standin) | |
315 lfdirstate.normal(lfile) | |
316 else: | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15171
diff
changeset
|
317 lfdirstate.drop(lfile) |
15168 | 318 lfdirstate.write() |
319 | |
320 # Cook up a new matcher that only matches regular files or | |
321 # standins corresponding to the big files requested by the | |
322 # user. Have to modify _files to prevent commit() from | |
323 # complaining "not tracked" for big files. | |
324 lfiles = lfutil.listlfiles(repo) | |
325 match = copy.copy(match) | |
326 orig_matchfn = match.matchfn | |
327 | |
328 # Check both the list of lfiles and the list of standins | |
329 # because if a lfile was removed, it won't be in the list of | |
330 # lfiles at this point | |
331 match._files += sorted(standins) | |
332 | |
333 actualfiles = [] | |
334 for f in match._files: | |
335 fstandin = lfutil.standin(f) | |
336 | |
337 # Ignore known lfiles and standins | |
338 if f in lfiles or fstandin in standins: | |
339 continue | |
340 | |
341 # Append directory separator to avoid collisions | |
342 if not fstandin.endswith(os.sep): | |
343 fstandin += os.sep | |
344 | |
345 # Prevalidate matching standin directories | |
346 if lfutil.any_(st for st in match._files if \ | |
347 st.startswith(fstandin)): | |
348 continue | |
349 actualfiles.append(f) | |
350 match._files = actualfiles | |
351 | |
352 def matchfn(f): | |
353 if orig_matchfn(f): | |
354 return f not in lfiles | |
355 else: | |
356 return f in standins | |
357 | |
358 match.matchfn = matchfn | |
359 return orig(text=text, user=user, date=date, match=match, | |
360 force=force, editor=editor, extra=extra) | |
361 finally: | |
362 wlock.release() | |
363 | |
364 def push(self, remote, force=False, revs=None, newbranch=False): | |
365 o = lfutil.findoutgoing(repo, remote, force) | |
366 if o: | |
367 toupload = set() | |
368 o = repo.changelog.nodesbetween(o, revs)[0] | |
369 for n in o: | |
370 parents = [p for p in repo.changelog.parents(n) if p != \ | |
371 node.nullid] | |
372 ctx = repo[n] | |
373 files = set(ctx.files()) | |
374 if len(parents) == 2: | |
375 mc = ctx.manifest() | |
376 mp1 = ctx.parents()[0].manifest() | |
377 mp2 = ctx.parents()[1].manifest() | |
378 for f in mp1: | |
379 if f not in mc: | |
380 files.add(f) | |
381 for f in mp2: | |
382 if f not in mc: | |
383 files.add(f) | |
384 for f in mc: | |
385 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, | |
386 None): | |
387 files.add(f) | |
388 | |
389 toupload = toupload.union(set([ctx[f].data().strip() for f\ | |
390 in files if lfutil.isstandin(f) and f in ctx])) | |
391 lfcommands.uploadlfiles(ui, self, remote, toupload) | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15171
diff
changeset
|
392 return super(lfiles_repo, self).push(remote, force, revs, |
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15171
diff
changeset
|
393 newbranch) |
15168 | 394 |
395 repo.__class__ = lfiles_repo | |
396 | |
397 def checkrequireslfiles(ui, repo, **kwargs): | |
398 if 'largefiles' not in repo.requirements and lfutil.any_( | |
399 lfutil.shortname+'/' in f[0] for f in repo.store.datafiles()): | |
400 # work around bug in mercurial 1.9 whereby requirements is a list | |
401 # on newly-cloned repos | |
402 repo.requirements = set(repo.requirements) | |
403 | |
404 repo.requirements |= set(['largefiles']) | |
405 repo._writerequirements() | |
406 | |
407 checkrequireslfiles(ui, repo) | |
408 | |
409 ui.setconfig('hooks', 'changegroup.lfiles', checkrequireslfiles) | |
410 ui.setconfig('hooks', 'commit.lfiles', checkrequireslfiles) |