Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/lfcommands.py @ 19160:0848be1f1aad stable
largefiles: check existence of the file with case awareness of the filesystem
Before this patch, largefiles extension always unlinks largefiles
untracked on the target context in merging/updating after updating
working directory.
For example, it is assumed that the revision X consists of ".hglf/A"
(and "A" implicitly) and revision Y consists of "a" (not ".hglf/A").
In the case of updating from X to Y, largefiles extension tries to
unlink "A" after updating "a" in working directory. This causes
unexpected unlinking "a" on the case insensitive filesystem.
This patch checks existence of the file in the working context with
case awareness of the filesystem to prevent from such unexpected
unlinking.
"lfcommands._updatelfile()" also unlinks target file in the case
"largefile is tracked in the target context, but fails to be fetched".
This patch doesn't apply "repo.dirstate.normalize()" in this case,
because it should be already ensured in the manifest merging that
there is no normal file colliding against any largefiles.
author | FUJIWARA Katsunori <foozy@lares.dti.ne.jp> |
---|---|
date | Tue, 07 May 2013 05:04:11 +0900 |
parents | 0509ae083ec1 |
children | 750d04e747aa |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
9 '''High-level command function for lfconvert, plus the cmdtable.''' |
15168 | 10 |
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
11 import os, errno |
15168 | 12 import shutil |
13 | |
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
14 from mercurial import util, match as match_, hg, node, context, error, \ |
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
15 cmdutil, scmutil, commands |
15168 | 16 from mercurial.i18n import _ |
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
17 from mercurial.lock import release |
15168 | 18 |
19 import lfutil | |
20 import basestore | |
21 | |
22 # -- Commands ---------------------------------------------------------- | |
23 | |
24 def lfconvert(ui, src, dest, *pats, **opts): | |
15230 | 25 '''convert a normal repository to a largefiles repository |
15168 | 26 |
15230 | 27 Convert repository SOURCE to a new repository DEST, identical to |
28 SOURCE except that certain files will be converted as largefiles: | |
29 specifically, any file that matches any PATTERN *or* whose size is | |
30 above the minimum size threshold is converted as a largefile. The | |
31 size used to determine whether or not to track a file as a | |
32 largefile is the size of the first version of the file. The | |
33 minimum size can be specified either with --size or in | |
34 configuration as ``largefiles.size``. | |
35 | |
36 After running this command you will need to make sure that | |
37 largefiles is enabled anywhere you intend to push the new | |
38 repository. | |
39 | |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
40 Use --to-normal to convert largefiles back to normal files; after |
15230 | 41 this, the DEST repository can be used without largefiles at all.''' |
15168 | 42 |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
43 if opts['to_normal']: |
15168 | 44 tolfile = False |
45 else: | |
46 tolfile = True | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15224
diff
changeset
|
47 size = lfutil.getminsize(ui, True, opts.get('size'), default=None) |
15340
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
48 |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
49 if not hg.islocal(src): |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
50 raise util.Abort(_('%s is not a local Mercurial repo') % src) |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
51 if not hg.islocal(dest): |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
52 raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
53 |
15339
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
54 rsrc = hg.repository(ui, src) |
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
55 ui.status(_('initializing destination %s\n') % dest) |
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
56 rdst = hg.repository(ui, dest, create=True) |
15168 | 57 |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
58 success = False |
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
59 dstwlock = dstlock = None |
15168 | 60 try: |
61 # Lock destination to prevent modification while it is converted to. | |
62 # Don't need to lock src because we are just reading from its history | |
63 # which can't change. | |
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
64 dstwlock = rdst.wlock() |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
65 dstlock = rdst.lock() |
15168 | 66 |
67 # Get a list of all changesets in the source. The easy way to do this | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
68 # is to simply walk the changelog, using changelog.nodesbetween(). |
15168 | 69 # Take a look at mercurial/revlog.py:639 for more details. |
70 # Use a generator instead of a list to decrease memory usage | |
71 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, | |
72 rsrc.heads())[0]) | |
73 revmap = {node.nullid: node.nullid} | |
74 if tolfile: | |
75 lfiles = set() | |
76 normalfiles = set() | |
77 if not pats: | |
15579
6c5e6ebe0812
largefiles: use "ui.configlist()" to get largefiles.patterns configuration
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15472
diff
changeset
|
78 pats = ui.configlist(lfutil.longname, 'patterns', default=[]) |
15168 | 79 if pats: |
80 matcher = match_.match(rsrc.root, '', list(pats)) | |
81 else: | |
82 matcher = None | |
83 | |
84 lfiletohash = {} | |
85 for ctx in ctxs: | |
86 ui.progress(_('converting revisions'), ctx.rev(), | |
87 unit=_('revision'), total=rsrc['tip'].rev()) | |
88 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, | |
89 lfiles, normalfiles, matcher, size, lfiletohash) | |
90 ui.progress(_('converting revisions'), None) | |
91 | |
92 if os.path.exists(rdst.wjoin(lfutil.shortname)): | |
93 shutil.rmtree(rdst.wjoin(lfutil.shortname)) | |
94 | |
95 for f in lfiletohash.keys(): | |
96 if os.path.isfile(rdst.wjoin(f)): | |
97 os.unlink(rdst.wjoin(f)) | |
98 try: | |
99 os.removedirs(os.path.dirname(rdst.wjoin(f))) | |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
100 except OSError: |
15168 | 101 pass |
102 | |
15303
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
103 # If there were any files converted to largefiles, add largefiles |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
104 # to the destination repository's requirements. |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
105 if lfiles: |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
106 rdst.requirements.add('largefiles') |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
107 rdst._writerequirements() |
15168 | 108 else: |
109 for ctx in ctxs: | |
110 ui.progress(_('converting revisions'), ctx.rev(), | |
111 unit=_('revision'), total=rsrc['tip'].rev()) | |
112 _addchangeset(ui, rsrc, rdst, ctx, revmap) | |
113 | |
114 ui.progress(_('converting revisions'), None) | |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
115 success = True |
15168 | 116 finally: |
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
117 rdst.dirstate.clear() |
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
118 release(dstlock, dstwlock) |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
119 if not success: |
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
120 # we failed, remove the new directory |
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
121 shutil.rmtree(rdst.root) |
15168 | 122 |
123 def _addchangeset(ui, rsrc, rdst, ctx, revmap): | |
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
124 # Convert src parents to dst parents |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
125 parents = _convertparents(ctx, revmap) |
15168 | 126 |
127 # Generate list of changed files | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
128 files = _getchangedfiles(ctx, parents) |
15168 | 129 |
130 def getfilectx(repo, memctx, f): | |
131 if lfutil.standin(f) in files: | |
132 # if the file isn't in the manifest then it was removed | |
133 # or renamed, raise IOError to indicate this | |
134 try: | |
135 fctx = ctx.filectx(lfutil.standin(f)) | |
136 except error.LookupError: | |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
137 raise IOError |
15168 | 138 renamed = fctx.renamed() |
139 if renamed: | |
140 renamed = lfutil.splitstandin(renamed[0]) | |
141 | |
142 hash = fctx.data().strip() | |
143 path = lfutil.findfile(rsrc, hash) | |
17823
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
144 |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
145 # If one file is missing, likely all files from this rev are |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
146 if path is None: |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
147 cachelfiles(ui, rsrc, ctx.node()) |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
148 path = lfutil.findfile(rsrc, hash) |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
149 |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
150 if path is None: |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
151 raise util.Abort( |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
152 _("missing largefile \'%s\' from revision %s") |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
153 % (f, node.hex(ctx.node()))) |
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
154 |
15168 | 155 data = '' |
156 fd = None | |
157 try: | |
158 fd = open(path, 'rb') | |
159 data = fd.read() | |
160 finally: | |
15172
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
161 if fd: |
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
162 fd.close() |
15168 | 163 return context.memfilectx(f, data, 'l' in fctx.flags(), |
164 'x' in fctx.flags(), renamed) | |
165 else: | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
166 return _getnormalcontext(repo.ui, ctx, f, revmap) |
15168 | 167 |
168 dstfiles = [] | |
169 for file in files: | |
170 if lfutil.isstandin(file): | |
171 dstfiles.append(lfutil.splitstandin(file)) | |
172 else: | |
173 dstfiles.append(file) | |
174 # Commit | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
175 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
15168 | 176 |
177 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, | |
178 matcher, size, lfiletohash): | |
179 # Convert src parents to dst parents | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
180 parents = _convertparents(ctx, revmap) |
15168 | 181 |
182 # Generate list of changed files | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
183 files = _getchangedfiles(ctx, parents) |
15168 | 184 |
185 dstfiles = [] | |
186 for f in files: | |
187 if f not in lfiles and f not in normalfiles: | |
188 islfile = _islfile(f, ctx, matcher, size) | |
189 # If this file was renamed or copied then copy | |
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
190 # the largefile-ness of its predecessor |
15168 | 191 if f in ctx.manifest(): |
192 fctx = ctx.filectx(f) | |
193 renamed = fctx.renamed() | |
194 renamedlfile = renamed and renamed[0] in lfiles | |
195 islfile |= renamedlfile | |
196 if 'l' in fctx.flags(): | |
197 if renamedlfile: | |
198 raise util.Abort( | |
15380
a53888685a6c
largefiles: fix uppercase in abort message
Martin Geisler <mg@aragost.com>
parents:
15371
diff
changeset
|
199 _('renamed/copied largefile %s becomes symlink') |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
200 % f) |
15168 | 201 islfile = False |
202 if islfile: | |
203 lfiles.add(f) | |
204 else: | |
205 normalfiles.add(f) | |
206 | |
207 if f in lfiles: | |
208 dstfiles.append(lfutil.standin(f)) | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
209 # largefile in manifest if it has not been removed/renamed |
15168 | 210 if f in ctx.manifest(): |
15808
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
211 fctx = ctx.filectx(f) |
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
212 if 'l' in fctx.flags(): |
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
213 renamed = fctx.renamed() |
15168 | 214 if renamed and renamed[0] in lfiles: |
215 raise util.Abort(_('largefile %s becomes symlink') % f) | |
216 | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
217 # largefile was modified, update standins |
15168 | 218 m = util.sha1('') |
219 m.update(ctx[f].data()) | |
220 hash = m.hexdigest() | |
221 if f not in lfiletohash or lfiletohash[f] != hash: | |
19089
0509ae083ec1
largefiles: use repo.wwrite for writing standins (issue3909)
Mads Kiilerich <madski@unity3d.com>
parents:
18976
diff
changeset
|
222 rdst.wwrite(f, ctx[f].data(), ctx[f].flags()) |
15168 | 223 executable = 'x' in ctx[f].flags() |
224 lfutil.writestandin(rdst, lfutil.standin(f), hash, | |
225 executable) | |
226 lfiletohash[f] = hash | |
227 else: | |
228 # normal file | |
229 dstfiles.append(f) | |
230 | |
231 def getfilectx(repo, memctx, f): | |
232 if lfutil.isstandin(f): | |
233 # if the file isn't in the manifest then it was removed | |
234 # or renamed, raise IOError to indicate this | |
235 srcfname = lfutil.splitstandin(f) | |
236 try: | |
237 fctx = ctx.filectx(srcfname) | |
238 except error.LookupError: | |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
239 raise IOError |
15168 | 240 renamed = fctx.renamed() |
241 if renamed: | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
242 # standin is always a largefile because largefile-ness |
15168 | 243 # doesn't change after rename or copy |
244 renamed = lfutil.standin(renamed[0]) | |
245 | |
15313
3eb1a90ea409
largefiles: fix newline for lfconverted repos
Eli Carter <eli.carter@tektronix.com>
parents:
15303
diff
changeset
|
246 return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in |
15168 | 247 fctx.flags(), 'x' in fctx.flags(), renamed) |
248 else: | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
249 return _getnormalcontext(repo.ui, ctx, f, revmap) |
15168 | 250 |
251 # Commit | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
252 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
253 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
254 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap): |
15168 | 255 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
256 getfilectx, ctx.user(), ctx.date(), ctx.extra()) | |
257 ret = rdst.commitctx(mctx) | |
16551
ebf6d38c9063
localrepo: add setparents() to adjust dirstate copies (issue3407)
Patrick Mezard <patrick@mezard.eu>
parents:
16439
diff
changeset
|
258 rdst.setparents(ret) |
15168 | 259 revmap[ctx.node()] = rdst.changelog.tip() |
260 | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
261 # Generate list of changed files |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
262 def _getchangedfiles(ctx, parents): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
263 files = set(ctx.files()) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
264 if node.nullid not in parents: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
265 mc = ctx.manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
266 mp1 = ctx.parents()[0].manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
267 mp2 = ctx.parents()[1].manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
268 files |= (set(mp1) | set(mp2)) - set(mc) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
269 for f in mc: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
270 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
271 files.add(f) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
272 return files |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
273 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
274 # Convert src parents to dst parents |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
275 def _convertparents(ctx, revmap): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
276 parents = [] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
277 for p in ctx.parents(): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
278 parents.append(revmap[p.node()]) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
279 while len(parents) < 2: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
280 parents.append(node.nullid) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
281 return parents |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
282 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
283 # Get memfilectx for a normal file |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
284 def _getnormalcontext(ui, ctx, f, revmap): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
285 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
286 fctx = ctx.filectx(f) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
287 except error.LookupError: |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
288 raise IOError |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
289 renamed = fctx.renamed() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
290 if renamed: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
291 renamed = renamed[0] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
292 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
293 data = fctx.data() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
294 if f == '.hgtags': |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
295 data = _converttags (ui, revmap, data) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
296 return context.memfilectx(f, data, 'l' in fctx.flags(), |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
297 'x' in fctx.flags(), renamed) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
298 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
299 # Remap tag data using a revision map |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
300 def _converttags(ui, revmap, data): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
301 newdata = [] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
302 for line in data.splitlines(): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
303 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
304 id, name = line.split(' ', 1) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
305 except ValueError: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
306 ui.warn(_('skipping incorrectly formatted tag %s\n' |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
307 % line)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
308 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
309 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
310 newid = node.bin(id) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
311 except TypeError: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
312 ui.warn(_('skipping incorrectly formatted id %s\n' |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
313 % id)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
314 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
315 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
316 newdata.append('%s %s\n' % (node.hex(revmap[newid]), |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
317 name)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
318 except KeyError: |
16231
ce292f1379ba
i18n: fix all remaining uses of % inside _()
Matt Mackall <mpm@selenic.com>
parents:
15909
diff
changeset
|
319 ui.warn(_('no mapping for id %s\n') % id) |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
320 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
321 return ''.join(newdata) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
322 |
15168 | 323 def _islfile(file, ctx, matcher, size): |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
324 '''Return true if file should be considered a largefile, i.e. |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
325 matcher matches it or it is larger than size.''' |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
326 # never store special .hg* files as largefiles |
15168 | 327 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
328 return False | |
329 if matcher and matcher(file): | |
330 return True | |
331 try: | |
332 return ctx.filectx(file).size() >= size * 1024 * 1024 | |
333 except error.LookupError: | |
334 return False | |
335 | |
336 def uploadlfiles(ui, rsrc, rdst, files): | |
337 '''upload largefiles to the central store''' | |
338 | |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15313
diff
changeset
|
339 if not files: |
15168 | 340 return |
341 | |
342 store = basestore._openstore(rsrc, rdst, put=True) | |
343 | |
344 at = 0 | |
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
345 ui.debug("sending statlfile command for %d largefiles\n" % len(files)) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
346 retval = store.exists(files) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
347 files = filter(lambda h: not retval[h], files) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
348 ui.debug("%d largefiles need to be uploaded\n" % len(files)) |
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
349 |
15168 | 350 for hash in files: |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
351 ui.progress(_('uploading largefiles'), at, unit='largefile', |
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
352 total=len(files)) |
15168 | 353 source = lfutil.findfile(rsrc, hash) |
354 if not source: | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
355 raise util.Abort(_('largefile %s missing from store' |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
356 ' (needs to be uploaded)') % hash) |
15168 | 357 # XXX check for errors here |
358 store.put(source, hash) | |
359 at += 1 | |
15173
3d27a8ff895f
largefiles: mark a string for translation
Matt Mackall <mpm@selenic.com>
parents:
15172
diff
changeset
|
360 ui.progress(_('uploading largefiles'), None) |
15168 | 361 |
362 def verifylfiles(ui, repo, all=False, contents=False): | |
18574
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18294
diff
changeset
|
363 '''Verify that every largefile revision in the current changeset |
15168 | 364 exists in the central store. With --contents, also verify that |
18574
4db9e31ae605
largefiles: docstrings for verify methods
Mads Kiilerich <mads@kiilerich.com>
parents:
18294
diff
changeset
|
365 the contents of each local largefile file revision are correct (SHA-1 hash |
15168 | 366 matches the revision ID). With --all, check every changeset in |
367 this repository.''' | |
368 if all: | |
369 # Pass a list to the function rather than an iterator because we know a | |
370 # list will work. | |
371 revs = range(len(repo)) | |
372 else: | |
373 revs = ['.'] | |
374 | |
375 store = basestore._openstore(repo) | |
376 return store.verify(revs, contents=contents) | |
377 | |
18144
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
378 def debugdirstate(ui, repo): |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
379 '''Show basic information for the largefiles dirstate''' |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
380 lfdirstate = lfutil.openlfdirstate(ui, repo) |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
381 for file_, ent in sorted(lfdirstate._map.iteritems()): |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
382 mode = '%3o' % (ent[1] & 0777 & ~util.umask) |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
383 ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_)) |
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
384 |
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
385 def cachelfiles(ui, repo, node, filelist=None): |
15168 | 386 '''cachelfiles ensures that all largefiles needed by the specified revision |
387 are present in the repository's largefile cache. | |
388 | |
389 returns a tuple (cached, missing). cached is the list of files downloaded | |
390 by this operation; missing is the list of files that were needed but could | |
391 not be found.''' | |
392 lfiles = lfutil.listlfiles(repo, node) | |
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
393 if filelist: |
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
394 lfiles = set(lfiles) & set(filelist) |
15168 | 395 toget = [] |
396 | |
397 for lfile in lfiles: | |
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
398 try: |
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
399 expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
400 except IOError, err: |
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
401 if err.errno == errno.ENOENT: |
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
402 continue # node must be None and standin wasn't found in wctx |
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
403 raise |
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
404 if not lfutil.findfile(repo, expectedhash): |
15168 | 405 toget.append((lfile, expectedhash)) |
406 | |
407 if toget: | |
408 store = basestore._openstore(repo) | |
409 ret = store.get(toget) | |
410 return ret | |
411 | |
412 return ([], []) | |
413 | |
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
414 def downloadlfiles(ui, repo, rev=None): |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
415 matchfn = scmutil.match(repo[None], |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
416 [repo.wjoin(lfutil.shortname)], {}) |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
417 def prepare(ctx, fns): |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
418 pass |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
419 totalsuccess = 0 |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
420 totalmissing = 0 |
18722
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
421 if rev != []: # walkchangerevs on empty list would return all revs |
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
422 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev}, |
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
423 prepare): |
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
424 success, missing = cachelfiles(ui, repo, ctx.node()) |
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
425 totalsuccess += len(success) |
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
426 totalmissing += len(missing) |
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
427 ui.status(_("%d additional largefiles cached\n") % totalsuccess) |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
428 if totalmissing > 0: |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
429 ui.status(_("%d largefiles failed to download\n") % totalmissing) |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
430 return totalsuccess, totalmissing |
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
431 |
15168 | 432 def updatelfiles(ui, repo, filelist=None, printmessage=True): |
433 wlock = repo.wlock() | |
434 try: | |
435 lfdirstate = lfutil.openlfdirstate(ui, repo) | |
436 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) | |
437 | |
438 if filelist is not None: | |
439 lfiles = [f for f in lfiles if f in filelist] | |
440 | |
441 printed = False | |
442 if printmessage and lfiles: | |
443 ui.status(_('getting changed largefiles\n')) | |
444 printed = True | |
18729
4e53ac3f466a
largefiles: updatelfiles should use working dir standins, not standins from p1
Mads Kiilerich <madski@unity3d.com>
parents:
18728
diff
changeset
|
445 cachelfiles(ui, repo, None, lfiles) |
15168 | 446 |
447 updated, removed = 0, 0 | |
17430
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
448 for f in lfiles: |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
449 i = _updatelfile(repo, lfdirstate, f) |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
450 if i: |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
451 if i > 0: |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
452 updated += i |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
453 else: |
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
454 removed -= i |
15168 | 455 if printmessage and (removed or updated) and not printed: |
456 ui.status(_('getting changed largefiles\n')) | |
457 printed = True | |
458 | |
459 lfdirstate.write() | |
460 if printed and printmessage: | |
461 ui.status(_('%d largefiles updated, %d removed\n') % (updated, | |
462 removed)) | |
463 finally: | |
464 wlock.release() | |
465 | |
466 def _updatelfile(repo, lfdirstate, lfile): | |
467 '''updates a single largefile and copies the state of its standin from | |
468 the repository's dirstate to its state in the lfdirstate. | |
469 | |
470 returns 1 if the file was modified, -1 if the file was removed, 0 if the | |
471 file was unchanged, and None if the needed largefile was missing from the | |
472 cache.''' | |
473 ret = 0 | |
474 abslfile = repo.wjoin(lfile) | |
475 absstandin = repo.wjoin(lfutil.standin(lfile)) | |
476 if os.path.exists(absstandin): | |
18292
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
477 if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile): |
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
478 shutil.copyfile(abslfile, abslfile + '.orig') |
15168 | 479 expecthash = lfutil.readstandin(repo, lfile) |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
480 if (expecthash != '' and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
481 (not os.path.exists(abslfile) or |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
482 expecthash != lfutil.hashfile(abslfile))): |
15168 | 483 if not lfutil.copyfromcache(repo, expecthash, lfile): |
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
484 # use normallookup() to allocate entry in largefiles dirstate, |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
485 # because lack of it misleads lfilesrepo.status() into |
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
486 # recognition that such cache missing files are REMOVED. |
18727
4846f2115927
largefiles: don't let update leave wrong largefiles in wd if fetch fails
Mads Kiilerich <madski@unity3d.com>
parents:
18722
diff
changeset
|
487 if lfile not in repo[None]: # not switched to normal file |
4846f2115927
largefiles: don't let update leave wrong largefiles in wd if fetch fails
Mads Kiilerich <madski@unity3d.com>
parents:
18722
diff
changeset
|
488 util.unlinkpath(abslfile, ignoremissing=True) |
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
489 lfdirstate.normallookup(lfile) |
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
490 return None # don't try to set the mode |
17155
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
491 else: |
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
492 # Synchronize largefile dirstate to the last modified time of |
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
493 # the file |
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
494 lfdirstate.normal(lfile) |
15168 | 495 ret = 1 |
496 mode = os.stat(absstandin).st_mode | |
497 if mode != os.stat(abslfile).st_mode: | |
498 os.chmod(abslfile, mode) | |
499 ret = 1 | |
500 else: | |
15663
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
501 # Remove lfiles for which the standin is deleted, unless the |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
502 # lfile is added to the repository again. This happens when a |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
503 # largefile is converted back to a normal file: the standin |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
504 # disappears, but a new (normal) file appears as the lfile. |
19160
0848be1f1aad
largefiles: check existence of the file with case awareness of the filesystem
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
19089
diff
changeset
|
505 if (os.path.exists(abslfile) and |
0848be1f1aad
largefiles: check existence of the file with case awareness of the filesystem
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
19089
diff
changeset
|
506 repo.dirstate.normalize(lfile) not in repo[None]): |
15900
29defa7d20f6
largefiles: remove empty directories upon update (issue3202)
Patrick Mezard <pmezard@gmail.com>
parents:
15663
diff
changeset
|
507 util.unlinkpath(abslfile) |
15168 | 508 ret = -1 |
509 state = repo.dirstate[lfutil.standin(lfile)] | |
510 if state == 'n': | |
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
511 # When rebasing, we need to synchronize the standin and the largefile, |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
512 # because otherwise the largefile will get reverted. But for commit's |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
513 # sake, we have to mark the file as unclean. |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
514 if getattr(repo, "_isrebasing", False): |
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
515 lfdirstate.normallookup(lfile) |
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
516 else: |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
517 lfdirstate.normal(lfile) |
15168 | 518 elif state == 'r': |
519 lfdirstate.remove(lfile) | |
520 elif state == 'a': | |
521 lfdirstate.add(lfile) | |
522 elif state == '?': | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15173
diff
changeset
|
523 lfdirstate.drop(lfile) |
15168 | 524 return ret |
525 | |
18976
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
526 def lfpull(ui, repo, source="default", **opts): |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
527 """pull largefiles for the specified revisions from the specified source |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
528 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
529 Pull largefiles that are referenced from local changesets but missing |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
530 locally, pulling from a remote repository to the local cache. |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
531 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
532 If SOURCE is omitted, the 'default' path will be used. |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
533 See :hg:`help urls` for more information. |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
534 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
535 .. container:: verbose |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
536 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
537 Some examples: |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
538 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
539 - pull largefiles for all branch heads:: |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
540 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
541 hg lfpull -r "head() and not closed()" |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
542 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
543 - pull largefiles on the default branch:: |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
544 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
545 hg lfpull -r "branch(default)" |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
546 """ |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
547 repo.lfpullsource = source |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
548 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
549 revs = opts.get('rev', []) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
550 if not revs: |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
551 raise util.Abort(_('no revisions specified')) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
552 revs = scmutil.revrange(repo, revs) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
553 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
554 numcached = 0 |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
555 for rev in revs: |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
556 ui.note(_('pulling largefiles for revision %s\n') % rev) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
557 (cached, missing) = cachelfiles(ui, repo, rev) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
558 numcached += len(cached) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
559 ui.status(_("%d largefiles cached\n") % numcached) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
560 |
15168 | 561 # -- hg commands declarations ------------------------------------------------ |
562 | |
563 cmdtable = { | |
564 'lfconvert': (lfconvert, | |
15230 | 565 [('s', 'size', '', |
566 _('minimum size (MB) for files to be converted ' | |
567 'as largefiles'), | |
568 'SIZE'), | |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
569 ('', 'to-normal', False, |
15230 | 570 _('convert from a largefiles repo to a normal repo')), |
571 ], | |
15168 | 572 _('hg lfconvert SOURCE DEST [FILE ...]')), |
18976
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
573 'lfpull': (lfpull, |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
574 [('r', 'rev', [], _('pull largefiles for these revisions')) |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
575 ] + commands.remoteopts, |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
576 _('-r REV... [-e CMD] [--remotecmd CMD] [SOURCE]') |
6734951e2d24
largefiles: introduce lfpull command for pulling missing largefiles
Mads Kiilerich <madski@unity3d.com>
parents:
18974
diff
changeset
|
577 ), |
15168 | 578 } |
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
579 |
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
580 commands.inferrepo += " lfconvert" |