Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/lfcommands.py @ 18722:f0aa8bbffe60 stable
largefiles: fix download of largefiles from an empty list of changesets
The empty list was interpreted as all revisions - just like None is.
The empty list is now handled explicitly.
| author | Mads Kiilerich <madski@unity3d.com> |
|---|---|
| date | Thu, 28 Feb 2013 13:44:24 +0100 |
| parents | 65cec7fa5472 |
| children | 4846f2115927 |
| rev | line source |
|---|---|
| 15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
| 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
| 3 # Copyright 2010-2011 Fog Creek Software | |
| 4 # Copyright 2010-2011 Unity Technologies | |
| 5 # | |
| 6 # This software may be used and distributed according to the terms of the | |
| 7 # GNU General Public License version 2 or any later version. | |
| 8 | |
|
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
9 '''High-level command function for lfconvert, plus the cmdtable.''' |
| 15168 | 10 |
| 11 import os | |
| 12 import shutil | |
| 13 | |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
14 from mercurial import util, match as match_, hg, node, context, error, \ |
|
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
15 cmdutil, scmutil, commands |
| 15168 | 16 from mercurial.i18n import _ |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
17 from mercurial.lock import release |
| 15168 | 18 |
| 19 import lfutil | |
| 20 import basestore | |
| 21 | |
| 22 # -- Commands ---------------------------------------------------------- | |
| 23 | |
| 24 def lfconvert(ui, src, dest, *pats, **opts): | |
| 15230 | 25 '''convert a normal repository to a largefiles repository |
| 15168 | 26 |
| 15230 | 27 Convert repository SOURCE to a new repository DEST, identical to |
| 28 SOURCE except that certain files will be converted as largefiles: | |
| 29 specifically, any file that matches any PATTERN *or* whose size is | |
| 30 above the minimum size threshold is converted as a largefile. The | |
| 31 size used to determine whether or not to track a file as a | |
| 32 largefile is the size of the first version of the file. The | |
| 33 minimum size can be specified either with --size or in | |
| 34 configuration as ``largefiles.size``. | |
| 35 | |
| 36 After running this command you will need to make sure that | |
| 37 largefiles is enabled anywhere you intend to push the new | |
| 38 repository. | |
| 39 | |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
40 Use --to-normal to convert largefiles back to normal files; after |
| 15230 | 41 this, the DEST repository can be used without largefiles at all.''' |
| 15168 | 42 |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
43 if opts['to_normal']: |
| 15168 | 44 tolfile = False |
| 45 else: | |
| 46 tolfile = True | |
|
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15224
diff
changeset
|
47 size = lfutil.getminsize(ui, True, opts.get('size'), default=None) |
|
15340
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
48 |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
49 if not hg.islocal(src): |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
50 raise util.Abort(_('%s is not a local Mercurial repo') % src) |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
51 if not hg.islocal(dest): |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
52 raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
53 |
|
15339
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
54 rsrc = hg.repository(ui, src) |
|
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
55 ui.status(_('initializing destination %s\n') % dest) |
|
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
56 rdst = hg.repository(ui, dest, create=True) |
| 15168 | 57 |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
58 success = False |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
59 dstwlock = dstlock = None |
| 15168 | 60 try: |
| 61 # Lock destination to prevent modification while it is converted to. | |
| 62 # Don't need to lock src because we are just reading from its history | |
| 63 # which can't change. | |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
64 dstwlock = rdst.wlock() |
|
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
65 dstlock = rdst.lock() |
| 15168 | 66 |
| 67 # Get a list of all changesets in the source. The easy way to do this | |
|
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
68 # is to simply walk the changelog, using changelog.nodesbetween(). |
| 15168 | 69 # Take a look at mercurial/revlog.py:639 for more details. |
| 70 # Use a generator instead of a list to decrease memory usage | |
| 71 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, | |
| 72 rsrc.heads())[0]) | |
| 73 revmap = {node.nullid: node.nullid} | |
| 74 if tolfile: | |
| 75 lfiles = set() | |
| 76 normalfiles = set() | |
| 77 if not pats: | |
|
15579
6c5e6ebe0812
largefiles: use "ui.configlist()" to get largefiles.patterns configuration
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15472
diff
changeset
|
78 pats = ui.configlist(lfutil.longname, 'patterns', default=[]) |
| 15168 | 79 if pats: |
| 80 matcher = match_.match(rsrc.root, '', list(pats)) | |
| 81 else: | |
| 82 matcher = None | |
| 83 | |
| 84 lfiletohash = {} | |
| 85 for ctx in ctxs: | |
| 86 ui.progress(_('converting revisions'), ctx.rev(), | |
| 87 unit=_('revision'), total=rsrc['tip'].rev()) | |
| 88 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, | |
| 89 lfiles, normalfiles, matcher, size, lfiletohash) | |
| 90 ui.progress(_('converting revisions'), None) | |
| 91 | |
| 92 if os.path.exists(rdst.wjoin(lfutil.shortname)): | |
| 93 shutil.rmtree(rdst.wjoin(lfutil.shortname)) | |
| 94 | |
| 95 for f in lfiletohash.keys(): | |
| 96 if os.path.isfile(rdst.wjoin(f)): | |
| 97 os.unlink(rdst.wjoin(f)) | |
| 98 try: | |
| 99 os.removedirs(os.path.dirname(rdst.wjoin(f))) | |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
100 except OSError: |
| 15168 | 101 pass |
| 102 | |
|
15303
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
103 # If there were any files converted to largefiles, add largefiles |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
104 # to the destination repository's requirements. |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
105 if lfiles: |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
106 rdst.requirements.add('largefiles') |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
107 rdst._writerequirements() |
| 15168 | 108 else: |
| 109 for ctx in ctxs: | |
| 110 ui.progress(_('converting revisions'), ctx.rev(), | |
| 111 unit=_('revision'), total=rsrc['tip'].rev()) | |
| 112 _addchangeset(ui, rsrc, rdst, ctx, revmap) | |
| 113 | |
| 114 ui.progress(_('converting revisions'), None) | |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
115 success = True |
| 15168 | 116 finally: |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
117 rdst.dirstate.clear() |
|
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
118 release(dstlock, dstwlock) |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
119 if not success: |
|
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
120 # we failed, remove the new directory |
|
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
121 shutil.rmtree(rdst.root) |
| 15168 | 122 |
| 123 def _addchangeset(ui, rsrc, rdst, ctx, revmap): | |
|
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
124 # Convert src parents to dst parents |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
125 parents = _convertparents(ctx, revmap) |
| 15168 | 126 |
| 127 # Generate list of changed files | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
128 files = _getchangedfiles(ctx, parents) |
| 15168 | 129 |
| 130 def getfilectx(repo, memctx, f): | |
| 131 if lfutil.standin(f) in files: | |
| 132 # if the file isn't in the manifest then it was removed | |
| 133 # or renamed, raise IOError to indicate this | |
| 134 try: | |
| 135 fctx = ctx.filectx(lfutil.standin(f)) | |
| 136 except error.LookupError: | |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
137 raise IOError |
| 15168 | 138 renamed = fctx.renamed() |
| 139 if renamed: | |
| 140 renamed = lfutil.splitstandin(renamed[0]) | |
| 141 | |
| 142 hash = fctx.data().strip() | |
| 143 path = lfutil.findfile(rsrc, hash) | |
|
17823
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
144 |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
145 # If one file is missing, likely all files from this rev are |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
146 if path is None: |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
147 cachelfiles(ui, rsrc, ctx.node()) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
148 path = lfutil.findfile(rsrc, hash) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
149 |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
150 if path is None: |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
151 raise util.Abort( |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
152 _("missing largefile \'%s\' from revision %s") |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
153 % (f, node.hex(ctx.node()))) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
154 |
| 15168 | 155 data = '' |
| 156 fd = None | |
| 157 try: | |
| 158 fd = open(path, 'rb') | |
| 159 data = fd.read() | |
| 160 finally: | |
|
15172
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
161 if fd: |
|
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
162 fd.close() |
| 15168 | 163 return context.memfilectx(f, data, 'l' in fctx.flags(), |
| 164 'x' in fctx.flags(), renamed) | |
| 165 else: | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
166 return _getnormalcontext(repo.ui, ctx, f, revmap) |
| 15168 | 167 |
| 168 dstfiles = [] | |
| 169 for file in files: | |
| 170 if lfutil.isstandin(file): | |
| 171 dstfiles.append(lfutil.splitstandin(file)) | |
| 172 else: | |
| 173 dstfiles.append(file) | |
| 174 # Commit | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
175 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
| 15168 | 176 |
| 177 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, | |
| 178 matcher, size, lfiletohash): | |
| 179 # Convert src parents to dst parents | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
180 parents = _convertparents(ctx, revmap) |
| 15168 | 181 |
| 182 # Generate list of changed files | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
183 files = _getchangedfiles(ctx, parents) |
| 15168 | 184 |
| 185 dstfiles = [] | |
| 186 for f in files: | |
| 187 if f not in lfiles and f not in normalfiles: | |
| 188 islfile = _islfile(f, ctx, matcher, size) | |
| 189 # If this file was renamed or copied then copy | |
|
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
190 # the largefile-ness of its predecessor |
| 15168 | 191 if f in ctx.manifest(): |
| 192 fctx = ctx.filectx(f) | |
| 193 renamed = fctx.renamed() | |
| 194 renamedlfile = renamed and renamed[0] in lfiles | |
| 195 islfile |= renamedlfile | |
| 196 if 'l' in fctx.flags(): | |
| 197 if renamedlfile: | |
| 198 raise util.Abort( | |
|
15380
a53888685a6c
largefiles: fix uppercase in abort message
Martin Geisler <mg@aragost.com>
parents:
15371
diff
changeset
|
199 _('renamed/copied largefile %s becomes symlink') |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
200 % f) |
| 15168 | 201 islfile = False |
| 202 if islfile: | |
| 203 lfiles.add(f) | |
| 204 else: | |
| 205 normalfiles.add(f) | |
| 206 | |
| 207 if f in lfiles: | |
| 208 dstfiles.append(lfutil.standin(f)) | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
209 # largefile in manifest if it has not been removed/renamed |
| 15168 | 210 if f in ctx.manifest(): |
|
15808
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
211 fctx = ctx.filectx(f) |
|
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
212 if 'l' in fctx.flags(): |
|
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
213 renamed = fctx.renamed() |
| 15168 | 214 if renamed and renamed[0] in lfiles: |
| 215 raise util.Abort(_('largefile %s becomes symlink') % f) | |
| 216 | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
217 # largefile was modified, update standins |
| 15168 | 218 fullpath = rdst.wjoin(f) |
|
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15340
diff
changeset
|
219 util.makedirs(os.path.dirname(fullpath)) |
| 15168 | 220 m = util.sha1('') |
| 221 m.update(ctx[f].data()) | |
| 222 hash = m.hexdigest() | |
| 223 if f not in lfiletohash or lfiletohash[f] != hash: | |
| 224 try: | |
| 225 fd = open(fullpath, 'wb') | |
| 226 fd.write(ctx[f].data()) | |
| 227 finally: | |
| 228 if fd: | |
| 229 fd.close() | |
| 230 executable = 'x' in ctx[f].flags() | |
| 231 os.chmod(fullpath, lfutil.getmode(executable)) | |
| 232 lfutil.writestandin(rdst, lfutil.standin(f), hash, | |
| 233 executable) | |
| 234 lfiletohash[f] = hash | |
| 235 else: | |
| 236 # normal file | |
| 237 dstfiles.append(f) | |
| 238 | |
| 239 def getfilectx(repo, memctx, f): | |
| 240 if lfutil.isstandin(f): | |
| 241 # if the file isn't in the manifest then it was removed | |
| 242 # or renamed, raise IOError to indicate this | |
| 243 srcfname = lfutil.splitstandin(f) | |
| 244 try: | |
| 245 fctx = ctx.filectx(srcfname) | |
| 246 except error.LookupError: | |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
247 raise IOError |
| 15168 | 248 renamed = fctx.renamed() |
| 249 if renamed: | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
250 # standin is always a largefile because largefile-ness |
| 15168 | 251 # doesn't change after rename or copy |
| 252 renamed = lfutil.standin(renamed[0]) | |
| 253 | |
|
15313
3eb1a90ea409
largefiles: fix newline for lfconverted repos
Eli Carter <eli.carter@tektronix.com>
parents:
15303
diff
changeset
|
254 return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in |
| 15168 | 255 fctx.flags(), 'x' in fctx.flags(), renamed) |
| 256 else: | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
257 return _getnormalcontext(repo.ui, ctx, f, revmap) |
| 15168 | 258 |
| 259 # Commit | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
260 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
261 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
262 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap): |
| 15168 | 263 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
| 264 getfilectx, ctx.user(), ctx.date(), ctx.extra()) | |
| 265 ret = rdst.commitctx(mctx) | |
|
16551
ebf6d38c9063
localrepo: add setparents() to adjust dirstate copies (issue3407)
Patrick Mezard <patrick@mezard.eu>
parents:
16439
diff
changeset
|
266 rdst.setparents(ret) |
| 15168 | 267 revmap[ctx.node()] = rdst.changelog.tip() |
| 268 | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
269 # Generate list of changed files |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
270 def _getchangedfiles(ctx, parents): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
271 files = set(ctx.files()) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
272 if node.nullid not in parents: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
273 mc = ctx.manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
274 mp1 = ctx.parents()[0].manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
275 mp2 = ctx.parents()[1].manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
276 files |= (set(mp1) | set(mp2)) - set(mc) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
277 for f in mc: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
278 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
279 files.add(f) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
280 return files |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
281 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
282 # Convert src parents to dst parents |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
283 def _convertparents(ctx, revmap): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
284 parents = [] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
285 for p in ctx.parents(): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
286 parents.append(revmap[p.node()]) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
287 while len(parents) < 2: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
288 parents.append(node.nullid) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
289 return parents |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
290 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
291 # Get memfilectx for a normal file |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
292 def _getnormalcontext(ui, ctx, f, revmap): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
293 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
294 fctx = ctx.filectx(f) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
295 except error.LookupError: |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
296 raise IOError |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
297 renamed = fctx.renamed() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
298 if renamed: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
299 renamed = renamed[0] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
300 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
301 data = fctx.data() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
302 if f == '.hgtags': |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
303 data = _converttags (ui, revmap, data) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
304 return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
305 'x' in fctx.flags(), renamed) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
306 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
307 # Remap tag data using a revision map |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
308 def _converttags(ui, revmap, data): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
309 newdata = [] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
310 for line in data.splitlines(): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
311 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
312 id, name = line.split(' ', 1) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
313 except ValueError: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
314 ui.warn(_('skipping incorrectly formatted tag %s\n' |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
315 % line)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
316 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
317 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
318 newid = node.bin(id) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
319 except TypeError: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
320 ui.warn(_('skipping incorrectly formatted id %s\n' |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
321 % id)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
322 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
323 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
324 newdata.append('%s %s\n' % (node.hex(revmap[newid]), |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
325 name)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
326 except KeyError: |
|
16231
ce292f1379ba
i18n: fix all remaining uses of % inside _()
Matt Mackall <mpm@selenic.com>
parents:
15909
diff
changeset
|
327 ui.warn(_('no mapping for id %s\n') % id) |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
328 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
329 return ''.join(newdata) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
330 |
| 15168 | 331 def _islfile(file, ctx, matcher, size): |
|
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
332 '''Return true if file should be considered a largefile, i.e. |
|
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
333 matcher matches it or it is larger than size.''' |
|
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
334 # never store special .hg* files as largefiles |
| 15168 | 335 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
| 336 return False | |
| 337 if matcher and matcher(file): | |
| 338 return True | |
| 339 try: | |
| 340 return ctx.filectx(file).size() >= size * 1024 * 1024 | |
| 341 except error.LookupError: | |
| 342 return False | |
| 343 | |
| 344 def uploadlfiles(ui, rsrc, rdst, files): | |
| 345 '''upload largefiles to the central store''' | |
| 346 | |
|
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15313
diff
changeset
|
347 if not files: |
| 15168 | 348 return |
| 349 | |
| 350 store = basestore._openstore(rsrc, rdst, put=True) | |
| 351 | |
| 352 at = 0 | |
|
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
353 ui.debug("sending statlfile command for %d largefiles\n" % len(files)) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
354 retval = store.exists(files) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
355 files = filter(lambda h: not retval[h], files) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
356 ui.debug("%d largefiles need to be uploaded\n" % len(files)) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
357 |
| 15168 | 358 for hash in files: |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
359 ui.progress(_('uploading largefiles'), at, unit='largefile', |
|
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
360 total=len(files)) |
| 15168 | 361 source = lfutil.findfile(rsrc, hash) |
| 362 if not source: | |
|
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
363 raise util.Abort(_('largefile %s missing from store' |
|
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
364 ' (needs to be uploaded)') % hash) |
| 15168 | 365 # XXX check for errors here |
| 366 store.put(source, hash) | |
| 367 at += 1 | |
|
15173
3d27a8ff895f
largefiles: mark a string for translation
Matt Mackall <mpm@selenic.com>
parents:
15172
diff
changeset
|
368 ui.progress(_('uploading largefiles'), None) |
| 15168 | 369 |
| 370 def verifylfiles(ui, repo, all=False, contents=False): | |
| 371 '''Verify that every big file revision in the current changeset | |
| 372 exists in the central store. With --contents, also verify that | |
| 373 the contents of each big file revision are correct (SHA-1 hash | |
| 374 matches the revision ID). With --all, check every changeset in | |
| 375 this repository.''' | |
| 376 if all: | |
| 377 # Pass a list to the function rather than an iterator because we know a | |
| 378 # list will work. | |
| 379 revs = range(len(repo)) | |
| 380 else: | |
| 381 revs = ['.'] | |
| 382 | |
| 383 store = basestore._openstore(repo) | |
| 384 return store.verify(revs, contents=contents) | |
| 385 | |
|
18144
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
386 def debugdirstate(ui, repo): |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
387 '''Show basic information for the largefiles dirstate''' |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
388 lfdirstate = lfutil.openlfdirstate(ui, repo) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
389 for file_, ent in sorted(lfdirstate._map.iteritems()): |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
390 mode = '%3o' % (ent[1] & 0777 & ~util.umask) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
391 ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_)) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
392 |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
393 def cachelfiles(ui, repo, node, filelist=None): |
| 15168 | 394 '''cachelfiles ensures that all largefiles needed by the specified revision |
| 395 are present in the repository's largefile cache. | |
| 396 | |
| 397 returns a tuple (cached, missing). cached is the list of files downloaded | |
| 398 by this operation; missing is the list of files that were needed but could | |
| 399 not be found.''' | |
| 400 lfiles = lfutil.listlfiles(repo, node) | |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
401 if filelist: |
|
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
402 lfiles = set(lfiles) & set(filelist) |
| 15168 | 403 toget = [] |
| 404 | |
| 405 for lfile in lfiles: | |
|
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
406 # If we are mid-merge, then we have to trust the standin that is in the |
|
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
407 # working copy to have the correct hashvalue. This is because the |
|
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
408 # original hg.merge() already updated the standin as part of the normal |
| 17526 | 409 # merge process -- we just have to update the largefile to match. |
|
15905
634d49a8b6db
largefiles: correctly handle newly added largefile on other side of merge
Na'Tosha Bard <natosha@unity3d.com>
parents:
15860
diff
changeset
|
410 if (getattr(repo, "_ismerging", False) and |
|
634d49a8b6db
largefiles: correctly handle newly added largefile on other side of merge
Na'Tosha Bard <natosha@unity3d.com>
parents:
15860
diff
changeset
|
411 os.path.exists(repo.wjoin(lfutil.standin(lfile)))): |
|
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
412 expectedhash = lfutil.readstandin(repo, lfile) |
|
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
413 else: |
|
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
414 expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
|
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
415 |
| 15168 | 416 # if it exists and its hash matches, it might have been locally |
| 417 # modified before updating and the user chose 'local'. in this case, | |
| 418 # it will not be in any store, so don't look for it. | |
|
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
419 if ((not os.path.exists(repo.wjoin(lfile)) or |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
420 expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
421 not lfutil.findfile(repo, expectedhash)): |
| 15168 | 422 toget.append((lfile, expectedhash)) |
| 423 | |
| 424 if toget: | |
| 425 store = basestore._openstore(repo) | |
| 426 ret = store.get(toget) | |
| 427 return ret | |
| 428 | |
| 429 return ([], []) | |
| 430 | |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
431 def downloadlfiles(ui, repo, rev=None): |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
432 matchfn = scmutil.match(repo[None], |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
433 [repo.wjoin(lfutil.shortname)], {}) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
434 def prepare(ctx, fns): |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
435 pass |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
436 totalsuccess = 0 |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
437 totalmissing = 0 |
|
18722
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
438 if rev != []: # walkchangerevs on empty list would return all revs |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
439 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev}, |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
440 prepare): |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
441 success, missing = cachelfiles(ui, repo, ctx.node()) |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
442 totalsuccess += len(success) |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
443 totalmissing += len(missing) |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
444 ui.status(_("%d additional largefiles cached\n") % totalsuccess) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
445 if totalmissing > 0: |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
446 ui.status(_("%d largefiles failed to download\n") % totalmissing) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
447 return totalsuccess, totalmissing |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
448 |
| 15168 | 449 def updatelfiles(ui, repo, filelist=None, printmessage=True): |
| 450 wlock = repo.wlock() | |
| 451 try: | |
| 452 lfdirstate = lfutil.openlfdirstate(ui, repo) | |
| 453 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) | |
| 454 | |
| 455 if filelist is not None: | |
| 456 lfiles = [f for f in lfiles if f in filelist] | |
| 457 | |
| 458 printed = False | |
| 459 if printmessage and lfiles: | |
| 460 ui.status(_('getting changed largefiles\n')) | |
| 461 printed = True | |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
462 cachelfiles(ui, repo, '.', lfiles) |
| 15168 | 463 |
| 464 updated, removed = 0, 0 | |
|
17430
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
465 for f in lfiles: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
466 i = _updatelfile(repo, lfdirstate, f) |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
467 if i: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
468 if i > 0: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
469 updated += i |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
470 else: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
471 removed -= i |
| 15168 | 472 if printmessage and (removed or updated) and not printed: |
| 473 ui.status(_('getting changed largefiles\n')) | |
| 474 printed = True | |
| 475 | |
| 476 lfdirstate.write() | |
| 477 if printed and printmessage: | |
| 478 ui.status(_('%d largefiles updated, %d removed\n') % (updated, | |
| 479 removed)) | |
| 480 finally: | |
| 481 wlock.release() | |
| 482 | |
| 483 def _updatelfile(repo, lfdirstate, lfile): | |
| 484 '''updates a single largefile and copies the state of its standin from | |
| 485 the repository's dirstate to its state in the lfdirstate. | |
| 486 | |
| 487 returns 1 if the file was modified, -1 if the file was removed, 0 if the | |
| 488 file was unchanged, and None if the needed largefile was missing from the | |
| 489 cache.''' | |
| 490 ret = 0 | |
| 491 abslfile = repo.wjoin(lfile) | |
| 492 absstandin = repo.wjoin(lfutil.standin(lfile)) | |
| 493 if os.path.exists(absstandin): | |
|
18292
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
494 if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile): |
|
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
495 shutil.copyfile(abslfile, abslfile + '.orig') |
| 15168 | 496 expecthash = lfutil.readstandin(repo, lfile) |
|
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
497 if (expecthash != '' and |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
498 (not os.path.exists(abslfile) or |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
499 expecthash != lfutil.hashfile(abslfile))): |
| 15168 | 500 if not lfutil.copyfromcache(repo, expecthash, lfile): |
|
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
501 # use normallookup() to allocate entry in largefiles dirstate, |
|
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
502 # because lack of it misleads lfilesrepo.status() into |
|
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
503 # recognition that such cache missing files are REMOVED. |
|
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
504 lfdirstate.normallookup(lfile) |
|
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
505 return None # don't try to set the mode |
|
17155
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
506 else: |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
507 # Synchronize largefile dirstate to the last modified time of |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
508 # the file |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
509 lfdirstate.normal(lfile) |
| 15168 | 510 ret = 1 |
| 511 mode = os.stat(absstandin).st_mode | |
| 512 if mode != os.stat(abslfile).st_mode: | |
| 513 os.chmod(abslfile, mode) | |
| 514 ret = 1 | |
| 515 else: | |
|
15663
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
516 # Remove lfiles for which the standin is deleted, unless the |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
517 # lfile is added to the repository again. This happens when a |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
518 # largefile is converted back to a normal file: the standin |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
519 # disappears, but a new (normal) file appears as the lfile. |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
520 if os.path.exists(abslfile) and lfile not in repo[None]: |
|
15900
29defa7d20f6
largefiles: remove empty directories upon update (issue3202)
Patrick Mezard <pmezard@gmail.com>
parents:
15663
diff
changeset
|
521 util.unlinkpath(abslfile) |
| 15168 | 522 ret = -1 |
| 523 state = repo.dirstate[lfutil.standin(lfile)] | |
| 524 if state == 'n': | |
|
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
525 # When rebasing, we need to synchronize the standin and the largefile, |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
526 # because otherwise the largefile will get reverted. But for commit's |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
527 # sake, we have to mark the file as unclean. |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
528 if getattr(repo, "_isrebasing", False): |
|
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
529 lfdirstate.normallookup(lfile) |
|
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
530 else: |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
531 lfdirstate.normal(lfile) |
| 15168 | 532 elif state == 'r': |
| 533 lfdirstate.remove(lfile) | |
| 534 elif state == 'a': | |
| 535 lfdirstate.add(lfile) | |
| 536 elif state == '?': | |
|
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15173
diff
changeset
|
537 lfdirstate.drop(lfile) |
| 15168 | 538 return ret |
| 539 | |
|
16439
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
540 def catlfile(repo, lfile, rev, filename): |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
541 hash = lfutil.readstandin(repo, lfile, rev) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
542 if not lfutil.inusercache(repo.ui, hash): |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
543 store = basestore._openstore(repo) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
544 success, missing = store.get([(lfile, hash)]) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
545 if len(success) != 1: |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
546 raise util.Abort( |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
547 _('largefile %s is not in cache and could not be downloaded') |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
548 % lfile) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
549 path = lfutil.usercachepath(repo.ui, hash) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
550 fpout = cmdutil.makefileobj(repo, filename) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
551 fpin = open(path, "rb") |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
552 fpout.write(fpin.read()) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
553 fpout.close() |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
554 fpin.close() |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
555 return 0 |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
556 |
| 15168 | 557 # -- hg commands declarations ------------------------------------------------ |
| 558 | |
| 559 cmdtable = { | |
| 560 'lfconvert': (lfconvert, | |
| 15230 | 561 [('s', 'size', '', |
| 562 _('minimum size (MB) for files to be converted ' | |
| 563 'as largefiles'), | |
| 564 'SIZE'), | |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
565 ('', 'to-normal', False, |
| 15230 | 566 _('convert from a largefiles repo to a normal repo')), |
| 567 ], | |
| 15168 | 568 _('hg lfconvert SOURCE DEST [FILE ...]')), |
| 569 } | |
|
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
570 |
|
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
571 commands.inferrepo += " lfconvert" |