Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/lfcommands.py @ 18728:1e636f7b1cfe stable
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
cachelfiles jumped through loops to handle merges and modified files ... but it
did apparently no longer have a valid reason to do so. It should just always
make sure that the largefiles referenced from the standins are present - no
matter which actual largefile is stored in the working directory. If there is
no standin then there is nothing to fetch.
The old code usually verified the hash of all largefiles every time this
function was invoked - for examply by 'update'.
This change makes a trivial noop update 5-10 seconds faster on our repo (with
the other 50% spent doing another unnecessary hashing of all largefiles).
| author | Mads Kiilerich <madski@unity3d.com> |
|---|---|
| date | Thu, 28 Feb 2013 13:45:18 +0100 |
| parents | 4846f2115927 |
| children | 4e53ac3f466a |
| rev | line source |
|---|---|
| 15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
| 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
| 3 # Copyright 2010-2011 Fog Creek Software | |
| 4 # Copyright 2010-2011 Unity Technologies | |
| 5 # | |
| 6 # This software may be used and distributed according to the terms of the | |
| 7 # GNU General Public License version 2 or any later version. | |
| 8 | |
|
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
9 '''High-level command function for lfconvert, plus the cmdtable.''' |
| 15168 | 10 |
|
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
11 import os, errno |
| 15168 | 12 import shutil |
| 13 | |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
14 from mercurial import util, match as match_, hg, node, context, error, \ |
|
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
15 cmdutil, scmutil, commands |
| 15168 | 16 from mercurial.i18n import _ |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
17 from mercurial.lock import release |
| 15168 | 18 |
| 19 import lfutil | |
| 20 import basestore | |
| 21 | |
| 22 # -- Commands ---------------------------------------------------------- | |
| 23 | |
| 24 def lfconvert(ui, src, dest, *pats, **opts): | |
| 15230 | 25 '''convert a normal repository to a largefiles repository |
| 15168 | 26 |
| 15230 | 27 Convert repository SOURCE to a new repository DEST, identical to |
| 28 SOURCE except that certain files will be converted as largefiles: | |
| 29 specifically, any file that matches any PATTERN *or* whose size is | |
| 30 above the minimum size threshold is converted as a largefile. The | |
| 31 size used to determine whether or not to track a file as a | |
| 32 largefile is the size of the first version of the file. The | |
| 33 minimum size can be specified either with --size or in | |
| 34 configuration as ``largefiles.size``. | |
| 35 | |
| 36 After running this command you will need to make sure that | |
| 37 largefiles is enabled anywhere you intend to push the new | |
| 38 repository. | |
| 39 | |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
40 Use --to-normal to convert largefiles back to normal files; after |
| 15230 | 41 this, the DEST repository can be used without largefiles at all.''' |
| 15168 | 42 |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
43 if opts['to_normal']: |
| 15168 | 44 tolfile = False |
| 45 else: | |
| 46 tolfile = True | |
|
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15224
diff
changeset
|
47 size = lfutil.getminsize(ui, True, opts.get('size'), default=None) |
|
15340
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
48 |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
49 if not hg.islocal(src): |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
50 raise util.Abort(_('%s is not a local Mercurial repo') % src) |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
51 if not hg.islocal(dest): |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
52 raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
|
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
53 |
|
15339
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
54 rsrc = hg.repository(ui, src) |
|
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
55 ui.status(_('initializing destination %s\n') % dest) |
|
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
56 rdst = hg.repository(ui, dest, create=True) |
| 15168 | 57 |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
58 success = False |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
59 dstwlock = dstlock = None |
| 15168 | 60 try: |
| 61 # Lock destination to prevent modification while it is converted to. | |
| 62 # Don't need to lock src because we are just reading from its history | |
| 63 # which can't change. | |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
64 dstwlock = rdst.wlock() |
|
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
65 dstlock = rdst.lock() |
| 15168 | 66 |
| 67 # Get a list of all changesets in the source. The easy way to do this | |
|
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
68 # is to simply walk the changelog, using changelog.nodesbetween(). |
| 15168 | 69 # Take a look at mercurial/revlog.py:639 for more details. |
| 70 # Use a generator instead of a list to decrease memory usage | |
| 71 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, | |
| 72 rsrc.heads())[0]) | |
| 73 revmap = {node.nullid: node.nullid} | |
| 74 if tolfile: | |
| 75 lfiles = set() | |
| 76 normalfiles = set() | |
| 77 if not pats: | |
|
15579
6c5e6ebe0812
largefiles: use "ui.configlist()" to get largefiles.patterns configuration
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15472
diff
changeset
|
78 pats = ui.configlist(lfutil.longname, 'patterns', default=[]) |
| 15168 | 79 if pats: |
| 80 matcher = match_.match(rsrc.root, '', list(pats)) | |
| 81 else: | |
| 82 matcher = None | |
| 83 | |
| 84 lfiletohash = {} | |
| 85 for ctx in ctxs: | |
| 86 ui.progress(_('converting revisions'), ctx.rev(), | |
| 87 unit=_('revision'), total=rsrc['tip'].rev()) | |
| 88 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, | |
| 89 lfiles, normalfiles, matcher, size, lfiletohash) | |
| 90 ui.progress(_('converting revisions'), None) | |
| 91 | |
| 92 if os.path.exists(rdst.wjoin(lfutil.shortname)): | |
| 93 shutil.rmtree(rdst.wjoin(lfutil.shortname)) | |
| 94 | |
| 95 for f in lfiletohash.keys(): | |
| 96 if os.path.isfile(rdst.wjoin(f)): | |
| 97 os.unlink(rdst.wjoin(f)) | |
| 98 try: | |
| 99 os.removedirs(os.path.dirname(rdst.wjoin(f))) | |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
100 except OSError: |
| 15168 | 101 pass |
| 102 | |
|
15303
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
103 # If there were any files converted to largefiles, add largefiles |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
104 # to the destination repository's requirements. |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
105 if lfiles: |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
106 rdst.requirements.add('largefiles') |
|
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
107 rdst._writerequirements() |
| 15168 | 108 else: |
| 109 for ctx in ctxs: | |
| 110 ui.progress(_('converting revisions'), ctx.rev(), | |
| 111 unit=_('revision'), total=rsrc['tip'].rev()) | |
| 112 _addchangeset(ui, rsrc, rdst, ctx, revmap) | |
| 113 | |
| 114 ui.progress(_('converting revisions'), None) | |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
115 success = True |
| 15168 | 116 finally: |
|
16717
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
117 rdst.dirstate.clear() |
|
1eede2ea2041
largefiles: use wlock for lfconvert (issue3444)
Mads Kiilerich <mads@kiilerich.com>
parents:
16551
diff
changeset
|
118 release(dstlock, dstwlock) |
|
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
119 if not success: |
|
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
120 # we failed, remove the new directory |
|
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
121 shutil.rmtree(rdst.root) |
| 15168 | 122 |
| 123 def _addchangeset(ui, rsrc, rdst, ctx, revmap): | |
|
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
124 # Convert src parents to dst parents |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
125 parents = _convertparents(ctx, revmap) |
| 15168 | 126 |
| 127 # Generate list of changed files | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
128 files = _getchangedfiles(ctx, parents) |
| 15168 | 129 |
| 130 def getfilectx(repo, memctx, f): | |
| 131 if lfutil.standin(f) in files: | |
| 132 # if the file isn't in the manifest then it was removed | |
| 133 # or renamed, raise IOError to indicate this | |
| 134 try: | |
| 135 fctx = ctx.filectx(lfutil.standin(f)) | |
| 136 except error.LookupError: | |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
137 raise IOError |
| 15168 | 138 renamed = fctx.renamed() |
| 139 if renamed: | |
| 140 renamed = lfutil.splitstandin(renamed[0]) | |
| 141 | |
| 142 hash = fctx.data().strip() | |
| 143 path = lfutil.findfile(rsrc, hash) | |
|
17823
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
144 |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
145 # If one file is missing, likely all files from this rev are |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
146 if path is None: |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
147 cachelfiles(ui, rsrc, ctx.node()) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
148 path = lfutil.findfile(rsrc, hash) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
149 |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
150 if path is None: |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
151 raise util.Abort( |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
152 _("missing largefile \'%s\' from revision %s") |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
153 % (f, node.hex(ctx.node()))) |
|
0fc1ce271ee6
largefiles: fix a traceback in lfconvert if a largefile is missing (issue3519)
Matt Harbison <matt_harbison@yahoo.com>
parents:
17773
diff
changeset
|
154 |
| 15168 | 155 data = '' |
| 156 fd = None | |
| 157 try: | |
| 158 fd = open(path, 'rb') | |
| 159 data = fd.read() | |
| 160 finally: | |
|
15172
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
161 if fd: |
|
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
162 fd.close() |
| 15168 | 163 return context.memfilectx(f, data, 'l' in fctx.flags(), |
| 164 'x' in fctx.flags(), renamed) | |
| 165 else: | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
166 return _getnormalcontext(repo.ui, ctx, f, revmap) |
| 15168 | 167 |
| 168 dstfiles = [] | |
| 169 for file in files: | |
| 170 if lfutil.isstandin(file): | |
| 171 dstfiles.append(lfutil.splitstandin(file)) | |
| 172 else: | |
| 173 dstfiles.append(file) | |
| 174 # Commit | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
175 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
| 15168 | 176 |
| 177 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, | |
| 178 matcher, size, lfiletohash): | |
| 179 # Convert src parents to dst parents | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
180 parents = _convertparents(ctx, revmap) |
| 15168 | 181 |
| 182 # Generate list of changed files | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
183 files = _getchangedfiles(ctx, parents) |
| 15168 | 184 |
| 185 dstfiles = [] | |
| 186 for f in files: | |
| 187 if f not in lfiles and f not in normalfiles: | |
| 188 islfile = _islfile(f, ctx, matcher, size) | |
| 189 # If this file was renamed or copied then copy | |
|
17424
e7cfe3587ea4
fix trivial spelling errors
Mads Kiilerich <mads@kiilerich.com>
parents:
17299
diff
changeset
|
190 # the largefile-ness of its predecessor |
| 15168 | 191 if f in ctx.manifest(): |
| 192 fctx = ctx.filectx(f) | |
| 193 renamed = fctx.renamed() | |
| 194 renamedlfile = renamed and renamed[0] in lfiles | |
| 195 islfile |= renamedlfile | |
| 196 if 'l' in fctx.flags(): | |
| 197 if renamedlfile: | |
| 198 raise util.Abort( | |
|
15380
a53888685a6c
largefiles: fix uppercase in abort message
Martin Geisler <mg@aragost.com>
parents:
15371
diff
changeset
|
199 _('renamed/copied largefile %s becomes symlink') |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
200 % f) |
| 15168 | 201 islfile = False |
| 202 if islfile: | |
| 203 lfiles.add(f) | |
| 204 else: | |
| 205 normalfiles.add(f) | |
| 206 | |
| 207 if f in lfiles: | |
| 208 dstfiles.append(lfutil.standin(f)) | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
209 # largefile in manifest if it has not been removed/renamed |
| 15168 | 210 if f in ctx.manifest(): |
|
15808
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
211 fctx = ctx.filectx(f) |
|
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
212 if 'l' in fctx.flags(): |
|
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
213 renamed = fctx.renamed() |
| 15168 | 214 if renamed and renamed[0] in lfiles: |
| 215 raise util.Abort(_('largefile %s becomes symlink') % f) | |
| 216 | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
217 # largefile was modified, update standins |
| 15168 | 218 fullpath = rdst.wjoin(f) |
|
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15340
diff
changeset
|
219 util.makedirs(os.path.dirname(fullpath)) |
| 15168 | 220 m = util.sha1('') |
| 221 m.update(ctx[f].data()) | |
| 222 hash = m.hexdigest() | |
| 223 if f not in lfiletohash or lfiletohash[f] != hash: | |
| 224 try: | |
| 225 fd = open(fullpath, 'wb') | |
| 226 fd.write(ctx[f].data()) | |
| 227 finally: | |
| 228 if fd: | |
| 229 fd.close() | |
| 230 executable = 'x' in ctx[f].flags() | |
| 231 os.chmod(fullpath, lfutil.getmode(executable)) | |
| 232 lfutil.writestandin(rdst, lfutil.standin(f), hash, | |
| 233 executable) | |
| 234 lfiletohash[f] = hash | |
| 235 else: | |
| 236 # normal file | |
| 237 dstfiles.append(f) | |
| 238 | |
| 239 def getfilectx(repo, memctx, f): | |
| 240 if lfutil.isstandin(f): | |
| 241 # if the file isn't in the manifest then it was removed | |
| 242 # or renamed, raise IOError to indicate this | |
| 243 srcfname = lfutil.splitstandin(f) | |
| 244 try: | |
| 245 fctx = ctx.filectx(srcfname) | |
| 246 except error.LookupError: | |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
247 raise IOError |
| 15168 | 248 renamed = fctx.renamed() |
| 249 if renamed: | |
|
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
250 # standin is always a largefile because largefile-ness |
| 15168 | 251 # doesn't change after rename or copy |
| 252 renamed = lfutil.standin(renamed[0]) | |
| 253 | |
|
15313
3eb1a90ea409
largefiles: fix newline for lfconverted repos
Eli Carter <eli.carter@tektronix.com>
parents:
15303
diff
changeset
|
254 return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in |
| 15168 | 255 fctx.flags(), 'x' in fctx.flags(), renamed) |
| 256 else: | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
257 return _getnormalcontext(repo.ui, ctx, f, revmap) |
| 15168 | 258 |
| 259 # Commit | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
260 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
261 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
262 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap): |
| 15168 | 263 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
| 264 getfilectx, ctx.user(), ctx.date(), ctx.extra()) | |
| 265 ret = rdst.commitctx(mctx) | |
|
16551
ebf6d38c9063
localrepo: add setparents() to adjust dirstate copies (issue3407)
Patrick Mezard <patrick@mezard.eu>
parents:
16439
diff
changeset
|
266 rdst.setparents(ret) |
| 15168 | 267 revmap[ctx.node()] = rdst.changelog.tip() |
| 268 | |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
269 # Generate list of changed files |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
270 def _getchangedfiles(ctx, parents): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
271 files = set(ctx.files()) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
272 if node.nullid not in parents: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
273 mc = ctx.manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
274 mp1 = ctx.parents()[0].manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
275 mp2 = ctx.parents()[1].manifest() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
276 files |= (set(mp1) | set(mp2)) - set(mc) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
277 for f in mc: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
278 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
279 files.add(f) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
280 return files |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
281 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
282 # Convert src parents to dst parents |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
283 def _convertparents(ctx, revmap): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
284 parents = [] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
285 for p in ctx.parents(): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
286 parents.append(revmap[p.node()]) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
287 while len(parents) < 2: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
288 parents.append(node.nullid) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
289 return parents |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
290 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
291 # Get memfilectx for a normal file |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
292 def _getnormalcontext(ui, ctx, f, revmap): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
293 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
294 fctx = ctx.filectx(f) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
295 except error.LookupError: |
|
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
296 raise IOError |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
297 renamed = fctx.renamed() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
298 if renamed: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
299 renamed = renamed[0] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
300 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
301 data = fctx.data() |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
302 if f == '.hgtags': |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
303 data = _converttags (ui, revmap, data) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
304 return context.memfilectx(f, data, 'l' in fctx.flags(), |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
305 'x' in fctx.flags(), renamed) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
306 |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
307 # Remap tag data using a revision map |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
308 def _converttags(ui, revmap, data): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
309 newdata = [] |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
310 for line in data.splitlines(): |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
311 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
312 id, name = line.split(' ', 1) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
313 except ValueError: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
314 ui.warn(_('skipping incorrectly formatted tag %s\n' |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
315 % line)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
316 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
317 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
318 newid = node.bin(id) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
319 except TypeError: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
320 ui.warn(_('skipping incorrectly formatted id %s\n' |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
321 % id)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
322 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
323 try: |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
324 newdata.append('%s %s\n' % (node.hex(revmap[newid]), |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
325 name)) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
326 except KeyError: |
|
16231
ce292f1379ba
i18n: fix all remaining uses of % inside _()
Matt Mackall <mpm@selenic.com>
parents:
15909
diff
changeset
|
327 ui.warn(_('no mapping for id %s\n') % id) |
|
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
328 continue |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
329 return ''.join(newdata) |
|
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
330 |
| 15168 | 331 def _islfile(file, ctx, matcher, size): |
|
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
332 '''Return true if file should be considered a largefile, i.e. |
|
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
333 matcher matches it or it is larger than size.''' |
|
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
334 # never store special .hg* files as largefiles |
| 15168 | 335 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
| 336 return False | |
| 337 if matcher and matcher(file): | |
| 338 return True | |
| 339 try: | |
| 340 return ctx.filectx(file).size() >= size * 1024 * 1024 | |
| 341 except error.LookupError: | |
| 342 return False | |
| 343 | |
| 344 def uploadlfiles(ui, rsrc, rdst, files): | |
| 345 '''upload largefiles to the central store''' | |
| 346 | |
|
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15313
diff
changeset
|
347 if not files: |
| 15168 | 348 return |
| 349 | |
| 350 store = basestore._openstore(rsrc, rdst, put=True) | |
| 351 | |
| 352 at = 0 | |
|
17127
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
353 ui.debug("sending statlfile command for %d largefiles\n" % len(files)) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
354 retval = store.exists(files) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
355 files = filter(lambda h: not retval[h], files) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
356 ui.debug("%d largefiles need to be uploaded\n" % len(files)) |
|
9e1616307c4c
largefiles: batch statlfile requests when pushing a largefiles repo (issue3386)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16727
diff
changeset
|
357 |
| 15168 | 358 for hash in files: |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
359 ui.progress(_('uploading largefiles'), at, unit='largefile', |
|
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
360 total=len(files)) |
| 15168 | 361 source = lfutil.findfile(rsrc, hash) |
| 362 if not source: | |
|
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
363 raise util.Abort(_('largefile %s missing from store' |
|
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
364 ' (needs to be uploaded)') % hash) |
| 15168 | 365 # XXX check for errors here |
| 366 store.put(source, hash) | |
| 367 at += 1 | |
|
15173
3d27a8ff895f
largefiles: mark a string for translation
Matt Mackall <mpm@selenic.com>
parents:
15172
diff
changeset
|
368 ui.progress(_('uploading largefiles'), None) |
| 15168 | 369 |
| 370 def verifylfiles(ui, repo, all=False, contents=False): | |
| 371 '''Verify that every big file revision in the current changeset | |
| 372 exists in the central store. With --contents, also verify that | |
| 373 the contents of each big file revision are correct (SHA-1 hash | |
| 374 matches the revision ID). With --all, check every changeset in | |
| 375 this repository.''' | |
| 376 if all: | |
| 377 # Pass a list to the function rather than an iterator because we know a | |
| 378 # list will work. | |
| 379 revs = range(len(repo)) | |
| 380 else: | |
| 381 revs = ['.'] | |
| 382 | |
| 383 store = basestore._openstore(repo) | |
| 384 return store.verify(revs, contents=contents) | |
| 385 | |
|
18144
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
386 def debugdirstate(ui, repo): |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
387 '''Show basic information for the largefiles dirstate''' |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
388 lfdirstate = lfutil.openlfdirstate(ui, repo) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
389 for file_, ent in sorted(lfdirstate._map.iteritems()): |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
390 mode = '%3o' % (ent[1] & 0777 & ~util.umask) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
391 ui.write("%c %s %10d %s\n" % (ent[0], mode, ent[2], file_)) |
|
e16982a74bf7
largefiles: introduce basic debugstate --large functionality
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
392 |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
393 def cachelfiles(ui, repo, node, filelist=None): |
| 15168 | 394 '''cachelfiles ensures that all largefiles needed by the specified revision |
| 395 are present in the repository's largefile cache. | |
| 396 | |
| 397 returns a tuple (cached, missing). cached is the list of files downloaded | |
| 398 by this operation; missing is the list of files that were needed but could | |
| 399 not be found.''' | |
| 400 lfiles = lfutil.listlfiles(repo, node) | |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
401 if filelist: |
|
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
402 lfiles = set(lfiles) & set(filelist) |
| 15168 | 403 toget = [] |
| 404 | |
| 405 for lfile in lfiles: | |
|
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
406 try: |
|
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
407 expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
|
18728
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
408 except IOError, err: |
|
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
409 if err.errno == errno.ENOENT: |
|
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
410 continue # node must be None and standin wasn't found in wctx |
|
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
411 raise |
|
1e636f7b1cfe
largefiles: simplify cachelfiles - don't spend a lot of time checking hashes
Mads Kiilerich <madski@unity3d.com>
parents:
18727
diff
changeset
|
412 if not lfutil.findfile(repo, expectedhash): |
| 15168 | 413 toget.append((lfile, expectedhash)) |
| 414 | |
| 415 if toget: | |
| 416 store = basestore._openstore(repo) | |
| 417 ret = store.get(toget) | |
| 418 return ret | |
| 419 | |
| 420 return ([], []) | |
| 421 | |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
422 def downloadlfiles(ui, repo, rev=None): |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
423 matchfn = scmutil.match(repo[None], |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
424 [repo.wjoin(lfutil.shortname)], {}) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
425 def prepare(ctx, fns): |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
426 pass |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
427 totalsuccess = 0 |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
428 totalmissing = 0 |
|
18722
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
429 if rev != []: # walkchangerevs on empty list would return all revs |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
430 for ctx in cmdutil.walkchangerevs(repo, matchfn, {'rev' : rev}, |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
431 prepare): |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
432 success, missing = cachelfiles(ui, repo, ctx.node()) |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
433 totalsuccess += len(success) |
|
f0aa8bbffe60
largefiles: fix download of largefiles from an empty list of changesets
Mads Kiilerich <madski@unity3d.com>
parents:
18294
diff
changeset
|
434 totalmissing += len(missing) |
|
16691
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
435 ui.status(_("%d additional largefiles cached\n") % totalsuccess) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
436 if totalmissing > 0: |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
437 ui.status(_("%d largefiles failed to download\n") % totalmissing) |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
438 return totalsuccess, totalmissing |
|
7d6a660ca151
largefiles: refactor downloading of all largefiles to generic function
Na'Tosha Bard <natosha@unity3d.com>
parents:
16687
diff
changeset
|
439 |
| 15168 | 440 def updatelfiles(ui, repo, filelist=None, printmessage=True): |
| 441 wlock = repo.wlock() | |
| 442 try: | |
| 443 lfdirstate = lfutil.openlfdirstate(ui, repo) | |
| 444 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) | |
| 445 | |
| 446 if filelist is not None: | |
| 447 lfiles = [f for f in lfiles if f in filelist] | |
| 448 | |
| 449 printed = False | |
| 450 if printmessage and lfiles: | |
| 451 ui.status(_('getting changed largefiles\n')) | |
| 452 printed = True | |
|
16700
28001e8a5149
largefiles: optimize performance when updating (issue3440)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16691
diff
changeset
|
453 cachelfiles(ui, repo, '.', lfiles) |
| 15168 | 454 |
| 455 updated, removed = 0, 0 | |
|
17430
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
456 for f in lfiles: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
457 i = _updatelfile(repo, lfdirstate, f) |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
458 if i: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
459 if i > 0: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
460 updated += i |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
461 else: |
|
e841ecc03765
largefiles: minor code cleanup
Mads Kiilerich <mads@kiilerich.com>
parents:
17424
diff
changeset
|
462 removed -= i |
| 15168 | 463 if printmessage and (removed or updated) and not printed: |
| 464 ui.status(_('getting changed largefiles\n')) | |
| 465 printed = True | |
| 466 | |
| 467 lfdirstate.write() | |
| 468 if printed and printmessage: | |
| 469 ui.status(_('%d largefiles updated, %d removed\n') % (updated, | |
| 470 removed)) | |
| 471 finally: | |
| 472 wlock.release() | |
| 473 | |
| 474 def _updatelfile(repo, lfdirstate, lfile): | |
| 475 '''updates a single largefile and copies the state of its standin from | |
| 476 the repository's dirstate to its state in the lfdirstate. | |
| 477 | |
| 478 returns 1 if the file was modified, -1 if the file was removed, 0 if the | |
| 479 file was unchanged, and None if the needed largefile was missing from the | |
| 480 cache.''' | |
| 481 ret = 0 | |
| 482 abslfile = repo.wjoin(lfile) | |
| 483 absstandin = repo.wjoin(lfutil.standin(lfile)) | |
| 484 if os.path.exists(absstandin): | |
|
18292
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
485 if os.path.exists(absstandin + '.orig') and os.path.exists(abslfile): |
|
40185df018d7
largefiles: make update with backup files in .hglf slightly less broken
Mads Kiilerich <madski@unity3d.com>
parents:
17823
diff
changeset
|
486 shutil.copyfile(abslfile, abslfile + '.orig') |
| 15168 | 487 expecthash = lfutil.readstandin(repo, lfile) |
|
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
488 if (expecthash != '' and |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
489 (not os.path.exists(abslfile) or |
|
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
490 expecthash != lfutil.hashfile(abslfile))): |
| 15168 | 491 if not lfutil.copyfromcache(repo, expecthash, lfile): |
|
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
492 # use normallookup() to allocate entry in largefiles dirstate, |
|
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
493 # because lack of it misleads lfilesrepo.status() into |
|
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
494 # recognition that such cache missing files are REMOVED. |
|
18727
4846f2115927
largefiles: don't let update leave wrong largefiles in wd if fetch fails
Mads Kiilerich <madski@unity3d.com>
parents:
18722
diff
changeset
|
495 if lfile not in repo[None]: # not switched to normal file |
|
4846f2115927
largefiles: don't let update leave wrong largefiles in wd if fetch fails
Mads Kiilerich <madski@unity3d.com>
parents:
18722
diff
changeset
|
496 util.unlinkpath(abslfile, ignoremissing=True) |
|
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
497 lfdirstate.normallookup(lfile) |
|
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
498 return None # don't try to set the mode |
|
17155
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
499 else: |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
500 # Synchronize largefile dirstate to the last modified time of |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
501 # the file |
|
88ff28bcd980
largefiles: optimize status by synchronizing lfdirstate with the largefile on update
Na'Tosha Bard <natosha@unity3d.com>
parents:
17127
diff
changeset
|
502 lfdirstate.normal(lfile) |
| 15168 | 503 ret = 1 |
| 504 mode = os.stat(absstandin).st_mode | |
| 505 if mode != os.stat(abslfile).st_mode: | |
| 506 os.chmod(abslfile, mode) | |
| 507 ret = 1 | |
| 508 else: | |
|
15663
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
509 # Remove lfiles for which the standin is deleted, unless the |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
510 # lfile is added to the repository again. This happens when a |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
511 # largefile is converted back to a normal file: the standin |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
512 # disappears, but a new (normal) file appears as the lfile. |
|
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
513 if os.path.exists(abslfile) and lfile not in repo[None]: |
|
15900
29defa7d20f6
largefiles: remove empty directories upon update (issue3202)
Patrick Mezard <pmezard@gmail.com>
parents:
15663
diff
changeset
|
514 util.unlinkpath(abslfile) |
| 15168 | 515 ret = -1 |
| 516 state = repo.dirstate[lfutil.standin(lfile)] | |
| 517 if state == 'n': | |
|
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
518 # When rebasing, we need to synchronize the standin and the largefile, |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
519 # because otherwise the largefile will get reverted. But for commit's |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
520 # sake, we have to mark the file as unclean. |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
521 if getattr(repo, "_isrebasing", False): |
|
17299
e51d4aedace9
check-code: indent 4 spaces in py files
Mads Kiilerich <mads@kiilerich.com>
parents:
17155
diff
changeset
|
522 lfdirstate.normallookup(lfile) |
|
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
523 else: |
|
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
524 lfdirstate.normal(lfile) |
| 15168 | 525 elif state == 'r': |
| 526 lfdirstate.remove(lfile) | |
| 527 elif state == 'a': | |
| 528 lfdirstate.add(lfile) | |
| 529 elif state == '?': | |
|
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15173
diff
changeset
|
530 lfdirstate.drop(lfile) |
| 15168 | 531 return ret |
| 532 | |
|
16439
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
533 def catlfile(repo, lfile, rev, filename): |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
534 hash = lfutil.readstandin(repo, lfile, rev) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
535 if not lfutil.inusercache(repo.ui, hash): |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
536 store = basestore._openstore(repo) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
537 success, missing = store.get([(lfile, hash)]) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
538 if len(success) != 1: |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
539 raise util.Abort( |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
540 _('largefile %s is not in cache and could not be downloaded') |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
541 % lfile) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
542 path = lfutil.usercachepath(repo.ui, hash) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
543 fpout = cmdutil.makefileobj(repo, filename) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
544 fpin = open(path, "rb") |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
545 fpout.write(fpin.read()) |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
546 fpout.close() |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
547 fpin.close() |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
548 return 0 |
|
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
549 |
| 15168 | 550 # -- hg commands declarations ------------------------------------------------ |
| 551 | |
| 552 cmdtable = { | |
| 553 'lfconvert': (lfconvert, | |
| 15230 | 554 [('s', 'size', '', |
| 555 _('minimum size (MB) for files to be converted ' | |
| 556 'as largefiles'), | |
| 557 'SIZE'), | |
|
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
558 ('', 'to-normal', False, |
| 15230 | 559 _('convert from a largefiles repo to a normal repo')), |
| 560 ], | |
| 15168 | 561 _('hg lfconvert SOURCE DEST [FILE ...]')), |
| 562 } | |
|
17773
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
563 |
|
434e5bd615fc
commands: don't infer repo for commands like update (issue2748)
Siddharth Agarwal <sid0@fb.com>
parents:
17537
diff
changeset
|
564 commands.inferrepo += " lfconvert" |