Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/lfcommands.py @ 16687:e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
author | Brodie Rao <brodie@sf.io> |
---|---|
date | Sat, 12 May 2012 16:00:58 +0200 |
parents | ebf6d38c9063 |
children | 7d6a660ca151 |
rev | line source |
---|---|
15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
3 # Copyright 2010-2011 Fog Creek Software | |
4 # Copyright 2010-2011 Unity Technologies | |
5 # | |
6 # This software may be used and distributed according to the terms of the | |
7 # GNU General Public License version 2 or any later version. | |
8 | |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
9 '''High-level command function for lfconvert, plus the cmdtable.''' |
15168 | 10 |
11 import os | |
12 import shutil | |
13 | |
16439
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
14 from mercurial import util, match as match_, hg, node, context, error, cmdutil |
15168 | 15 from mercurial.i18n import _ |
16 | |
17 import lfutil | |
18 import basestore | |
19 | |
20 # -- Commands ---------------------------------------------------------- | |
21 | |
22 def lfconvert(ui, src, dest, *pats, **opts): | |
15230 | 23 '''convert a normal repository to a largefiles repository |
15168 | 24 |
15230 | 25 Convert repository SOURCE to a new repository DEST, identical to |
26 SOURCE except that certain files will be converted as largefiles: | |
27 specifically, any file that matches any PATTERN *or* whose size is | |
28 above the minimum size threshold is converted as a largefile. The | |
29 size used to determine whether or not to track a file as a | |
30 largefile is the size of the first version of the file. The | |
31 minimum size can be specified either with --size or in | |
32 configuration as ``largefiles.size``. | |
33 | |
34 After running this command you will need to make sure that | |
35 largefiles is enabled anywhere you intend to push the new | |
36 repository. | |
37 | |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
38 Use --to-normal to convert largefiles back to normal files; after |
15230 | 39 this, the DEST repository can be used without largefiles at all.''' |
15168 | 40 |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
41 if opts['to_normal']: |
15168 | 42 tolfile = False |
43 else: | |
44 tolfile = True | |
15227
a7686abf73a6
largefiles: factor out lfutil.getminsize()
Greg Ward <greg@gerg.ca>
parents:
15224
diff
changeset
|
45 size = lfutil.getminsize(ui, True, opts.get('size'), default=None) |
15340
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
46 |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
47 if not hg.islocal(src): |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
48 raise util.Abort(_('%s is not a local Mercurial repo') % src) |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
49 if not hg.islocal(dest): |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
50 raise util.Abort(_('%s is not a local Mercurial repo') % dest) |
0e58513cc59a
largefiles: rearrange how lfconvert detects non-local repos
Greg Ward <greg@gerg.ca>
parents:
15339
diff
changeset
|
51 |
15339
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
52 rsrc = hg.repository(ui, src) |
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
53 ui.status(_('initializing destination %s\n') % dest) |
be1377d19018
largefiles: test lfconvert error handling; remove redundant code
Greg Ward <greg@gerg.ca>
parents:
15332
diff
changeset
|
54 rdst = hg.repository(ui, dest, create=True) |
15168 | 55 |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
56 success = False |
15168 | 57 try: |
58 # Lock destination to prevent modification while it is converted to. | |
59 # Don't need to lock src because we are just reading from its history | |
60 # which can't change. | |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
61 dstlock = rdst.lock() |
15168 | 62 |
63 # Get a list of all changesets in the source. The easy way to do this | |
64 # is to simply walk the changelog, using changelog.nodesbewteen(). | |
65 # Take a look at mercurial/revlog.py:639 for more details. | |
66 # Use a generator instead of a list to decrease memory usage | |
67 ctxs = (rsrc[ctx] for ctx in rsrc.changelog.nodesbetween(None, | |
68 rsrc.heads())[0]) | |
69 revmap = {node.nullid: node.nullid} | |
70 if tolfile: | |
71 lfiles = set() | |
72 normalfiles = set() | |
73 if not pats: | |
15579
6c5e6ebe0812
largefiles: use "ui.configlist()" to get largefiles.patterns configuration
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15472
diff
changeset
|
74 pats = ui.configlist(lfutil.longname, 'patterns', default=[]) |
15168 | 75 if pats: |
76 matcher = match_.match(rsrc.root, '', list(pats)) | |
77 else: | |
78 matcher = None | |
79 | |
80 lfiletohash = {} | |
81 for ctx in ctxs: | |
82 ui.progress(_('converting revisions'), ctx.rev(), | |
83 unit=_('revision'), total=rsrc['tip'].rev()) | |
84 _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, | |
85 lfiles, normalfiles, matcher, size, lfiletohash) | |
86 ui.progress(_('converting revisions'), None) | |
87 | |
88 if os.path.exists(rdst.wjoin(lfutil.shortname)): | |
89 shutil.rmtree(rdst.wjoin(lfutil.shortname)) | |
90 | |
91 for f in lfiletohash.keys(): | |
92 if os.path.isfile(rdst.wjoin(f)): | |
93 os.unlink(rdst.wjoin(f)) | |
94 try: | |
95 os.removedirs(os.path.dirname(rdst.wjoin(f))) | |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
96 except OSError: |
15168 | 97 pass |
98 | |
15303
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
99 # If there were any files converted to largefiles, add largefiles |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
100 # to the destination repository's requirements. |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
101 if lfiles: |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
102 rdst.requirements.add('largefiles') |
07811b3b119b
largefiles: include 'largefiles' in converted repository requirements
Eli Carter <eli.carter@tektronix.com>
parents:
15255
diff
changeset
|
103 rdst._writerequirements() |
15168 | 104 else: |
105 for ctx in ctxs: | |
106 ui.progress(_('converting revisions'), ctx.rev(), | |
107 unit=_('revision'), total=rsrc['tip'].rev()) | |
108 _addchangeset(ui, rsrc, rdst, ctx, revmap) | |
109 | |
110 ui.progress(_('converting revisions'), None) | |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
111 success = True |
15168 | 112 finally: |
15171
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
113 if not success: |
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
114 # we failed, remove the new directory |
547da6115d1d
largefiles: eliminate naked exceptions
Matt Mackall <mpm@selenic.com>
parents:
15170
diff
changeset
|
115 shutil.rmtree(rdst.root) |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
116 dstlock.release() |
15168 | 117 |
118 def _addchangeset(ui, rsrc, rdst, ctx, revmap): | |
119 # Convert src parents to dst parents | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
120 parents = _convertparents(ctx, revmap) |
15168 | 121 |
122 # Generate list of changed files | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
123 files = _getchangedfiles(ctx, parents) |
15168 | 124 |
125 def getfilectx(repo, memctx, f): | |
126 if lfutil.standin(f) in files: | |
127 # if the file isn't in the manifest then it was removed | |
128 # or renamed, raise IOError to indicate this | |
129 try: | |
130 fctx = ctx.filectx(lfutil.standin(f)) | |
131 except error.LookupError: | |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
132 raise IOError |
15168 | 133 renamed = fctx.renamed() |
134 if renamed: | |
135 renamed = lfutil.splitstandin(renamed[0]) | |
136 | |
137 hash = fctx.data().strip() | |
138 path = lfutil.findfile(rsrc, hash) | |
139 ### TODO: What if the file is not cached? | |
140 data = '' | |
141 fd = None | |
142 try: | |
143 fd = open(path, 'rb') | |
144 data = fd.read() | |
145 finally: | |
15172
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
146 if fd: |
fb1dcd2aae2a
largefiles: fix multistatement line
Matt Mackall <mpm@selenic.com>
parents:
15171
diff
changeset
|
147 fd.close() |
15168 | 148 return context.memfilectx(f, data, 'l' in fctx.flags(), |
149 'x' in fctx.flags(), renamed) | |
150 else: | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
151 return _getnormalcontext(repo.ui, ctx, f, revmap) |
15168 | 152 |
153 dstfiles = [] | |
154 for file in files: | |
155 if lfutil.isstandin(file): | |
156 dstfiles.append(lfutil.splitstandin(file)) | |
157 else: | |
158 dstfiles.append(file) | |
159 # Commit | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
160 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
15168 | 161 |
162 def _lfconvert_addchangeset(rsrc, rdst, ctx, revmap, lfiles, normalfiles, | |
163 matcher, size, lfiletohash): | |
164 # Convert src parents to dst parents | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
165 parents = _convertparents(ctx, revmap) |
15168 | 166 |
167 # Generate list of changed files | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
168 files = _getchangedfiles(ctx, parents) |
15168 | 169 |
170 dstfiles = [] | |
171 for f in files: | |
172 if f not in lfiles and f not in normalfiles: | |
173 islfile = _islfile(f, ctx, matcher, size) | |
174 # If this file was renamed or copied then copy | |
175 # the lfileness of its predecessor | |
176 if f in ctx.manifest(): | |
177 fctx = ctx.filectx(f) | |
178 renamed = fctx.renamed() | |
179 renamedlfile = renamed and renamed[0] in lfiles | |
180 islfile |= renamedlfile | |
181 if 'l' in fctx.flags(): | |
182 if renamedlfile: | |
183 raise util.Abort( | |
15380
a53888685a6c
largefiles: fix uppercase in abort message
Martin Geisler <mg@aragost.com>
parents:
15371
diff
changeset
|
184 _('renamed/copied largefile %s becomes symlink') |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
185 % f) |
15168 | 186 islfile = False |
187 if islfile: | |
188 lfiles.add(f) | |
189 else: | |
190 normalfiles.add(f) | |
191 | |
192 if f in lfiles: | |
193 dstfiles.append(lfutil.standin(f)) | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
194 # largefile in manifest if it has not been removed/renamed |
15168 | 195 if f in ctx.manifest(): |
15808
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
196 fctx = ctx.filectx(f) |
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
197 if 'l' in fctx.flags(): |
62098aeb1e15
largefiles: don't reference uninitialized variable (issue3092)
Levi Bard <levi@unity3d.com>
parents:
15793
diff
changeset
|
198 renamed = fctx.renamed() |
15168 | 199 if renamed and renamed[0] in lfiles: |
200 raise util.Abort(_('largefile %s becomes symlink') % f) | |
201 | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
202 # largefile was modified, update standins |
15168 | 203 fullpath = rdst.wjoin(f) |
15371
f26ed4ea46d8
largefiles: remove lfutil.createdir, replace calls with util.makedirs
Hao Lian <hao@fogcreek.com>
parents:
15340
diff
changeset
|
204 util.makedirs(os.path.dirname(fullpath)) |
15168 | 205 m = util.sha1('') |
206 m.update(ctx[f].data()) | |
207 hash = m.hexdigest() | |
208 if f not in lfiletohash or lfiletohash[f] != hash: | |
209 try: | |
210 fd = open(fullpath, 'wb') | |
211 fd.write(ctx[f].data()) | |
212 finally: | |
213 if fd: | |
214 fd.close() | |
215 executable = 'x' in ctx[f].flags() | |
216 os.chmod(fullpath, lfutil.getmode(executable)) | |
217 lfutil.writestandin(rdst, lfutil.standin(f), hash, | |
218 executable) | |
219 lfiletohash[f] = hash | |
220 else: | |
221 # normal file | |
222 dstfiles.append(f) | |
223 | |
224 def getfilectx(repo, memctx, f): | |
225 if lfutil.isstandin(f): | |
226 # if the file isn't in the manifest then it was removed | |
227 # or renamed, raise IOError to indicate this | |
228 srcfname = lfutil.splitstandin(f) | |
229 try: | |
230 fctx = ctx.filectx(srcfname) | |
231 except error.LookupError: | |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
232 raise IOError |
15168 | 233 renamed = fctx.renamed() |
234 if renamed: | |
15254
dd03d3a9f888
largefiles: more work on cleaning up comments
Greg Ward <greg@gerg.ca>
parents:
15253
diff
changeset
|
235 # standin is always a largefile because largefile-ness |
15168 | 236 # doesn't change after rename or copy |
237 renamed = lfutil.standin(renamed[0]) | |
238 | |
15313
3eb1a90ea409
largefiles: fix newline for lfconverted repos
Eli Carter <eli.carter@tektronix.com>
parents:
15303
diff
changeset
|
239 return context.memfilectx(f, lfiletohash[srcfname] + '\n', 'l' in |
15168 | 240 fctx.flags(), 'x' in fctx.flags(), renamed) |
241 else: | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
242 return _getnormalcontext(repo.ui, ctx, f, revmap) |
15168 | 243 |
244 # Commit | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
245 _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
246 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
247 def _commitcontext(rdst, parents, ctx, dstfiles, getfilectx, revmap): |
15168 | 248 mctx = context.memctx(rdst, parents, ctx.description(), dstfiles, |
249 getfilectx, ctx.user(), ctx.date(), ctx.extra()) | |
250 ret = rdst.commitctx(mctx) | |
16551
ebf6d38c9063
localrepo: add setparents() to adjust dirstate copies (issue3407)
Patrick Mezard <patrick@mezard.eu>
parents:
16439
diff
changeset
|
251 rdst.setparents(ret) |
15168 | 252 revmap[ctx.node()] = rdst.changelog.tip() |
253 | |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
254 # Generate list of changed files |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
255 def _getchangedfiles(ctx, parents): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
256 files = set(ctx.files()) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
257 if node.nullid not in parents: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
258 mc = ctx.manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
259 mp1 = ctx.parents()[0].manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
260 mp2 = ctx.parents()[1].manifest() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
261 files |= (set(mp1) | set(mp2)) - set(mc) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
262 for f in mc: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
263 if mc[f] != mp1.get(f, None) or mc[f] != mp2.get(f, None): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
264 files.add(f) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
265 return files |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
266 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
267 # Convert src parents to dst parents |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
268 def _convertparents(ctx, revmap): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
269 parents = [] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
270 for p in ctx.parents(): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
271 parents.append(revmap[p.node()]) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
272 while len(parents) < 2: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
273 parents.append(node.nullid) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
274 return parents |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
275 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
276 # Get memfilectx for a normal file |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
277 def _getnormalcontext(ui, ctx, f, revmap): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
278 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
279 fctx = ctx.filectx(f) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
280 except error.LookupError: |
16687
e34106fa0dc3
cleanup: "raise SomeException()" -> "raise SomeException"
Brodie Rao <brodie@sf.io>
parents:
16551
diff
changeset
|
281 raise IOError |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
282 renamed = fctx.renamed() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
283 if renamed: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
284 renamed = renamed[0] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
285 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
286 data = fctx.data() |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
287 if f == '.hgtags': |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
288 data = _converttags (ui, revmap, data) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
289 return context.memfilectx(f, data, 'l' in fctx.flags(), |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
290 'x' in fctx.flags(), renamed) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
291 |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
292 # Remap tag data using a revision map |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
293 def _converttags(ui, revmap, data): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
294 newdata = [] |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
295 for line in data.splitlines(): |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
296 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
297 id, name = line.split(' ', 1) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
298 except ValueError: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
299 ui.warn(_('skipping incorrectly formatted tag %s\n' |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
300 % line)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
301 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
302 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
303 newid = node.bin(id) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
304 except TypeError: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
305 ui.warn(_('skipping incorrectly formatted id %s\n' |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
306 % id)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
307 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
308 try: |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
309 newdata.append('%s %s\n' % (node.hex(revmap[newid]), |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
310 name)) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
311 except KeyError: |
16231
ce292f1379ba
i18n: fix all remaining uses of % inside _()
Matt Mackall <mpm@selenic.com>
parents:
15909
diff
changeset
|
312 ui.warn(_('no mapping for id %s\n') % id) |
15811
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
313 continue |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
314 return ''.join(newdata) |
b9886dde3649
largefiles: remove pasted code
Levi Bard <levi@unity3d.com>
parents:
15809
diff
changeset
|
315 |
15168 | 316 def _islfile(file, ctx, matcher, size): |
15252
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
317 '''Return true if file should be considered a largefile, i.e. |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
318 matcher matches it or it is larger than size.''' |
6e809bb4f969
largefiles: improve comments, internal docstrings
Greg Ward <greg@gerg.ca>
parents:
15230
diff
changeset
|
319 # never store special .hg* files as largefiles |
15168 | 320 if file == '.hgtags' or file == '.hgignore' or file == '.hgsigs': |
321 return False | |
322 if matcher and matcher(file): | |
323 return True | |
324 try: | |
325 return ctx.filectx(file).size() >= size * 1024 * 1024 | |
326 except error.LookupError: | |
327 return False | |
328 | |
329 def uploadlfiles(ui, rsrc, rdst, files): | |
330 '''upload largefiles to the central store''' | |
331 | |
15317
41f371150ccb
largefiles: make the store primary, and the user cache secondary
Benjamin Pollack <benjamin@bitquabit.com>
parents:
15313
diff
changeset
|
332 if not files: |
15168 | 333 return |
334 | |
335 store = basestore._openstore(rsrc, rdst, put=True) | |
336 | |
337 at = 0 | |
338 files = filter(lambda h: not store.exists(h), files) | |
339 for hash in files: | |
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
340 ui.progress(_('uploading largefiles'), at, unit='largefile', |
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
341 total=len(files)) |
15168 | 342 source = lfutil.findfile(rsrc, hash) |
343 if not source: | |
15253
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
344 raise util.Abort(_('largefile %s missing from store' |
67d010779907
largefiles: improve error reporting
Greg Ward <greg@gerg.ca>
parents:
15252
diff
changeset
|
345 ' (needs to be uploaded)') % hash) |
15168 | 346 # XXX check for errors here |
347 store.put(source, hash) | |
348 at += 1 | |
15173
3d27a8ff895f
largefiles: mark a string for translation
Matt Mackall <mpm@selenic.com>
parents:
15172
diff
changeset
|
349 ui.progress(_('uploading largefiles'), None) |
15168 | 350 |
351 def verifylfiles(ui, repo, all=False, contents=False): | |
352 '''Verify that every big file revision in the current changeset | |
353 exists in the central store. With --contents, also verify that | |
354 the contents of each big file revision are correct (SHA-1 hash | |
355 matches the revision ID). With --all, check every changeset in | |
356 this repository.''' | |
357 if all: | |
358 # Pass a list to the function rather than an iterator because we know a | |
359 # list will work. | |
360 revs = range(len(repo)) | |
361 else: | |
362 revs = ['.'] | |
363 | |
364 store = basestore._openstore(repo) | |
365 return store.verify(revs, contents=contents) | |
366 | |
367 def cachelfiles(ui, repo, node): | |
368 '''cachelfiles ensures that all largefiles needed by the specified revision | |
369 are present in the repository's largefile cache. | |
370 | |
371 returns a tuple (cached, missing). cached is the list of files downloaded | |
372 by this operation; missing is the list of files that were needed but could | |
373 not be found.''' | |
374 lfiles = lfutil.listlfiles(repo, node) | |
375 toget = [] | |
376 | |
377 for lfile in lfiles: | |
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
378 # If we are mid-merge, then we have to trust the standin that is in the |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
379 # working copy to have the correct hashvalue. This is because the |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
380 # original hg.merge() already updated the standin as part of the normal |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
381 # merge process -- we just have to udpate the largefile to match. |
15905
634d49a8b6db
largefiles: correctly handle newly added largefile on other side of merge
Na'Tosha Bard <natosha@unity3d.com>
parents:
15860
diff
changeset
|
382 if (getattr(repo, "_ismerging", False) and |
634d49a8b6db
largefiles: correctly handle newly added largefile on other side of merge
Na'Tosha Bard <natosha@unity3d.com>
parents:
15860
diff
changeset
|
383 os.path.exists(repo.wjoin(lfutil.standin(lfile)))): |
15860
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
384 expectedhash = lfutil.readstandin(repo, lfile) |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
385 else: |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
386 expectedhash = repo[node][lfutil.standin(lfile)].data().strip() |
3ecce805ac13
largefiles: correctly download new largefiles when merging
Na'Tosha Bard <natosha@unity3d.com>
parents:
15811
diff
changeset
|
387 |
15168 | 388 # if it exists and its hash matches, it might have been locally |
389 # modified before updating and the user chose 'local'. in this case, | |
390 # it will not be in any store, so don't look for it. | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
391 if ((not os.path.exists(repo.wjoin(lfile)) or |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
392 expectedhash != lfutil.hashfile(repo.wjoin(lfile))) and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
393 not lfutil.findfile(repo, expectedhash)): |
15168 | 394 toget.append((lfile, expectedhash)) |
395 | |
396 if toget: | |
397 store = basestore._openstore(repo) | |
398 ret = store.get(toget) | |
399 return ret | |
400 | |
401 return ([], []) | |
402 | |
403 def updatelfiles(ui, repo, filelist=None, printmessage=True): | |
404 wlock = repo.wlock() | |
405 try: | |
406 lfdirstate = lfutil.openlfdirstate(ui, repo) | |
407 lfiles = set(lfutil.listlfiles(repo)) | set(lfdirstate) | |
408 | |
409 if filelist is not None: | |
410 lfiles = [f for f in lfiles if f in filelist] | |
411 | |
412 printed = False | |
413 if printmessage and lfiles: | |
414 ui.status(_('getting changed largefiles\n')) | |
415 printed = True | |
416 cachelfiles(ui, repo, '.') | |
417 | |
418 updated, removed = 0, 0 | |
419 for i in map(lambda f: _updatelfile(repo, lfdirstate, f), lfiles): | |
420 # increment the appropriate counter according to _updatelfile's | |
421 # return value | |
422 updated += i > 0 and i or 0 | |
423 removed -= i < 0 and i or 0 | |
424 if printmessage and (removed or updated) and not printed: | |
425 ui.status(_('getting changed largefiles\n')) | |
426 printed = True | |
427 | |
428 lfdirstate.write() | |
429 if printed and printmessage: | |
430 ui.status(_('%d largefiles updated, %d removed\n') % (updated, | |
431 removed)) | |
432 finally: | |
433 wlock.release() | |
434 | |
435 def _updatelfile(repo, lfdirstate, lfile): | |
436 '''updates a single largefile and copies the state of its standin from | |
437 the repository's dirstate to its state in the lfdirstate. | |
438 | |
439 returns 1 if the file was modified, -1 if the file was removed, 0 if the | |
440 file was unchanged, and None if the needed largefile was missing from the | |
441 cache.''' | |
442 ret = 0 | |
443 abslfile = repo.wjoin(lfile) | |
444 absstandin = repo.wjoin(lfutil.standin(lfile)) | |
445 if os.path.exists(absstandin): | |
446 if os.path.exists(absstandin+'.orig'): | |
447 shutil.copyfile(abslfile, abslfile+'.orig') | |
448 expecthash = lfutil.readstandin(repo, lfile) | |
15255
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
449 if (expecthash != '' and |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
450 (not os.path.exists(abslfile) or |
7ab05d752405
largefiles: cosmetics, whitespace, code style
Greg Ward <greg@gerg.ca>
parents:
15254
diff
changeset
|
451 expecthash != lfutil.hashfile(abslfile))): |
15168 | 452 if not lfutil.copyfromcache(repo, expecthash, lfile): |
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
453 # use normallookup() to allocate entry in largefiles dirstate, |
16247
d87d9d8a8e03
largefiles: remove use of underscores that breaks coding convention
Na'Tosha Bard <natosha@unity3d.com>
parents:
16231
diff
changeset
|
454 # because lack of it misleads lfilesrepo.status() into |
15472
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
455 # recognition that such cache missing files are REMOVED. |
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
456 lfdirstate.normallookup(lfile) |
6a7e874390b0
largefiles: treat status of cache missed largefiles as "missing" correctly
FUJIWARA Katsunori <foozy@lares.dti.ne.jp>
parents:
15380
diff
changeset
|
457 return None # don't try to set the mode |
15168 | 458 ret = 1 |
459 mode = os.stat(absstandin).st_mode | |
460 if mode != os.stat(abslfile).st_mode: | |
461 os.chmod(abslfile, mode) | |
462 ret = 1 | |
463 else: | |
15663
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
464 # Remove lfiles for which the standin is deleted, unless the |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
465 # lfile is added to the repository again. This happens when a |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
466 # largefile is converted back to a normal file: the standin |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
467 # disappears, but a new (normal) file appears as the lfile. |
9036c7d106bf
largefiles: handle merges between normal files and largefiles (issue3084)
Martin Geisler <mg@aragost.com>
parents:
15579
diff
changeset
|
468 if os.path.exists(abslfile) and lfile not in repo[None]: |
15900
29defa7d20f6
largefiles: remove empty directories upon update (issue3202)
Patrick Mezard <pmezard@gmail.com>
parents:
15663
diff
changeset
|
469 util.unlinkpath(abslfile) |
15168 | 470 ret = -1 |
471 state = repo.dirstate[lfutil.standin(lfile)] | |
472 if state == 'n': | |
15793
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
473 # When rebasing, we need to synchronize the standin and the largefile, |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
474 # because otherwise the largefile will get reverted. But for commit's |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
475 # sake, we have to mark the file as unclean. |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
476 if getattr(repo, "_isrebasing", False): |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
477 lfdirstate.normallookup(lfile) |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
478 else: |
3ef07ecdb0d5
largefiles: correctly handle dirstate status when rebasing
Na'Tosha Bard <natosha@unity3d.com>
parents:
15663
diff
changeset
|
479 lfdirstate.normal(lfile) |
15168 | 480 elif state == 'r': |
481 lfdirstate.remove(lfile) | |
482 elif state == 'a': | |
483 lfdirstate.add(lfile) | |
484 elif state == '?': | |
15224
7c604d8c7e83
largefiles: remove pre-1.9 code from extension first bundled with 1.9
Na'Tosha Bard <natosha@unity3d.com>
parents:
15173
diff
changeset
|
485 lfdirstate.drop(lfile) |
15168 | 486 return ret |
487 | |
16439
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
488 def catlfile(repo, lfile, rev, filename): |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
489 hash = lfutil.readstandin(repo, lfile, rev) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
490 if not lfutil.inusercache(repo.ui, hash): |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
491 store = basestore._openstore(repo) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
492 success, missing = store.get([(lfile, hash)]) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
493 if len(success) != 1: |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
494 raise util.Abort( |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
495 _('largefile %s is not in cache and could not be downloaded') |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
496 % lfile) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
497 path = lfutil.usercachepath(repo.ui, hash) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
498 fpout = cmdutil.makefileobj(repo, filename) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
499 fpin = open(path, "rb") |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
500 fpout.write(fpin.read()) |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
501 fpout.close() |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
502 fpin.close() |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
503 return 0 |
290850e7aa43
largefiles: fix cat for largefiles (issue3352)
Na'Tosha Bard <natosha@unity3d.com>
parents:
16247
diff
changeset
|
504 |
15168 | 505 # -- hg commands declarations ------------------------------------------------ |
506 | |
507 cmdtable = { | |
508 'lfconvert': (lfconvert, | |
15230 | 509 [('s', 'size', '', |
510 _('minimum size (MB) for files to be converted ' | |
511 'as largefiles'), | |
512 'SIZE'), | |
15332
0db47b8d025f
largefiles: rename lfconvert --tonormal option to --to-normal
Greg Ward <greg@gerg.ca>
parents:
15317
diff
changeset
|
513 ('', 'to-normal', False, |
15230 | 514 _('convert from a largefiles repo to a normal repo')), |
515 ], | |
15168 | 516 _('hg lfconvert SOURCE DEST [FILE ...]')), |
517 } |