Mercurial > public > mercurial-scm > hg
annotate hgext/largefiles/lfutil.py @ 15206:f85c76b16f27
largefiles: fix commit of specified file on non-windows
| author | Na'Tosha Bard <natosha@unity3d.com> |
|---|---|
| date | Thu, 06 Oct 2011 11:10:06 +0200 |
| parents | 8e115063950d |
| children | 7c604d8c7e83 |
| rev | line source |
|---|---|
| 15168 | 1 # Copyright 2009-2010 Gregory P. Ward |
| 2 # Copyright 2009-2010 Intelerad Medical Systems Incorporated | |
| 3 # Copyright 2010-2011 Fog Creek Software | |
| 4 # Copyright 2010-2011 Unity Technologies | |
| 5 # | |
| 6 # This software may be used and distributed according to the terms of the | |
| 7 # GNU General Public License version 2 or any later version. | |
| 8 | |
| 9 '''largefiles utility code: must not import other modules in this package.''' | |
| 10 | |
| 11 import os | |
| 12 import errno | |
| 13 import inspect | |
| 14 import shutil | |
| 15 import stat | |
| 16 import hashlib | |
| 17 | |
| 18 from mercurial import cmdutil, dirstate, httpconnection, match as match_, \ | |
| 19 url as url_, util | |
| 20 from mercurial.i18n import _ | |
| 21 | |
| 22 try: | |
| 23 from mercurial import scmutil | |
| 24 except ImportError: | |
| 25 pass | |
| 26 | |
| 27 shortname = '.hglf' | |
| 28 longname = 'largefiles' | |
| 29 | |
| 30 | |
| 31 # -- Portability wrappers ---------------------------------------------- | |
| 32 | |
| 33 if 'subrepos' in inspect.getargspec(dirstate.dirstate.status)[0]: | |
| 34 # for Mercurial >= 1.5 | |
| 35 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): | |
| 36 return dirstate.walk(matcher, [], unknown, ignored) | |
| 37 else: | |
| 38 # for Mercurial <= 1.4 | |
| 39 def dirstate_walk(dirstate, matcher, unknown=False, ignored=False): | |
| 40 return dirstate.walk(matcher, unknown, ignored) | |
| 41 | |
| 42 def repo_add(repo, list): | |
| 43 try: | |
| 44 # Mercurial <= 1.5 | |
| 45 add = repo.add | |
| 46 except AttributeError: | |
| 47 # Mercurial >= 1.6 | |
| 48 add = repo[None].add | |
| 49 return add(list) | |
| 50 | |
| 51 def repo_remove(repo, list, unlink=False): | |
| 52 try: | |
| 53 # Mercurial <= 1.5 | |
| 54 remove = repo.remove | |
| 55 except AttributeError: | |
| 56 # Mercurial >= 1.6 | |
| 57 try: | |
| 58 # Mercurial <= 1.8 | |
| 59 remove = repo[None].remove | |
| 60 except AttributeError: | |
| 61 # Mercurial >= 1.9 | |
| 62 def remove(list, unlink): | |
| 63 wlock = repo.wlock() | |
| 64 try: | |
| 65 if unlink: | |
| 66 for f in list: | |
| 67 try: | |
| 68 util.unlinkpath(repo.wjoin(f)) | |
| 69 except OSError, inst: | |
| 70 if inst.errno != errno.ENOENT: | |
| 71 raise | |
| 72 repo[None].forget(list) | |
| 73 finally: | |
| 74 wlock.release() | |
| 75 | |
| 76 return remove(list, unlink=unlink) | |
| 77 | |
| 78 def repo_forget(repo, list): | |
| 79 try: | |
| 80 # Mercurial <= 1.5 | |
| 81 forget = repo.forget | |
| 82 except AttributeError: | |
| 83 # Mercurial >= 1.6 | |
| 84 forget = repo[None].forget | |
| 85 return forget(list) | |
| 86 | |
| 87 def findoutgoing(repo, remote, force): | |
| 88 # First attempt is for Mercurial <= 1.5 second is for >= 1.6 | |
| 89 try: | |
| 90 return repo.findoutgoing(remote) | |
| 91 except AttributeError: | |
| 92 from mercurial import discovery | |
| 93 try: | |
| 94 # Mercurial <= 1.8 | |
| 95 return discovery.findoutgoing(repo, remote, force=force) | |
| 96 except AttributeError: | |
| 97 # Mercurial >= 1.9 | |
| 98 common, _anyinc, _heads = discovery.findcommonincoming(repo, | |
| 99 remote, force=force) | |
| 100 return repo.changelog.findmissing(common) | |
| 101 | |
| 102 # -- Private worker functions ------------------------------------------ | |
| 103 | |
| 104 def link(src, dest): | |
| 105 try: | |
|
15206
f85c76b16f27
largefiles: fix commit of specified file on non-windows
Na'Tosha Bard <natosha@unity3d.com>
parents:
15188
diff
changeset
|
106 util.oslink(src, dest) |
| 15168 | 107 except OSError: |
| 108 # If hardlinks fail fall back on copy | |
| 109 shutil.copyfile(src, dest) | |
| 110 os.chmod(dest, os.stat(src).st_mode) | |
| 111 | |
| 112 def systemcachepath(ui, hash): | |
| 113 path = ui.config(longname, 'systemcache', None) | |
| 114 if path: | |
| 115 path = os.path.join(path, hash) | |
| 116 else: | |
| 117 if os.name == 'nt': | |
| 118 path = os.path.join(os.getenv('LOCALAPPDATA') or \ | |
| 119 os.getenv('APPDATA'), longname, hash) | |
| 120 elif os.name == 'posix': | |
| 121 path = os.path.join(os.getenv('HOME'), '.' + longname, hash) | |
| 122 else: | |
| 123 raise util.Abort(_('Unknown operating system: %s\n') % os.name) | |
| 124 return path | |
| 125 | |
| 126 def insystemcache(ui, hash): | |
| 127 return os.path.exists(systemcachepath(ui, hash)) | |
| 128 | |
| 129 def findfile(repo, hash): | |
| 130 if incache(repo, hash): | |
| 131 repo.ui.note(_('Found %s in cache\n') % hash) | |
| 132 return cachepath(repo, hash) | |
| 133 if insystemcache(repo.ui, hash): | |
| 134 repo.ui.note(_('Found %s in system cache\n') % hash) | |
| 135 return systemcachepath(repo.ui, hash) | |
| 136 return None | |
| 137 | |
| 138 class largefiles_dirstate(dirstate.dirstate): | |
| 139 def __getitem__(self, key): | |
| 140 return super(largefiles_dirstate, self).__getitem__(unixpath(key)) | |
| 141 def normal(self, f): | |
| 142 return super(largefiles_dirstate, self).normal(unixpath(f)) | |
| 143 def remove(self, f): | |
| 144 return super(largefiles_dirstate, self).remove(unixpath(f)) | |
| 145 def add(self, f): | |
| 146 return super(largefiles_dirstate, self).add(unixpath(f)) | |
| 147 def drop(self, f): | |
| 148 return super(largefiles_dirstate, self).drop(unixpath(f)) | |
| 149 def forget(self, f): | |
| 150 return super(largefiles_dirstate, self).forget(unixpath(f)) | |
| 151 | |
| 152 def openlfdirstate(ui, repo): | |
| 153 ''' | |
| 154 Return a dirstate object that tracks big files: i.e. its root is the | |
| 155 repo root, but it is saved in .hg/largefiles/dirstate. | |
| 156 ''' | |
| 157 admin = repo.join(longname) | |
| 158 try: | |
| 159 # Mercurial >= 1.9 | |
| 160 opener = scmutil.opener(admin) | |
| 161 except ImportError: | |
| 162 # Mercurial <= 1.8 | |
| 163 opener = util.opener(admin) | |
|
15169
aa262fff87ac
largefile: fix up hasattr usage
Matt Mackall <mpm@selenic.com>
parents:
15168
diff
changeset
|
164 if util.safehasattr(repo.dirstate, '_validate'): |
| 15168 | 165 lfdirstate = largefiles_dirstate(opener, ui, repo.root, |
| 166 repo.dirstate._validate) | |
| 167 else: | |
| 168 lfdirstate = largefiles_dirstate(opener, ui, repo.root) | |
| 169 | |
| 170 # If the largefiles dirstate does not exist, populate and create it. This | |
| 171 # ensures that we create it on the first meaningful largefiles operation in | |
| 172 # a new clone. It also gives us an easy way to forcibly rebuild largefiles | |
| 173 # state: | |
| 174 # rm .hg/largefiles/dirstate && hg status | |
| 175 # Or even, if things are really messed up: | |
| 176 # rm -rf .hg/largefiles && hg status | |
| 177 if not os.path.exists(os.path.join(admin, 'dirstate')): | |
| 178 util.makedirs(admin) | |
| 179 matcher = getstandinmatcher(repo) | |
| 180 for standin in dirstate_walk(repo.dirstate, matcher): | |
| 181 lfile = splitstandin(standin) | |
| 182 hash = readstandin(repo, lfile) | |
| 183 lfdirstate.normallookup(lfile) | |
| 184 try: | |
| 185 if hash == hashfile(lfile): | |
| 186 lfdirstate.normal(lfile) | |
| 187 except IOError, err: | |
| 188 if err.errno != errno.ENOENT: | |
| 189 raise | |
| 190 | |
| 191 lfdirstate.write() | |
| 192 | |
| 193 return lfdirstate | |
| 194 | |
| 195 def lfdirstate_status(lfdirstate, repo, rev): | |
| 196 wlock = repo.wlock() | |
| 197 try: | |
| 198 match = match_.always(repo.root, repo.getcwd()) | |
| 199 s = lfdirstate.status(match, [], False, False, False) | |
| 200 unsure, modified, added, removed, missing, unknown, ignored, clean = s | |
| 201 for lfile in unsure: | |
| 202 if repo[rev][standin(lfile)].data().strip() != \ | |
| 203 hashfile(repo.wjoin(lfile)): | |
| 204 modified.append(lfile) | |
| 205 else: | |
| 206 clean.append(lfile) | |
| 207 lfdirstate.normal(lfile) | |
| 208 lfdirstate.write() | |
| 209 finally: | |
| 210 wlock.release() | |
| 211 return (modified, added, removed, missing, unknown, ignored, clean) | |
| 212 | |
| 213 def listlfiles(repo, rev=None, matcher=None): | |
| 214 '''list largefiles in the working copy or specified changeset''' | |
| 215 | |
| 216 if matcher is None: | |
| 217 matcher = getstandinmatcher(repo) | |
| 218 | |
| 219 # ignore unknown files in working directory | |
| 220 return [splitstandin(f) for f in repo[rev].walk(matcher) \ | |
| 221 if rev is not None or repo.dirstate[f] != '?'] | |
| 222 | |
| 223 def incache(repo, hash): | |
| 224 return os.path.exists(cachepath(repo, hash)) | |
| 225 | |
| 226 def createdir(dir): | |
| 227 if not os.path.exists(dir): | |
| 228 os.makedirs(dir) | |
| 229 | |
| 230 def cachepath(repo, hash): | |
| 231 return repo.join(os.path.join(longname, hash)) | |
| 232 | |
| 233 def copyfromcache(repo, hash, filename): | |
| 234 '''copyfromcache copies the specified largefile from the repo or system | |
| 235 cache to the specified location in the repository. It will not throw an | |
| 236 exception on failure, as it is meant to be called only after ensuring that | |
| 237 the needed largefile exists in the cache.''' | |
| 238 path = findfile(repo, hash) | |
| 239 if path is None: | |
| 240 return False | |
| 241 util.makedirs(os.path.dirname(repo.wjoin(filename))) | |
| 242 shutil.copy(path, repo.wjoin(filename)) | |
| 243 return True | |
| 244 | |
| 245 def copytocache(repo, rev, file, uploaded=False): | |
| 246 hash = readstandin(repo, file) | |
| 247 if incache(repo, hash): | |
| 248 return | |
| 249 copytocacheabsolute(repo, repo.wjoin(file), hash) | |
| 250 | |
| 251 def copytocacheabsolute(repo, file, hash): | |
| 252 createdir(os.path.dirname(cachepath(repo, hash))) | |
| 253 if insystemcache(repo.ui, hash): | |
| 254 link(systemcachepath(repo.ui, hash), cachepath(repo, hash)) | |
| 255 else: | |
| 256 shutil.copyfile(file, cachepath(repo, hash)) | |
| 257 os.chmod(cachepath(repo, hash), os.stat(file).st_mode) | |
| 258 linktosystemcache(repo, hash) | |
| 259 | |
| 260 def linktosystemcache(repo, hash): | |
| 261 createdir(os.path.dirname(systemcachepath(repo.ui, hash))) | |
| 262 link(cachepath(repo, hash), systemcachepath(repo.ui, hash)) | |
| 263 | |
| 264 def getstandinmatcher(repo, pats=[], opts={}): | |
| 265 '''Return a match object that applies pats to the standin directory''' | |
| 266 standindir = repo.pathto(shortname) | |
| 267 if pats: | |
| 268 # patterns supplied: search standin directory relative to current dir | |
| 269 cwd = repo.getcwd() | |
| 270 if os.path.isabs(cwd): | |
| 271 # cwd is an absolute path for hg -R <reponame> | |
| 272 # work relative to the repository root in this case | |
| 273 cwd = '' | |
| 274 pats = [os.path.join(standindir, cwd, pat) for pat in pats] | |
| 275 elif os.path.isdir(standindir): | |
| 276 # no patterns: relative to repo root | |
| 277 pats = [standindir] | |
| 278 else: | |
| 279 # no patterns and no standin dir: return matcher that matches nothing | |
| 280 match = match_.match(repo.root, None, [], exact=True) | |
| 281 match.matchfn = lambda f: False | |
| 282 return match | |
| 283 return getmatcher(repo, pats, opts, showbad=False) | |
| 284 | |
| 285 def getmatcher(repo, pats=[], opts={}, showbad=True): | |
| 286 '''Wrapper around scmutil.match() that adds showbad: if false, neuter | |
| 287 the match object\'s bad() method so it does not print any warnings | |
| 288 about missing files or directories.''' | |
| 289 try: | |
| 290 # Mercurial >= 1.9 | |
| 291 match = scmutil.match(repo[None], pats, opts) | |
| 292 except ImportError: | |
| 293 # Mercurial <= 1.8 | |
| 294 match = cmdutil.match(repo, pats, opts) | |
| 295 | |
| 296 if not showbad: | |
| 297 match.bad = lambda f, msg: None | |
| 298 return match | |
| 299 | |
| 300 def composestandinmatcher(repo, rmatcher): | |
| 301 '''Return a matcher that accepts standins corresponding to the files | |
| 302 accepted by rmatcher. Pass the list of files in the matcher as the | |
| 303 paths specified by the user.''' | |
| 304 smatcher = getstandinmatcher(repo, rmatcher.files()) | |
| 305 isstandin = smatcher.matchfn | |
| 306 def composed_matchfn(f): | |
| 307 return isstandin(f) and rmatcher.matchfn(splitstandin(f)) | |
| 308 smatcher.matchfn = composed_matchfn | |
| 309 | |
| 310 return smatcher | |
| 311 | |
| 312 def standin(filename): | |
| 313 '''Return the repo-relative path to the standin for the specified big | |
| 314 file.''' | |
| 315 # Notes: | |
| 316 # 1) Most callers want an absolute path, but _create_standin() needs | |
| 317 # it repo-relative so lfadd() can pass it to repo_add(). So leave | |
| 318 # it up to the caller to use repo.wjoin() to get an absolute path. | |
| 319 # 2) Join with '/' because that's what dirstate always uses, even on | |
| 320 # Windows. Change existing separator to '/' first in case we are | |
| 321 # passed filenames from an external source (like the command line). | |
| 322 return shortname + '/' + filename.replace(os.sep, '/') | |
| 323 | |
| 324 def isstandin(filename): | |
| 325 '''Return true if filename is a big file standin. filename must | |
| 326 be in Mercurial\'s internal form (slash-separated).''' | |
| 327 return filename.startswith(shortname + '/') | |
| 328 | |
| 329 def splitstandin(filename): | |
| 330 # Split on / because that's what dirstate always uses, even on Windows. | |
| 331 # Change local separator to / first just in case we are passed filenames | |
| 332 # from an external source (like the command line). | |
| 333 bits = filename.replace(os.sep, '/').split('/', 1) | |
| 334 if len(bits) == 2 and bits[0] == shortname: | |
| 335 return bits[1] | |
| 336 else: | |
| 337 return None | |
| 338 | |
| 339 def updatestandin(repo, standin): | |
| 340 file = repo.wjoin(splitstandin(standin)) | |
| 341 if os.path.exists(file): | |
| 342 hash = hashfile(file) | |
| 343 executable = getexecutable(file) | |
| 344 writestandin(repo, standin, hash, executable) | |
| 345 | |
| 346 def readstandin(repo, filename, node=None): | |
| 347 '''read hex hash from standin for filename at given node, or working | |
| 348 directory if no node is given''' | |
| 349 return repo[node][standin(filename)].data().strip() | |
| 350 | |
| 351 def writestandin(repo, standin, hash, executable): | |
| 352 '''write hhash to <repo.root>/<standin>''' | |
| 353 writehash(hash, repo.wjoin(standin), executable) | |
| 354 | |
| 355 def copyandhash(instream, outfile): | |
| 356 '''Read bytes from instream (iterable) and write them to outfile, | |
| 357 computing the SHA-1 hash of the data along the way. Close outfile | |
| 358 when done and return the binary hash.''' | |
| 359 hasher = util.sha1('') | |
| 360 for data in instream: | |
| 361 hasher.update(data) | |
| 362 outfile.write(data) | |
| 363 | |
| 364 # Blecch: closing a file that somebody else opened is rude and | |
| 365 # wrong. But it's so darn convenient and practical! After all, | |
| 366 # outfile was opened just to copy and hash. | |
| 367 outfile.close() | |
| 368 | |
| 369 return hasher.digest() | |
| 370 | |
| 371 def hashrepofile(repo, file): | |
| 372 return hashfile(repo.wjoin(file)) | |
| 373 | |
| 374 def hashfile(file): | |
| 375 if not os.path.exists(file): | |
| 376 return '' | |
| 377 hasher = util.sha1('') | |
| 378 fd = open(file, 'rb') | |
| 379 for data in blockstream(fd): | |
| 380 hasher.update(data) | |
| 381 fd.close() | |
| 382 return hasher.hexdigest() | |
| 383 | |
| 384 class limitreader(object): | |
| 385 def __init__(self, f, limit): | |
| 386 self.f = f | |
| 387 self.limit = limit | |
| 388 | |
| 389 def read(self, length): | |
| 390 if self.limit == 0: | |
| 391 return '' | |
| 392 length = length > self.limit and self.limit or length | |
| 393 self.limit -= length | |
| 394 return self.f.read(length) | |
| 395 | |
| 396 def close(self): | |
| 397 pass | |
| 398 | |
| 399 def blockstream(infile, blocksize=128 * 1024): | |
| 400 """Generator that yields blocks of data from infile and closes infile.""" | |
| 401 while True: | |
| 402 data = infile.read(blocksize) | |
| 403 if not data: | |
| 404 break | |
| 405 yield data | |
| 406 # Same blecch as above. | |
| 407 infile.close() | |
| 408 | |
| 409 def readhash(filename): | |
| 410 rfile = open(filename, 'rb') | |
| 411 hash = rfile.read(40) | |
| 412 rfile.close() | |
| 413 if len(hash) < 40: | |
| 414 raise util.Abort(_('bad hash in \'%s\' (only %d bytes long)') | |
| 415 % (filename, len(hash))) | |
| 416 return hash | |
| 417 | |
| 418 def writehash(hash, filename, executable): | |
| 419 util.makedirs(os.path.dirname(filename)) | |
| 420 if os.path.exists(filename): | |
| 421 os.unlink(filename) | |
| 422 wfile = open(filename, 'wb') | |
| 423 | |
| 424 try: | |
| 425 wfile.write(hash) | |
| 426 wfile.write('\n') | |
| 427 finally: | |
| 428 wfile.close() | |
| 429 if os.path.exists(filename): | |
| 430 os.chmod(filename, getmode(executable)) | |
| 431 | |
| 432 def getexecutable(filename): | |
| 433 mode = os.stat(filename).st_mode | |
| 434 return (mode & stat.S_IXUSR) and (mode & stat.S_IXGRP) and (mode & \ | |
| 435 stat.S_IXOTH) | |
| 436 | |
| 437 def getmode(executable): | |
| 438 if executable: | |
| 439 return 0755 | |
| 440 else: | |
| 441 return 0644 | |
| 442 | |
| 443 def urljoin(first, second, *arg): | |
| 444 def join(left, right): | |
| 445 if not left.endswith('/'): | |
| 446 left += '/' | |
| 447 if right.startswith('/'): | |
| 448 right = right[1:] | |
| 449 return left + right | |
| 450 | |
| 451 url = join(first, second) | |
| 452 for a in arg: | |
| 453 url = join(url, a) | |
| 454 return url | |
| 455 | |
| 456 def hexsha1(data): | |
| 457 """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like | |
| 458 object data""" | |
| 459 h = hashlib.sha1() | |
| 460 for chunk in util.filechunkiter(data): | |
| 461 h.update(chunk) | |
| 462 return h.hexdigest() | |
| 463 | |
| 464 def httpsendfile(ui, filename): | |
| 465 try: | |
| 466 # Mercurial >= 1.9 | |
| 467 return httpconnection.httpsendfile(ui, filename, 'rb') | |
| 468 except ImportError: | |
| 469 if 'ui' in inspect.getargspec(url_.httpsendfile.__init__)[0]: | |
| 470 # Mercurial == 1.8 | |
| 471 return url_.httpsendfile(ui, filename, 'rb') | |
| 472 else: | |
| 473 # Mercurial <= 1.7 | |
| 474 return url_.httpsendfile(filename, 'rb') | |
| 475 | |
| 476 # Convert a path to a unix style path. This is used to give a | |
| 477 # canonical path to the lfdirstate. | |
| 478 def unixpath(path): | |
| 479 return os.path.normpath(path).replace(os.sep, '/') | |
| 480 | |
| 481 def islfilesrepo(repo): | |
|
15170
c1a4a3220711
largefiles: fix over-long lines
Matt Mackall <mpm@selenic.com>
parents:
15169
diff
changeset
|
482 return ('largefiles' in repo.requirements and |
|
15188
8e115063950d
largefiles: don't break existing tests (syntax error, bad imports)
Greg Ward <greg@gerg.ca>
parents:
15171
diff
changeset
|
483 any_(shortname + '/' in f[0] for f in repo.store.datafiles())) |
| 15168 | 484 |
| 485 def any_(gen): | |
| 486 for x in gen: | |
| 487 if x: | |
| 488 return True | |
| 489 return False | |
| 490 | |
| 491 class storeprotonotcapable(BaseException): | |
| 492 def __init__(self, storetypes): | |
| 493 self.storetypes = storetypes |