Mercurial > public > mercurial-scm > hg-stable

annotate hgext/lfs/blobstore.py @ 35403:c8edeb03ca94

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lfs: correct the directory list value returned by lfsvfs.walk() Spotted by Yuya.
author Matt Harbison <matt_harbison@yahoo.com>
date Tue, 12 Dec 2017 20:22:38 -0500
parents 79968f91ad0c
children f98fac24b757
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
2 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
3 # Copyright 2017 Facebook, Inc.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
4 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
5 # This software may be used and distributed according to the terms of the
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
6 # GNU General Public License version 2 or any later version.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
7
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
8 from __future__ import absolute_import
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
9
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
10 import json
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
11 import os
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
12 import re
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
13
35122
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35121
diff changeset
14 from mercurial.i18n import _
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35121
diff changeset
15
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
16 from mercurial import (
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
17 error,
35371
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
18 pathutil,
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
19 url as urlmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
20 util,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
21 vfs as vfsmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
22 )
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
23
35289
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
24 from ..largefiles import lfutil
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
25
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
26 # 64 bytes for SHA256
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
27 _lfsre = re.compile(r'\A[a-f0-9]{64}\Z')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
28
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
29 class lfsvfs(vfsmod.vfs):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
30 def join(self, path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
31 """split the path at first two characters, like: XX/XXXXX..."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
32 if not _lfsre.match(path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
33 raise error.ProgrammingError('unexpected lfs path: %s' % path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
34 return super(lfsvfs, self).join(path[0:2], path[2:])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
35
35371
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
36 def walk(self, path=None, onerror=None):
35403
c8edeb03ca94 lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents: 35371
diff changeset
37 """Yield (dirpath, [], oids) tuple for blobs under path
35371
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
38
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
39 Oids only exist in the root of this vfs, so dirpath is always ''.
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
40 """
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
41 root = os.path.normpath(self.base)
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
42 # when dirpath == root, dirpath[prefixlen:] becomes empty
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
43 # because len(dirpath) < prefixlen.
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
44 prefixlen = len(pathutil.normasprefix(root))
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
45 oids = []
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
46
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
47 for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''),
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
48 onerror=onerror):
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
49 dirpath = dirpath[prefixlen:]
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
50
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
51 # Silently skip unexpected files and directories
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
52 if len(dirpath) == 2:
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
53 oids.extend([dirpath + f for f in files
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
54 if _lfsre.match(dirpath + f)])
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
55
35403
c8edeb03ca94 lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents: 35371
diff changeset
56 yield ('', [], oids)
35371
79968f91ad0c lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents: 35289
diff changeset
57
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
58 class filewithprogress(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
59 """a file-like object that supports __len__ and read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
60
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
61 Useful to provide progress information for how many bytes are read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
62 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
63
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
64 def __init__(self, fp, callback):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
65 self._fp = fp
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
66 self._callback = callback # func(readsize)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
67 fp.seek(0, os.SEEK_END)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
68 self._len = fp.tell()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
69 fp.seek(0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
70
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
71 def __len__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
72 return self._len
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
73
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
74 def read(self, size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
75 if self._fp is None:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
76 return b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
77 data = self._fp.read(size)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
78 if data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
79 if self._callback:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
80 self._callback(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
81 else:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
82 self._fp.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
83 self._fp = None
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
84 return data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
85
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
86 class local(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
87 """Local blobstore for large file contents.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
88
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
89 This blobstore is used both as a cache and as a staging area for large blobs
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
90 to be uploaded to the remote blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
91 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
92
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
93 def __init__(self, repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
94 fullpath = repo.svfs.join('lfs/objects')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
95 self.vfs = lfsvfs(fullpath)
35289
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
96 usercache = lfutil._usercachedir(repo.ui, 'lfs')
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
97 self.cachevfs = lfsvfs(usercache)
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
98
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
99 def write(self, oid, data):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
100 """Write blob to local blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
101 with self.vfs(oid, 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
102 fp.write(data)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
103
35289
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
104 # XXX: should we verify the content of the cache, and hardlink back to
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
105 # the local store on success, but truncate, write and link on failure?
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
106 if not self.cachevfs.exists(oid):
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
107 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
108
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
109 def read(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
110 """Read blob from local blobstore."""
35289
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
111 if not self.vfs.exists(oid):
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
112 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
113 return self.vfs.read(oid)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
114
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
115 def has(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
116 """Returns True if the local blobstore contains the requested blob,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
117 False otherwise."""
35289
8e72f9152c4d lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents: 35123
diff changeset
118 return self.cachevfs.exists(oid) or self.vfs.exists(oid)
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
119
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
120 class _gitlfsremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
122 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
123 ui = repo.ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
124 self.ui = ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
125 baseurl, authinfo = url.authinfo()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
126 self.baseurl = baseurl.rstrip('/')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
127 self.urlopener = urlmod.opener(ui, authinfo)
35123
07e97998d385 lfs: register config options
Matt Harbison <matt_harbison@yahoo.com>
parents: 35122
diff changeset
128 self.retry = ui.configint('lfs', 'retry')
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
129
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
130 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
131 """Batch upload from local to remote blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
132 self._batch(pointers, fromstore, 'upload')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
133
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
134 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
135 """Batch download from remote to local blostore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
136 self._batch(pointers, tostore, 'download')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
137
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
138 def _batchrequest(self, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
139 """Get metadata about objects pointed by pointers for given action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
140
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
141 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
142 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
143 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
144 objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
145 requestdata = json.dumps({
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
146 'objects': objects,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
147 'operation': action,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
148 })
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
149 batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
150 data=requestdata)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
151 batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
152 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
153 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
154 rawjson = self.urlopener.open(batchreq).read()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
155 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
156 raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
157 % (ex, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
158 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
159 response = json.loads(rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
160 except ValueError:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
161 raise LfsRemoteError(_('LFS server returns invalid JSON: %s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
162 % rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
163 return response
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
164
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
165 def _checkforservererror(self, pointers, responses):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
166 """Scans errors from objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
167
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
168 Returns LfsRemoteError if any objects has an error"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
169 for response in responses:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
170 error = response.get('error')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
171 if error:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
172 ptrmap = {p.oid(): p for p in pointers}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
173 p = ptrmap.get(response['oid'], None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
174 if error['code'] == 404 and p:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
175 filename = getattr(p, 'filename', 'unknown')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
176 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
177 _(('LFS server error. Remote object '
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
178 'for file %s not found: %r')) % (filename, response))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
179 raise LfsRemoteError(_('LFS server error: %r') % response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
180
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
181 def _extractobjects(self, response, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
182 """extract objects from response of the batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
183
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
184 response: parsed JSON object returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
185 return response['objects'] filtered by action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
186 raise if any object has an error
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
187 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
188 # Scan errors from objects - fail early
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
189 objects = response.get('objects', [])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
190 self._checkforservererror(pointers, objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
191
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
192 # Filter objects with given action. Practically, this skips uploading
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
193 # objects which exist in the server.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
194 filteredobjects = [o for o in objects if action in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
195 # But for downloading, we want all objects. Therefore missing objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
196 # should be considered an error.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
197 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
198 if len(filteredobjects) < len(objects):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
199 missing = [o.get('oid', '?')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
200 for o in objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
201 if action not in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
202 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
203 _('LFS server claims required objects do not exist:\n%s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
204 % '\n'.join(missing))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
205
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
206 return filteredobjects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
207
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
208 def _basictransfer(self, obj, action, localstore, progress=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
209 """Download or upload a single object using basic transfer protocol
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
210
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
211 obj: dict, an object description returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
212 action: string, one of ['upload', 'download']
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
213 localstore: blobstore.local
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
214
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
215 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
216 basic-transfers.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
217 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
218 oid = str(obj['oid'])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
219
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
220 href = str(obj['actions'][action].get('href'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
221 headers = obj['actions'][action].get('header', {}).items()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
222
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
223 request = util.urlreq.request(href)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
224 if action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
225 # If uploading blobs, read data from local blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
226 request.data = filewithprogress(localstore.vfs(oid), progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
227 request.get_method = lambda: 'PUT'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
228
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
229 for k, v in headers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
230 request.add_header(k, v)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
231
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
232 response = b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
233 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
234 req = self.urlopener.open(request)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
235 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
236 data = req.read(1048576)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
237 if not data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
238 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
239 if action == 'download' and progress:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
240 progress(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
241 response += data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
242 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
243 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
244 % (ex, oid, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
245
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
246 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
247 # If downloading blobs, store downloaded data to local blobstore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
248 localstore.write(oid, response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
249
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
250 def _batch(self, pointers, localstore, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
251 if action not in ['upload', 'download']:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
252 raise error.ProgrammingError('invalid Git-LFS action: %s' % action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
253
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
254 response = self._batchrequest(pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
255 prunningsize = [0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
256 objects = self._extractobjects(response, pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
257 total = sum(x.get('size', 0) for x in objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
258 topic = {'upload': _('lfs uploading'),
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
259 'download': _('lfs downloading')}[action]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
260 if self.ui.verbose and len(objects) > 1:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
261 self.ui.write(_('lfs: need to transfer %d objects (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
262 % (len(objects), util.bytecount(total)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
263 self.ui.progress(topic, 0, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
264 def progress(size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
265 # advance progress bar by "size" bytes
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
266 prunningsize[0] += size
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
267 self.ui.progress(topic, prunningsize[0], total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
268 for obj in sorted(objects, key=lambda o: o.get('oid')):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
269 objsize = obj.get('size', 0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
270 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
271 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
272 msg = _('lfs: downloading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
273 elif action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
274 msg = _('lfs: uploading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
275 self.ui.write(msg % (obj.get('oid'), util.bytecount(objsize)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
276 origrunningsize = prunningsize[0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
277 retry = self.retry
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
278 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
279 prunningsize[0] = origrunningsize
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
280 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
281 self._basictransfer(obj, action, localstore,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
282 progress=progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
283 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
284 except Exception as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
285 if retry > 0:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
286 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
287 self.ui.write(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
288 _('lfs: failed: %r (remaining retry %d)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
289 % (ex, retry))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
290 retry -= 1
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
291 continue
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
292 raise
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
293
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
294 self.ui.progress(topic, pos=None, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
295
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
296 def __del__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
297 # copied from mercurial/httppeer.py
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
298 urlopener = getattr(self, 'urlopener', None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
299 if urlopener:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
300 for h in urlopener.handlers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
301 h.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
302 getattr(h, "close_all", lambda : None)()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
303
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
304 class _dummyremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
305 """Dummy store storing blobs to temp directory."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
306
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
307 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
308 fullpath = repo.vfs.join('lfs', url.path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
309 self.vfs = lfsvfs(fullpath)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
310
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
311 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
312 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
313 content = fromstore.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
314 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
315 fp.write(content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
316
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
317 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
318 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
319 content = self.vfs.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
320 tostore.write(p.oid(), content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
321
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
322 class _nullremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
323 """Null store storing blobs to /dev/null."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
324
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
325 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
326 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
327
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
328 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
329 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
330
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
331 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
332 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
333
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
334 class _promptremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
335 """Prompt user to set lfs.url when accessed."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
336
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
337 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
338 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
339
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
340 def writebatch(self, pointers, fromstore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
341 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
342
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
343 def readbatch(self, pointers, tostore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
344 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
345
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
346 def _prompt(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
347 raise error.Abort(_('lfs.url needs to be configured'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
348
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
349 _storemap = {
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
350 'https': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
351 'http': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
352 'file': _dummyremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
353 'null': _nullremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
354 None: _promptremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
355 }
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
356
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
357 def remote(repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
358 """remotestore factory. return a store in _storemap depending on config"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
359 defaulturl = ''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
360
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
361 # convert deprecated configs to the new url. TODO: remove this if other
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
362 # places are migrated to the new url config.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
363 # deprecated config: lfs.remotestore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
364 deprecatedstore = repo.ui.config('lfs', 'remotestore')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
365 if deprecatedstore == 'dummy':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
366 # deprecated config: lfs.remotepath
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
367 defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
368 elif deprecatedstore == 'git-lfs':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
369 # deprecated config: lfs.remoteurl
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
370 defaulturl = repo.ui.config('lfs', 'remoteurl')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
371 elif deprecatedstore == 'null':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
372 defaulturl = 'null://'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
373
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
374 url = util.url(repo.ui.config('lfs', 'url', defaulturl))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
375 scheme = url.scheme
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
376 if scheme not in _storemap:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
377 raise error.Abort(_('lfs: unknown url scheme: %s') % scheme)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
378 return _storemap[scheme](repo, url)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
379
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
380 class LfsRemoteError(error.RevlogError):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
381 pass