Mercurial > public > mercurial-scm > hg-stable
annotate hgext/lfs/blobstore.py @ 35477:02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
The following corruption related patches were written prior to adding the user
level cache, and it took awhile to track down why the tests changed. (It
generally made things more resilient.) But I think this will be useful to the
end user as well. I didn't make it --debug level, because there can be a ton of
info coming out of clone/push/pull --debug. The pointers are sorted for test
stability.
I opted for ui.note() instead of checking ui.verbose and then using ui.write()
for convenience, but I see most of this extension does the latter. I have no
idea what the preferred form is.
author | Matt Harbison <matt_harbison@yahoo.com> |
---|---|
date | Tue, 19 Dec 2017 17:53:44 -0500 |
parents | e333d27514b0 |
children | b0c01a5ee35c |
rev | line source |
---|---|
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
2 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
3 # Copyright 2017 Facebook, Inc. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
4 # |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms of the |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
6 # GNU General Public License version 2 or any later version. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
7 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
8 from __future__ import absolute_import |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
9 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
10 import json |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
11 import os |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
12 import re |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
13 |
35122
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35121
diff
changeset
|
14 from mercurial.i18n import _ |
b8e5fb8d2389
lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents:
35121
diff
changeset
|
15 |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
16 from mercurial import ( |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
17 error, |
35371
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
18 pathutil, |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
19 url as urlmod, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
20 util, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
21 vfs as vfsmod, |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
22 worker, |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
23 ) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
24 |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
25 from ..largefiles import lfutil |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
26 |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
27 # 64 bytes for SHA256 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
28 _lfsre = re.compile(r'\A[a-f0-9]{64}\Z') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
29 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
30 class lfsvfs(vfsmod.vfs): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
31 def join(self, path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
32 """split the path at first two characters, like: XX/XXXXX...""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
33 if not _lfsre.match(path): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
34 raise error.ProgrammingError('unexpected lfs path: %s' % path) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
35 return super(lfsvfs, self).join(path[0:2], path[2:]) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
36 |
35371
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
37 def walk(self, path=None, onerror=None): |
35403
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35371
diff
changeset
|
38 """Yield (dirpath, [], oids) tuple for blobs under path |
35371
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
39 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
40 Oids only exist in the root of this vfs, so dirpath is always ''. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
41 """ |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
42 root = os.path.normpath(self.base) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
43 # when dirpath == root, dirpath[prefixlen:] becomes empty |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
44 # because len(dirpath) < prefixlen. |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
45 prefixlen = len(pathutil.normasprefix(root)) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
46 oids = [] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
47 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
48 for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''), |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
49 onerror=onerror): |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
50 dirpath = dirpath[prefixlen:] |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
51 |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
52 # Silently skip unexpected files and directories |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
53 if len(dirpath) == 2: |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
54 oids.extend([dirpath + f for f in files |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
55 if _lfsre.match(dirpath + f)]) |
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
56 |
35403
c8edeb03ca94
lfs: correct the directory list value returned by lfsvfs.walk()
Matt Harbison <matt_harbison@yahoo.com>
parents:
35371
diff
changeset
|
57 yield ('', [], oids) |
35371
79968f91ad0c
lfs: override walk() in lfsvfs
Matt Harbison <matt_harbison@yahoo.com>
parents:
35289
diff
changeset
|
58 |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
59 class filewithprogress(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
60 """a file-like object that supports __len__ and read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
61 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
62 Useful to provide progress information for how many bytes are read. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
63 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
64 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
65 def __init__(self, fp, callback): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
66 self._fp = fp |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
67 self._callback = callback # func(readsize) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
68 fp.seek(0, os.SEEK_END) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
69 self._len = fp.tell() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
70 fp.seek(0) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
71 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
72 def __len__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
73 return self._len |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
74 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
75 def read(self, size): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
76 if self._fp is None: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
77 return b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
78 data = self._fp.read(size) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
79 if data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
80 if self._callback: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
81 self._callback(len(data)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
82 else: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
83 self._fp.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
84 self._fp = None |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
85 return data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
86 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
87 class local(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
88 """Local blobstore for large file contents. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
89 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
90 This blobstore is used both as a cache and as a staging area for large blobs |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
91 to be uploaded to the remote blobstore. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
92 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
93 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
94 def __init__(self, repo): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
95 fullpath = repo.svfs.join('lfs/objects') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
96 self.vfs = lfsvfs(fullpath) |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
97 usercache = lfutil._usercachedir(repo.ui, 'lfs') |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
98 self.cachevfs = lfsvfs(usercache) |
35477
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35444
diff
changeset
|
99 self.ui = repo.ui |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
100 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
101 def write(self, oid, data): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
102 """Write blob to local blobstore.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
103 with self.vfs(oid, 'wb', atomictemp=True) as fp: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
104 fp.write(data) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
105 |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
106 # XXX: should we verify the content of the cache, and hardlink back to |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
107 # the local store on success, but truncate, write and link on failure? |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
108 if not self.cachevfs.exists(oid): |
35477
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35444
diff
changeset
|
109 self.ui.note(_('lfs: adding %s to the usercache\n') % oid) |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
110 lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
111 |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
112 def read(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
113 """Read blob from local blobstore.""" |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
114 if not self.vfs.exists(oid): |
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
115 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) |
35477
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35444
diff
changeset
|
116 self.ui.note(_('lfs: found %s in the usercache\n') % oid) |
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35444
diff
changeset
|
117 else: |
02f54a1ec9eb
lfs: add note messages indicating what store holds the lfs blob
Matt Harbison <matt_harbison@yahoo.com>
parents:
35444
diff
changeset
|
118 self.ui.note(_('lfs: found %s in the local lfs store\n') % oid) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
119 return self.vfs.read(oid) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
120 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
121 def has(self, oid): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
122 """Returns True if the local blobstore contains the requested blob, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
123 False otherwise.""" |
35289
8e72f9152c4d
lfs: introduce a user level cache for lfs files
Matt Harbison <matt_harbison@yahoo.com>
parents:
35123
diff
changeset
|
124 return self.cachevfs.exists(oid) or self.vfs.exists(oid) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
125 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
126 class _gitlfsremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
127 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
128 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
129 ui = repo.ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
130 self.ui = ui |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
131 baseurl, authinfo = url.authinfo() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
132 self.baseurl = baseurl.rstrip('/') |
35444
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35443
diff
changeset
|
133 useragent = repo.ui.config('experimental', 'lfs.user-agent') |
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35443
diff
changeset
|
134 if not useragent: |
e333d27514b0
lfs: add an experimental config to override User-Agent for the blob transfer
Matt Harbison <matt_harbison@yahoo.com>
parents:
35443
diff
changeset
|
135 useragent = 'mercurial/%s git/2.15.1' % util.version() |
35443
e7bb5fc4570c
lfs: add git to the User-Agent header for blob transfers
Matt Harbison <matt_harbison@yahoo.com>
parents:
35437
diff
changeset
|
136 self.urlopener = urlmod.opener(ui, authinfo, useragent) |
35123
07e97998d385
lfs: register config options
Matt Harbison <matt_harbison@yahoo.com>
parents:
35122
diff
changeset
|
137 self.retry = ui.configint('lfs', 'retry') |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
138 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
139 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
140 """Batch upload from local to remote blobstore.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
141 self._batch(pointers, fromstore, 'upload') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
142 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
143 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
144 """Batch download from remote to local blostore.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
145 self._batch(pointers, tostore, 'download') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
146 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
147 def _batchrequest(self, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
148 """Get metadata about objects pointed by pointers for given action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
149 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
150 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
151 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
152 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
153 objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
154 requestdata = json.dumps({ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
155 'objects': objects, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
156 'operation': action, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
157 }) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
158 batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
159 data=requestdata) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
160 batchreq.add_header('Accept', 'application/vnd.git-lfs+json') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
161 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
162 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
163 rawjson = self.urlopener.open(batchreq).read() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
164 except util.urlerr.httperror as ex: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
165 raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
166 % (ex, action)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
167 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
168 response = json.loads(rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
169 except ValueError: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
170 raise LfsRemoteError(_('LFS server returns invalid JSON: %s') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
171 % rawjson) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
172 return response |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
173 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
174 def _checkforservererror(self, pointers, responses): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
175 """Scans errors from objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
176 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
177 Returns LfsRemoteError if any objects has an error""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
178 for response in responses: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
179 error = response.get('error') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
180 if error: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
181 ptrmap = {p.oid(): p for p in pointers} |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
182 p = ptrmap.get(response['oid'], None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
183 if error['code'] == 404 and p: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
184 filename = getattr(p, 'filename', 'unknown') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
185 raise LfsRemoteError( |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
186 _(('LFS server error. Remote object ' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
187 'for file %s not found: %r')) % (filename, response)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
188 raise LfsRemoteError(_('LFS server error: %r') % response) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
189 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
190 def _extractobjects(self, response, pointers, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
191 """extract objects from response of the batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
192 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
193 response: parsed JSON object returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
194 return response['objects'] filtered by action |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
195 raise if any object has an error |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
196 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
197 # Scan errors from objects - fail early |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
198 objects = response.get('objects', []) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
199 self._checkforservererror(pointers, objects) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
200 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
201 # Filter objects with given action. Practically, this skips uploading |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
202 # objects which exist in the server. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
203 filteredobjects = [o for o in objects if action in o.get('actions', [])] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
204 # But for downloading, we want all objects. Therefore missing objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
205 # should be considered an error. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
206 if action == 'download': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
207 if len(filteredobjects) < len(objects): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
208 missing = [o.get('oid', '?') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
209 for o in objects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
210 if action not in o.get('actions', [])] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
211 raise LfsRemoteError( |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
212 _('LFS server claims required objects do not exist:\n%s') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
213 % '\n'.join(missing)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
214 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
215 return filteredobjects |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
216 |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
217 def _basictransfer(self, obj, action, localstore): |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
218 """Download or upload a single object using basic transfer protocol |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
219 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
220 obj: dict, an object description returned by batch API |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
221 action: string, one of ['upload', 'download'] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
222 localstore: blobstore.local |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
223 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
224 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
225 basic-transfers.md |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
226 """ |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
227 oid = str(obj['oid']) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
228 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
229 href = str(obj['actions'][action].get('href')) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
230 headers = obj['actions'][action].get('header', {}).items() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
231 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
232 request = util.urlreq.request(href) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
233 if action == 'upload': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
234 # If uploading blobs, read data from local blobstore. |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
235 request.data = filewithprogress(localstore.vfs(oid), None) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
236 request.get_method = lambda: 'PUT' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
237 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
238 for k, v in headers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
239 request.add_header(k, v) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
240 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
241 response = b'' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
242 try: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
243 req = self.urlopener.open(request) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
244 while True: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
245 data = req.read(1048576) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
246 if not data: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
247 break |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
248 response += data |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
249 except util.urlerr.httperror as ex: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
250 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
251 % (ex, oid, action)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
252 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
253 if action == 'download': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
254 # If downloading blobs, store downloaded data to local blobstore |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
255 localstore.write(oid, response) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
256 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
257 def _batch(self, pointers, localstore, action): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
258 if action not in ['upload', 'download']: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
259 raise error.ProgrammingError('invalid Git-LFS action: %s' % action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
260 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
261 response = self._batchrequest(pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
262 objects = self._extractobjects(response, pointers, action) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
263 total = sum(x.get('size', 0) for x in objects) |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
264 sizes = {} |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
265 for obj in objects: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
266 sizes[obj.get('oid')] = obj.get('size', 0) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
267 topic = {'upload': _('lfs uploading'), |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
268 'download': _('lfs downloading')}[action] |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
269 if self.ui.verbose and len(objects) > 1: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
270 self.ui.write(_('lfs: need to transfer %d objects (%s)\n') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
271 % (len(objects), util.bytecount(total))) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
272 self.ui.progress(topic, 0, total=total) |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
273 def transfer(chunk): |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
274 for obj in chunk: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
275 objsize = obj.get('size', 0) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
276 if self.ui.verbose: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
277 if action == 'download': |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
278 msg = _('lfs: downloading %s (%s)\n') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
279 elif action == 'upload': |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
280 msg = _('lfs: uploading %s (%s)\n') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
281 self.ui.write(msg % (obj.get('oid'), |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
282 util.bytecount(objsize))) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
283 retry = self.retry |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
284 while True: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
285 try: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
286 self._basictransfer(obj, action, localstore) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
287 yield 1, obj.get('oid') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
288 break |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
289 except Exception as ex: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
290 if retry > 0: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
291 if self.ui.verbose: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
292 self.ui.write( |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
293 _('lfs: failed: %r (remaining retry %d)\n') |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
294 % (ex, retry)) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
295 retry -= 1 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
296 continue |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
297 raise |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
298 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
299 oids = worker.worker(self.ui, 0.1, transfer, (), |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
300 sorted(objects, key=lambda o: o.get('oid'))) |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
301 processed = 0 |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
302 for _one, oid in oids: |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
303 processed += sizes[oid] |
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
304 self.ui.progress(topic, processed, total=total) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
305 if self.ui.verbose: |
35437
f98fac24b757
lfs: using workers in lfs prefetch
Wojciech Lis <wlis@fb.com>
parents:
35403
diff
changeset
|
306 self.ui.write(_('lfs: processed: %s\n') % oid) |
35121
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
307 self.ui.progress(topic, pos=None, total=total) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
308 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
309 def __del__(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
310 # copied from mercurial/httppeer.py |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
311 urlopener = getattr(self, 'urlopener', None) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
312 if urlopener: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
313 for h in urlopener.handlers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
314 h.close() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
315 getattr(h, "close_all", lambda : None)() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
316 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
317 class _dummyremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
318 """Dummy store storing blobs to temp directory.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
319 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
320 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
321 fullpath = repo.vfs.join('lfs', url.path) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
322 self.vfs = lfsvfs(fullpath) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
323 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
324 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
325 for p in pointers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
326 content = fromstore.read(p.oid()) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
327 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
328 fp.write(content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
329 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
330 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
331 for p in pointers: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
332 content = self.vfs.read(p.oid()) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
333 tostore.write(p.oid(), content) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
334 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
335 class _nullremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
336 """Null store storing blobs to /dev/null.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
337 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
338 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
339 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
340 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
341 def writebatch(self, pointers, fromstore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
342 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
343 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
344 def readbatch(self, pointers, tostore): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
345 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
346 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
347 class _promptremote(object): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
348 """Prompt user to set lfs.url when accessed.""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
349 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
350 def __init__(self, repo, url): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
351 pass |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
352 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
353 def writebatch(self, pointers, fromstore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
354 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
355 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
356 def readbatch(self, pointers, tostore, ui=None): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
357 self._prompt() |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
358 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
359 def _prompt(self): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
360 raise error.Abort(_('lfs.url needs to be configured')) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
361 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
362 _storemap = { |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
363 'https': _gitlfsremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
364 'http': _gitlfsremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
365 'file': _dummyremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
366 'null': _nullremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
367 None: _promptremote, |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
368 } |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
369 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
370 def remote(repo): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
371 """remotestore factory. return a store in _storemap depending on config""" |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
372 defaulturl = '' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
373 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
374 # convert deprecated configs to the new url. TODO: remove this if other |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
375 # places are migrated to the new url config. |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
376 # deprecated config: lfs.remotestore |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
377 deprecatedstore = repo.ui.config('lfs', 'remotestore') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
378 if deprecatedstore == 'dummy': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
379 # deprecated config: lfs.remotepath |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
380 defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
381 elif deprecatedstore == 'git-lfs': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
382 # deprecated config: lfs.remoteurl |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
383 defaulturl = repo.ui.config('lfs', 'remoteurl') |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
384 elif deprecatedstore == 'null': |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
385 defaulturl = 'null://' |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
386 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
387 url = util.url(repo.ui.config('lfs', 'url', defaulturl)) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
388 scheme = url.scheme |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
389 if scheme not in _storemap: |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
390 raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
391 return _storemap[scheme](repo, url) |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
392 |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
393 class LfsRemoteError(error.RevlogError): |
66c5a8cf2868
lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff
changeset
|
394 pass |