Mercurial > public > mercurial-scm > hg

annotate hgext/lfs/blobstore.py @ 35100:07e97998d385

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lfs: register config options I'm not sure at what point we can get rid of the deprecated options, but for the sake of making progress, they are registered too.
author Matt Harbison <matt_harbison@yahoo.com>
date Tue, 14 Nov 2017 01:03:22 -0500
parents b8e5fb8d2389
children 8e72f9152c4d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
35098
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
2 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
3 # Copyright 2017 Facebook, Inc.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
4 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
5 # This software may be used and distributed according to the terms of the
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
6 # GNU General Public License version 2 or any later version.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
7
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
8 from __future__ import absolute_import
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
9
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
10 import json
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
11 import os
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
12 import re
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
13
35099
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35098
diff changeset
14 from mercurial.i18n import _
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35098
diff changeset
15
35098
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
16 from mercurial import (
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
17 error,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
18 url as urlmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
19 util,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
20 vfs as vfsmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
21 )
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
22
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
23 # 64 bytes for SHA256
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
24 _lfsre = re.compile(r'\A[a-f0-9]{64}\Z')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
25
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
26 class lfsvfs(vfsmod.vfs):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
27 def join(self, path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
28 """split the path at first two characters, like: XX/XXXXX..."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
29 if not _lfsre.match(path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
30 raise error.ProgrammingError('unexpected lfs path: %s' % path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
31 return super(lfsvfs, self).join(path[0:2], path[2:])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
32
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
33 class filewithprogress(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
34 """a file-like object that supports __len__ and read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
35
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
36 Useful to provide progress information for how many bytes are read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
37 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
38
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
39 def __init__(self, fp, callback):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
40 self._fp = fp
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
41 self._callback = callback # func(readsize)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
42 fp.seek(0, os.SEEK_END)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
43 self._len = fp.tell()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
44 fp.seek(0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
45
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
46 def __len__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
47 return self._len
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
48
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
49 def read(self, size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
50 if self._fp is None:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
51 return b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
52 data = self._fp.read(size)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
53 if data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
54 if self._callback:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
55 self._callback(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
56 else:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
57 self._fp.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
58 self._fp = None
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
59 return data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
60
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
61 class local(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
62 """Local blobstore for large file contents.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
63
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
64 This blobstore is used both as a cache and as a staging area for large blobs
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
65 to be uploaded to the remote blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
66 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
67
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
68 def __init__(self, repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
69 fullpath = repo.svfs.join('lfs/objects')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
70 self.vfs = lfsvfs(fullpath)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
71
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
72 def write(self, oid, data):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
73 """Write blob to local blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
74 with self.vfs(oid, 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
75 fp.write(data)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
76
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
77 def read(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
78 """Read blob from local blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
79 return self.vfs.read(oid)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
80
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
81 def has(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
82 """Returns True if the local blobstore contains the requested blob,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
83 False otherwise."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
84 return self.vfs.exists(oid)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
85
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
86 class _gitlfsremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
87
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
88 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
89 ui = repo.ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
90 self.ui = ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
91 baseurl, authinfo = url.authinfo()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
92 self.baseurl = baseurl.rstrip('/')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
93 self.urlopener = urlmod.opener(ui, authinfo)
35100
07e97998d385 lfs: register config options
Matt Harbison <matt_harbison@yahoo.com>
parents: 35099
diff changeset
94 self.retry = ui.configint('lfs', 'retry')
35098
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
95
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
96 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
97 """Batch upload from local to remote blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
98 self._batch(pointers, fromstore, 'upload')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
99
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
100 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
101 """Batch download from remote to local blostore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
102 self._batch(pointers, tostore, 'download')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
103
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
104 def _batchrequest(self, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
105 """Get metadata about objects pointed by pointers for given action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
106
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
107 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
108 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
109 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
110 objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
111 requestdata = json.dumps({
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
112 'objects': objects,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
113 'operation': action,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
114 })
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
115 batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
116 data=requestdata)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
117 batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
118 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
119 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
120 rawjson = self.urlopener.open(batchreq).read()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
121 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
122 raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
123 % (ex, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
124 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
125 response = json.loads(rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
126 except ValueError:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
127 raise LfsRemoteError(_('LFS server returns invalid JSON: %s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
128 % rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
129 return response
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
130
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
131 def _checkforservererror(self, pointers, responses):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
132 """Scans errors from objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
133
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
134 Returns LfsRemoteError if any objects has an error"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
135 for response in responses:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
136 error = response.get('error')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
137 if error:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
138 ptrmap = {p.oid(): p for p in pointers}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
139 p = ptrmap.get(response['oid'], None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
140 if error['code'] == 404 and p:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
141 filename = getattr(p, 'filename', 'unknown')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
142 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
143 _(('LFS server error. Remote object '
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
144 'for file %s not found: %r')) % (filename, response))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
145 raise LfsRemoteError(_('LFS server error: %r') % response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
146
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
147 def _extractobjects(self, response, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
148 """extract objects from response of the batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
149
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
150 response: parsed JSON object returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
151 return response['objects'] filtered by action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
152 raise if any object has an error
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
153 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
154 # Scan errors from objects - fail early
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
155 objects = response.get('objects', [])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
156 self._checkforservererror(pointers, objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
157
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
158 # Filter objects with given action. Practically, this skips uploading
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
159 # objects which exist in the server.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
160 filteredobjects = [o for o in objects if action in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
161 # But for downloading, we want all objects. Therefore missing objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
162 # should be considered an error.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
163 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
164 if len(filteredobjects) < len(objects):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
165 missing = [o.get('oid', '?')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
166 for o in objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
167 if action not in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
168 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
169 _('LFS server claims required objects do not exist:\n%s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
170 % '\n'.join(missing))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
171
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
172 return filteredobjects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
173
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
174 def _basictransfer(self, obj, action, localstore, progress=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
175 """Download or upload a single object using basic transfer protocol
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
176
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
177 obj: dict, an object description returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
178 action: string, one of ['upload', 'download']
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
179 localstore: blobstore.local
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
180
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
181 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
182 basic-transfers.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
183 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
184 oid = str(obj['oid'])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
185
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
186 href = str(obj['actions'][action].get('href'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
187 headers = obj['actions'][action].get('header', {}).items()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
188
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
189 request = util.urlreq.request(href)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
190 if action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
191 # If uploading blobs, read data from local blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
192 request.data = filewithprogress(localstore.vfs(oid), progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
193 request.get_method = lambda: 'PUT'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
194
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
195 for k, v in headers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
196 request.add_header(k, v)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
197
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
198 response = b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
199 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
200 req = self.urlopener.open(request)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
201 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
202 data = req.read(1048576)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
203 if not data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
204 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
205 if action == 'download' and progress:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
206 progress(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
207 response += data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
208 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
209 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
210 % (ex, oid, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
211
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
212 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
213 # If downloading blobs, store downloaded data to local blobstore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
214 localstore.write(oid, response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
215
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
216 def _batch(self, pointers, localstore, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
217 if action not in ['upload', 'download']:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
218 raise error.ProgrammingError('invalid Git-LFS action: %s' % action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
219
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
220 response = self._batchrequest(pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
221 prunningsize = [0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
222 objects = self._extractobjects(response, pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
223 total = sum(x.get('size', 0) for x in objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
224 topic = {'upload': _('lfs uploading'),
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
225 'download': _('lfs downloading')}[action]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
226 if self.ui.verbose and len(objects) > 1:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
227 self.ui.write(_('lfs: need to transfer %d objects (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
228 % (len(objects), util.bytecount(total)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
229 self.ui.progress(topic, 0, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
230 def progress(size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
231 # advance progress bar by "size" bytes
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
232 prunningsize[0] += size
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
233 self.ui.progress(topic, prunningsize[0], total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
234 for obj in sorted(objects, key=lambda o: o.get('oid')):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
235 objsize = obj.get('size', 0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
236 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
237 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
238 msg = _('lfs: downloading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
239 elif action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
240 msg = _('lfs: uploading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
241 self.ui.write(msg % (obj.get('oid'), util.bytecount(objsize)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
242 origrunningsize = prunningsize[0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
243 retry = self.retry
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
244 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
245 prunningsize[0] = origrunningsize
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
246 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
247 self._basictransfer(obj, action, localstore,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
248 progress=progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
249 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
250 except Exception as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
251 if retry > 0:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
252 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
253 self.ui.write(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
254 _('lfs: failed: %r (remaining retry %d)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
255 % (ex, retry))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
256 retry -= 1
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
257 continue
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
258 raise
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
259
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
260 self.ui.progress(topic, pos=None, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
261
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
262 def __del__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
263 # copied from mercurial/httppeer.py
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
264 urlopener = getattr(self, 'urlopener', None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
265 if urlopener:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
266 for h in urlopener.handlers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
267 h.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
268 getattr(h, "close_all", lambda : None)()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
269
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
270 class _dummyremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
271 """Dummy store storing blobs to temp directory."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
272
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
273 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
274 fullpath = repo.vfs.join('lfs', url.path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
275 self.vfs = lfsvfs(fullpath)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
276
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
277 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
278 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
279 content = fromstore.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
280 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
281 fp.write(content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
282
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
283 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
284 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
285 content = self.vfs.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
286 tostore.write(p.oid(), content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
287
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
288 class _nullremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
289 """Null store storing blobs to /dev/null."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
290
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
291 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
292 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
293
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
294 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
295 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
296
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
297 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
298 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
299
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
300 class _promptremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
301 """Prompt user to set lfs.url when accessed."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
302
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
303 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
304 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
305
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
306 def writebatch(self, pointers, fromstore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
307 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
308
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
309 def readbatch(self, pointers, tostore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
310 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
311
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
312 def _prompt(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
313 raise error.Abort(_('lfs.url needs to be configured'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
314
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
315 _storemap = {
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
316 'https': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
317 'http': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
318 'file': _dummyremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
319 'null': _nullremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
320 None: _promptremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
321 }
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
322
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
323 def remote(repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
324 """remotestore factory. return a store in _storemap depending on config"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
325 defaulturl = ''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
326
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
327 # convert deprecated configs to the new url. TODO: remove this if other
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
328 # places are migrated to the new url config.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
329 # deprecated config: lfs.remotestore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
330 deprecatedstore = repo.ui.config('lfs', 'remotestore')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
331 if deprecatedstore == 'dummy':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
332 # deprecated config: lfs.remotepath
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
333 defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
334 elif deprecatedstore == 'git-lfs':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
335 # deprecated config: lfs.remoteurl
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
336 defaulturl = repo.ui.config('lfs', 'remoteurl')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
337 elif deprecatedstore == 'null':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
338 defaulturl = 'null://'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
339
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
340 url = util.url(repo.ui.config('lfs', 'url', defaulturl))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
341 scheme = url.scheme
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
342 if scheme not in _storemap:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
343 raise error.Abort(_('lfs: unknown url scheme: %s') % scheme)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
344 return _storemap[scheme](repo, url)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
345
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
346 class LfsRemoteError(error.RevlogError):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
347 pass