Mercurial > public > mercurial-scm > hg-stable

annotate hgext/lfs/blobstore.py @ 35122:b8e5fb8d2389

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lfs: quiesce check-module-import warnings Specifically, 'symbol import follows non-symbol import: mercurial.i18n'
author Matt Harbison <matt_harbison@yahoo.com>
date Tue, 14 Nov 2017 00:14:52 -0500
parents 66c5a8cf2868
children 07e97998d385
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
1 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
2 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
3 # Copyright 2017 Facebook, Inc.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
4 #
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
5 # This software may be used and distributed according to the terms of the
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
6 # GNU General Public License version 2 or any later version.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
7
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
8 from __future__ import absolute_import
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
9
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
10 import json
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
11 import os
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
12 import re
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
13
35122
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35121
diff changeset
14 from mercurial.i18n import _
b8e5fb8d2389 lfs: quiesce check-module-import warnings
Matt Harbison <matt_harbison@yahoo.com>
parents: 35121
diff changeset
15
35121
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
16 from mercurial import (
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
17 error,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
18 url as urlmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
19 util,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
20 vfs as vfsmod,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
21 )
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
22
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
23 # 64 bytes for SHA256
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
24 _lfsre = re.compile(r'\A[a-f0-9]{64}\Z')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
25
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
26 class lfsvfs(vfsmod.vfs):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
27 def join(self, path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
28 """split the path at first two characters, like: XX/XXXXX..."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
29 if not _lfsre.match(path):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
30 raise error.ProgrammingError('unexpected lfs path: %s' % path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
31 return super(lfsvfs, self).join(path[0:2], path[2:])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
32
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
33 class filewithprogress(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
34 """a file-like object that supports __len__ and read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
35
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
36 Useful to provide progress information for how many bytes are read.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
37 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
38
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
39 def __init__(self, fp, callback):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
40 self._fp = fp
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
41 self._callback = callback # func(readsize)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
42 fp.seek(0, os.SEEK_END)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
43 self._len = fp.tell()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
44 fp.seek(0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
45
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
46 def __len__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
47 return self._len
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
48
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
49 def read(self, size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
50 if self._fp is None:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
51 return b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
52 data = self._fp.read(size)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
53 if data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
54 if self._callback:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
55 self._callback(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
56 else:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
57 self._fp.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
58 self._fp = None
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
59 return data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
60
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
61 class local(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
62 """Local blobstore for large file contents.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
63
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
64 This blobstore is used both as a cache and as a staging area for large blobs
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
65 to be uploaded to the remote blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
66 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
67
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
68 def __init__(self, repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
69 fullpath = repo.svfs.join('lfs/objects')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
70 self.vfs = lfsvfs(fullpath)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
71
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
72 def write(self, oid, data):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
73 """Write blob to local blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
74 with self.vfs(oid, 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
75 fp.write(data)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
76
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
77 def read(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
78 """Read blob from local blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
79 return self.vfs.read(oid)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
80
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
81 def has(self, oid):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
82 """Returns True if the local blobstore contains the requested blob,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
83 False otherwise."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
84 return self.vfs.exists(oid)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
85
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
86 class _gitlfsremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
87
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
88 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
89 ui = repo.ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
90 self.ui = ui
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
91 baseurl, authinfo = url.authinfo()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
92 self.baseurl = baseurl.rstrip('/')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
93 self.urlopener = urlmod.opener(ui, authinfo)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
94 self.retry = ui.configint('lfs', 'retry', 5)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
95
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
96 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
97 """Batch upload from local to remote blobstore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
98 self._batch(pointers, fromstore, 'upload')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
99
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
100 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
101 """Batch download from remote to local blostore."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
102 self._batch(pointers, tostore, 'download')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
103
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
104 def _batchrequest(self, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
105 """Get metadata about objects pointed by pointers for given action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
106
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
107 Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
108 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
109 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
110 objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
111 requestdata = json.dumps({
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
112 'objects': objects,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
113 'operation': action,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
114 })
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
115 batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
116 data=requestdata)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
117 batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
118 batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
119 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
120 rawjson = self.urlopener.open(batchreq).read()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
121 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
122 raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
123 % (ex, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
124 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
125 response = json.loads(rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
126 except ValueError:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
127 raise LfsRemoteError(_('LFS server returns invalid JSON: %s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
128 % rawjson)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
129 return response
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
130
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
131 def _checkforservererror(self, pointers, responses):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
132 """Scans errors from objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
133
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
134 Returns LfsRemoteError if any objects has an error"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
135 for response in responses:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
136 error = response.get('error')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
137 if error:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
138 ptrmap = {p.oid(): p for p in pointers}
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
139 p = ptrmap.get(response['oid'], None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
140 if error['code'] == 404 and p:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
141 filename = getattr(p, 'filename', 'unknown')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
142 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
143 _(('LFS server error. Remote object '
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
144 'for file %s not found: %r')) % (filename, response))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
145 raise LfsRemoteError(_('LFS server error: %r') % response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
146
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
147 def _extractobjects(self, response, pointers, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
148 """extract objects from response of the batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
149
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
150 response: parsed JSON object returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
151 return response['objects'] filtered by action
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
152 raise if any object has an error
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
153 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
154 # Scan errors from objects - fail early
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
155 objects = response.get('objects', [])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
156 self._checkforservererror(pointers, objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
157
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
158 # Filter objects with given action. Practically, this skips uploading
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
159 # objects which exist in the server.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
160 filteredobjects = [o for o in objects if action in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
161 # But for downloading, we want all objects. Therefore missing objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
162 # should be considered an error.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
163 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
164 if len(filteredobjects) < len(objects):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
165 missing = [o.get('oid', '?')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
166 for o in objects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
167 if action not in o.get('actions', [])]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
168 raise LfsRemoteError(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
169 _('LFS server claims required objects do not exist:\n%s')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
170 % '\n'.join(missing))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
171
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
172 return filteredobjects
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
173
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
174 def _basictransfer(self, obj, action, localstore, progress=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
175 """Download or upload a single object using basic transfer protocol
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
176
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
177 obj: dict, an object description returned by batch API
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
178 action: string, one of ['upload', 'download']
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
179 localstore: blobstore.local
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
180
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
181 See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
182 basic-transfers.md
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
183 """
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
184 oid = str(obj['oid'])
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
185
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
186 href = str(obj['actions'][action].get('href'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
187 headers = obj['actions'][action].get('header', {}).items()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
188
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
189 request = util.urlreq.request(href)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
190 if action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
191 # If uploading blobs, read data from local blobstore.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
192 request.data = filewithprogress(localstore.vfs(oid), progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
193 request.get_method = lambda: 'PUT'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
194
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
195 for k, v in headers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
196 request.add_header(k, v)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
197
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
198 response = b''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
199 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
200 req = self.urlopener.open(request)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
201 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
202 data = req.read(1048576)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
203 if not data:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
204 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
205 if action == 'download' and progress:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
206 progress(len(data))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
207 response += data
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
208 except util.urlerr.httperror as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
209 raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
210 % (ex, oid, action))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
211
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
212 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
213 # If downloading blobs, store downloaded data to local blobstore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
214 localstore.write(oid, response)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
215
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
216 def _batch(self, pointers, localstore, action):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
217 if action not in ['upload', 'download']:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
218 raise error.ProgrammingError('invalid Git-LFS action: %s' % action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
219
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
220 response = self._batchrequest(pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
221 prunningsize = [0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
222 objects = self._extractobjects(response, pointers, action)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
223 total = sum(x.get('size', 0) for x in objects)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
224 topic = {'upload': _('lfs uploading'),
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
225 'download': _('lfs downloading')}[action]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
226 if self.ui.verbose and len(objects) > 1:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
227 self.ui.write(_('lfs: need to transfer %d objects (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
228 % (len(objects), util.bytecount(total)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
229 self.ui.progress(topic, 0, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
230 def progress(size):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
231 # advance progress bar by "size" bytes
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
232 prunningsize[0] += size
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
233 self.ui.progress(topic, prunningsize[0], total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
234 for obj in sorted(objects, key=lambda o: o.get('oid')):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
235 objsize = obj.get('size', 0)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
236 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
237 if action == 'download':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
238 msg = _('lfs: downloading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
239 elif action == 'upload':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
240 msg = _('lfs: uploading %s (%s)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
241 self.ui.write(msg % (obj.get('oid'), util.bytecount(objsize)))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
242 origrunningsize = prunningsize[0]
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
243 retry = self.retry
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
244 while True:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
245 prunningsize[0] = origrunningsize
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
246 try:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
247 self._basictransfer(obj, action, localstore,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
248 progress=progress)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
249 break
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
250 except Exception as ex:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
251 if retry > 0:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
252 if self.ui.verbose:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
253 self.ui.write(
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
254 _('lfs: failed: %r (remaining retry %d)\n')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
255 % (ex, retry))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
256 retry -= 1
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
257 continue
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
258 raise
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
259
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
260 self.ui.progress(topic, pos=None, total=total)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
261
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
262 def __del__(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
263 # copied from mercurial/httppeer.py
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
264 urlopener = getattr(self, 'urlopener', None)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
265 if urlopener:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
266 for h in urlopener.handlers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
267 h.close()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
268 getattr(h, "close_all", lambda : None)()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
269
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
270 class _dummyremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
271 """Dummy store storing blobs to temp directory."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
272
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
273 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
274 fullpath = repo.vfs.join('lfs', url.path)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
275 self.vfs = lfsvfs(fullpath)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
276
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
277 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
278 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
279 content = fromstore.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
280 with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
281 fp.write(content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
282
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
283 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
284 for p in pointers:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
285 content = self.vfs.read(p.oid())
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
286 tostore.write(p.oid(), content)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
287
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
288 class _nullremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
289 """Null store storing blobs to /dev/null."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
290
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
291 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
292 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
293
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
294 def writebatch(self, pointers, fromstore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
295 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
296
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
297 def readbatch(self, pointers, tostore):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
298 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
299
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
300 class _promptremote(object):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
301 """Prompt user to set lfs.url when accessed."""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
302
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
303 def __init__(self, repo, url):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
304 pass
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
305
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
306 def writebatch(self, pointers, fromstore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
307 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
308
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
309 def readbatch(self, pointers, tostore, ui=None):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
310 self._prompt()
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
311
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
312 def _prompt(self):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
313 raise error.Abort(_('lfs.url needs to be configured'))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
314
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
315 _storemap = {
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
316 'https': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
317 'http': _gitlfsremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
318 'file': _dummyremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
319 'null': _nullremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
320 None: _promptremote,
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
321 }
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
322
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
323 def remote(repo):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
324 """remotestore factory. return a store in _storemap depending on config"""
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
325 defaulturl = ''
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
326
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
327 # convert deprecated configs to the new url. TODO: remove this if other
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
328 # places are migrated to the new url config.
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
329 # deprecated config: lfs.remotestore
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
330 deprecatedstore = repo.ui.config('lfs', 'remotestore')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
331 if deprecatedstore == 'dummy':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
332 # deprecated config: lfs.remotepath
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
333 defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
334 elif deprecatedstore == 'git-lfs':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
335 # deprecated config: lfs.remoteurl
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
336 defaulturl = repo.ui.config('lfs', 'remoteurl')
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
337 elif deprecatedstore == 'null':
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
338 defaulturl = 'null://'
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
339
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
340 url = util.url(repo.ui.config('lfs', 'url', defaulturl))
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
341 scheme = url.scheme
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
342 if scheme not in _storemap:
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
343 raise error.Abort(_('lfs: unknown url scheme: %s') % scheme)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
344 return _storemap[scheme](repo, url)
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
345
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
346 class LfsRemoteError(error.RevlogError):
66c5a8cf2868 lfs: import the Facebook git-lfs client extension
Matt Harbison <matt_harbison@yahoo.com>
parents:
diff changeset
347 pass