Mercurial > public > mercurial-scm > hg
comparison mercurial/wireprotov2server.py @ 40329:ed55a0077490
wireprotov2: implement command for retrieving raw store files
Implementing shallow clone of the changelog is hard. We want the 4.8
release to have a fast implementation of partial clone in wireprotov2. In
order to achieve fast, we can't use deltas for transferring changelog and
manifestlog data.
Per discussions at the 4.8 sprint, this commit implements a somwwhat hacky
and likely-to-be-changed-drastically-or-dropped command in wireprotov2 that
facilitates access to raw store files, namely the changelog and manifestlog.
Using this command, clients can perform a "stream clone" of sorts for just
the changelog and manifestlog. This will allow clients to fetch the changelog
and manifest revlogs, stream them to disk (which should be fast), then follow
up filesdata requests for files revision data for a particular changeset.
Differential Revision: https://phab.mercurial-scm.org/D5134
| author | Gregory Szorc <gregory.szorc@gmail.com> |
|---|---|
| date | Tue, 16 Oct 2018 21:31:21 +0200 |
| parents | 46a40bce3ae0 |
| children | abbd077965c0 |
comparison
equal
deleted
inserted
replaced
| 40328:2c55716f8a1c | 40329:ed55a0077490 |
|---|---|
| 20 encoding, | 20 encoding, |
| 21 error, | 21 error, |
| 22 match as matchmod, | 22 match as matchmod, |
| 23 narrowspec, | 23 narrowspec, |
| 24 pycompat, | 24 pycompat, |
| 25 streamclone, | |
| 26 util, | |
| 25 wireprotoframing, | 27 wireprotoframing, |
| 26 wireprototypes, | 28 wireprototypes, |
| 27 ) | 29 ) |
| 28 from .utils import ( | 30 from .utils import ( |
| 29 cborutil, | 31 cborutil, |
| 513 args[arg][b'default'] = meta['default']() | 515 args[arg][b'default'] = meta['default']() |
| 514 | 516 |
| 515 if meta['validvalues']: | 517 if meta['validvalues']: |
| 516 args[arg][b'validvalues'] = meta['validvalues'] | 518 args[arg][b'validvalues'] = meta['validvalues'] |
| 517 | 519 |
| 520 # TODO this type of check should be defined in a per-command callback. | |
| 521 if (command == b'rawstorefiledata' | |
| 522 and not streamclone.allowservergeneration(repo)): | |
| 523 continue | |
| 524 | |
| 518 caps['commands'][command] = { | 525 caps['commands'][command] = { |
| 519 'args': args, | 526 'args': args, |
| 520 'permissions': [entry.permission], | 527 'permissions': [entry.permission], |
| 521 } | 528 } |
| 522 | 529 |
| 1367 # TODO handle ui output redirection | 1374 # TODO handle ui output redirection |
| 1368 yield repo.pushkey(encoding.tolocal(namespace), | 1375 yield repo.pushkey(encoding.tolocal(namespace), |
| 1369 encoding.tolocal(key), | 1376 encoding.tolocal(key), |
| 1370 encoding.tolocal(old), | 1377 encoding.tolocal(old), |
| 1371 encoding.tolocal(new)) | 1378 encoding.tolocal(new)) |
| 1379 | |
| 1380 | |
| 1381 @wireprotocommand( | |
| 1382 'rawstorefiledata', | |
| 1383 args={ | |
| 1384 'files': { | |
| 1385 'type': 'list', | |
| 1386 'example': [b'changelog', b'manifestlog'], | |
| 1387 }, | |
| 1388 'pathfilter': { | |
| 1389 'type': 'list', | |
| 1390 'default': lambda: None, | |
| 1391 'example': {b'include': [b'path:tests']}, | |
| 1392 }, | |
| 1393 }, | |
| 1394 permission='pull') | |
| 1395 def rawstorefiledata(repo, proto, files, pathfilter): | |
| 1396 if not streamclone.allowservergeneration(repo): | |
| 1397 raise error.WireprotoCommandError(b'stream clone is disabled') | |
| 1398 | |
| 1399 # TODO support dynamically advertising what store files "sets" are | |
| 1400 # available. For now, we support changelog, manifestlog, and files. | |
| 1401 files = set(files) | |
| 1402 allowedfiles = {b'changelog', b'manifestlog'} | |
| 1403 | |
| 1404 unsupported = files - allowedfiles | |
| 1405 if unsupported: | |
| 1406 raise error.WireprotoCommandError(b'unknown file type: %s', | |
| 1407 (b', '.join(sorted(unsupported)),)) | |
| 1408 | |
| 1409 with repo.lock(): | |
| 1410 topfiles = list(repo.store.topfiles()) | |
| 1411 | |
| 1412 sendfiles = [] | |
| 1413 totalsize = 0 | |
| 1414 | |
| 1415 # TODO this is a bunch of storage layer interface abstractions because | |
| 1416 # it assumes revlogs. | |
| 1417 for name, encodedname, size in topfiles: | |
| 1418 if b'changelog' in files and name.startswith(b'00changelog'): | |
| 1419 pass | |
| 1420 elif b'manifestlog' in files and name.startswith(b'00manifest'): | |
| 1421 pass | |
| 1422 else: | |
| 1423 continue | |
| 1424 | |
| 1425 sendfiles.append((b'store', name, size)) | |
| 1426 totalsize += size | |
| 1427 | |
| 1428 yield { | |
| 1429 b'filecount': len(sendfiles), | |
| 1430 b'totalsize': totalsize, | |
| 1431 } | |
| 1432 | |
| 1433 for location, name, size in sendfiles: | |
| 1434 yield { | |
| 1435 b'location': location, | |
| 1436 b'path': name, | |
| 1437 b'size': size, | |
| 1438 } | |
| 1439 | |
| 1440 # We have to use a closure for this to ensure the context manager is | |
| 1441 # closed only after sending the final chunk. | |
| 1442 def getfiledata(): | |
| 1443 with repo.svfs(name, 'rb', auditpath=False) as fh: | |
| 1444 for chunk in util.filechunkiter(fh, limit=size): | |
| 1445 yield chunk | |
| 1446 | |
| 1447 yield wireprototypes.indefinitebytestringresponse( | |
| 1448 getfiledata()) |