Details
Details
- Reviewers
quark - Group Reviewers
Restricted Project - Commits
- rFBHGX04b9aa619610: lfs: delete now that it's in upstream
Diff Detail
Diff Detail
- Repository
- rFBHGX Facebook Mercurial Extensions
- Lint
Lint Skipped - Unit
Unit Tests Skipped
quark |
Restricted Project |
Lint Skipped |
Unit Tests Skipped |
Path | Packages | |||
---|---|---|---|---|
D | M | hgext3rd/lfs/__init__.py (126 lines) | ||
D | M | hgext3rd/lfs/blobstore.py (351 lines) | ||
D | M | hgext3rd/lfs/pointer.py (72 lines) | ||
D | M | hgext3rd/lfs/wrapper.py (256 lines) | ||
D | M | tests/test-lfs-pointer.py (41 lines) | ||
D | M | tests/test-lfs-pointer.py.out (12 lines) | ||
D | M | tests/test-lfs-test-server.t (108 lines) | ||
D | M | tests/test-lfs.t (544 lines) |
Commit | Parents | Author | Summary | Date |
---|---|---|---|---|
Martin von Zweigbergk | Jul 10 2018, 1:16 AM |
Status | Author | Revision | |
---|---|---|---|
Closed | martinvonz | ||
Closed | martinvonz |
# lfs - hash-preserving large file support using Git-LFS protocol | |||||
# | |||||
# Copyright 2017 Facebook, Inc. | |||||
# | |||||
# This software may be used and distributed according to the terms of the | |||||
# GNU General Public License version 2 or any later version. | |||||
"""lfs - large file support | |||||
Configs:: | |||||
[lfs] | |||||
# Remote endpoint. Multiple protocols are supported: | |||||
# - http(s)://user:pass@example.com/path | |||||
# git-lfs endpoint | |||||
# - file:///tmp/path | |||||
# local filesystem, usually for testing | |||||
# if unset, lfs will prompt setting this when it must use this value. | |||||
# (default: unset) | |||||
url = https://example.com/lfs | |||||
# size of a file to make it use LFS | |||||
threshold = 10M | |||||
# how many times to retry before giving up on transferring an object | |||||
retry = 5 | |||||
""" | |||||
from __future__ import absolute_import | |||||
from mercurial import ( | |||||
bundle2, | |||||
changegroup, | |||||
context, | |||||
exchange, | |||||
extensions, | |||||
filelog, | |||||
registrar, | |||||
revlog, | |||||
scmutil, | |||||
vfs as vfsmod, | |||||
) | |||||
from mercurial.i18n import _ | |||||
from . import ( | |||||
blobstore, | |||||
wrapper, | |||||
) | |||||
cmdtable = {} | |||||
command = registrar.command(cmdtable) | |||||
templatekeyword = registrar.templatekeyword() | |||||
def reposetup(ui, repo): | |||||
# Nothing to do with a remote repo | |||||
if not repo.local(): | |||||
return | |||||
threshold = repo.ui.configbytes('lfs', 'threshold', None) | |||||
repo.svfs.options['lfsthreshold'] = threshold | |||||
repo.svfs.lfslocalblobstore = blobstore.local(repo) | |||||
repo.svfs.lfsremoteblobstore = blobstore.remote(repo) | |||||
# Push hook | |||||
repo.prepushoutgoinghooks.add('lfs', wrapper.prepush) | |||||
def wrapfilelog(filelog): | |||||
wrapfunction = extensions.wrapfunction | |||||
wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) | |||||
wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) | |||||
wrapfunction(filelog, 'size', wrapper.filelogsize) | |||||
def extsetup(ui): | |||||
wrapfilelog(filelog.filelog) | |||||
wrapfunction = extensions.wrapfunction | |||||
wrapfunction(changegroup, | |||||
'supportedoutgoingversions', | |||||
wrapper.supportedoutgoingversions) | |||||
wrapfunction(changegroup, | |||||
'allsupportedversions', | |||||
wrapper.allsupportedversions) | |||||
wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp) | |||||
wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary) | |||||
context.basefilectx.islfs = wrapper.filectxislfs | |||||
revlog.addflagprocessor( | |||||
revlog.REVIDX_EXTSTORED, | |||||
( | |||||
wrapper.readfromstore, | |||||
wrapper.writetostore, | |||||
wrapper.bypasscheckhash, | |||||
), | |||||
) | |||||
# Make bundle choose changegroup3 instead of changegroup2. This affects | |||||
# "hg bundle" command. Note: it does not cover all bundle formats like | |||||
# "packed1". Using "packed1" with lfs will likely cause trouble. | |||||
names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02'] | |||||
for k in names: | |||||
exchange._bundlespeccgversions[k] = '03' | |||||
# bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs | |||||
# options and blob stores are passed from othervfs to the new readonlyvfs. | |||||
wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit) | |||||
# when writing a bundle via "hg bundle" command, upload related LFS blobs | |||||
wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle) | |||||
@templatekeyword('lfs_files') | |||||
def lfsfiles(repo, ctx, **args): | |||||
"""List of strings. LFS files added or modified by the changeset.""" | |||||
pointers = wrapper.pointersfromctx(ctx) # {path: pointer} | |||||
return sorted(pointers.keys()) | |||||
@command('debuglfsupload', | |||||
[('r', 'rev', [], _('upload large files introduced by REV'))]) | |||||
def debuglfsupload(ui, repo, **opts): | |||||
"""upload lfs blobs added by the working copy parent or given revisions""" | |||||
revs = opts.get('rev', []) | |||||
pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs)) | |||||
wrapper.uploadblobs(repo, pointers) |
# blobstore.py - local and remote (speaking Git-LFS protocol) blob storages | |||||
# | |||||
# Copyright 2017 Facebook, Inc. | |||||
# | |||||
# This software may be used and distributed according to the terms of the | |||||
# GNU General Public License version 2 or any later version. | |||||
from __future__ import absolute_import | |||||
import json | |||||
import os | |||||
import re | |||||
from mercurial import ( | |||||
error, | |||||
url as urlmod, | |||||
util, | |||||
vfs as vfsmod, | |||||
worker, | |||||
) | |||||
from mercurial.i18n import _ | |||||
# 64 bytes for SHA256 | |||||
_lfsre = re.compile(r'\A[a-f0-9]{64}\Z') | |||||
class lfsvfs(vfsmod.vfs): | |||||
def join(self, path): | |||||
"""split the path at first two characters, like: XX/XXXXX...""" | |||||
if not _lfsre.match(path): | |||||
raise error.ProgrammingError('unexpected lfs path: %s' % path) | |||||
return super(lfsvfs, self).join(path[0:2], path[2:]) | |||||
class filewithprogress(object): | |||||
"""a file-like object that supports __len__ and read. | |||||
Useful to provide progress information for how many bytes are read. | |||||
""" | |||||
def __init__(self, fp, callback): | |||||
self._fp = fp | |||||
self._callback = callback # func(readsize) | |||||
fp.seek(0, os.SEEK_END) | |||||
self._len = fp.tell() | |||||
fp.seek(0) | |||||
def __len__(self): | |||||
return self._len | |||||
def read(self, size): | |||||
if self._fp is None: | |||||
return b'' | |||||
data = self._fp.read(size) | |||||
if data: | |||||
if self._callback: | |||||
self._callback(len(data)) | |||||
else: | |||||
self._fp.close() | |||||
self._fp = None | |||||
return data | |||||
class local(object): | |||||
"""Local blobstore for large file contents. | |||||
This blobstore is used both as a cache and as a staging area for large blobs | |||||
to be uploaded to the remote blobstore. | |||||
""" | |||||
def __init__(self, repo): | |||||
fullpath = repo.svfs.join('lfs/objects') | |||||
self.vfs = lfsvfs(fullpath) | |||||
def write(self, oid, data): | |||||
"""Write blob to local blobstore.""" | |||||
with self.vfs(oid, 'wb', atomictemp=True) as fp: | |||||
fp.write(data) | |||||
def read(self, oid): | |||||
"""Read blob from local blobstore.""" | |||||
return self.vfs.read(oid) | |||||
def has(self, oid): | |||||
"""Returns True if the local blobstore contains the requested blob, | |||||
False otherwise.""" | |||||
return self.vfs.exists(oid) | |||||
class _gitlfsremote(object): | |||||
def __init__(self, repo, url): | |||||
ui = repo.ui | |||||
self.ui = ui | |||||
baseurl, authinfo = url.authinfo() | |||||
self.baseurl = baseurl.rstrip('/') | |||||
self.urlopener = urlmod.opener(ui, authinfo) | |||||
self.retry = ui.configint('lfs', 'retry') or 5 | |||||
def writebatch(self, pointers, fromstore): | |||||
"""Batch upload from local to remote blobstore.""" | |||||
self._batch(pointers, fromstore, 'upload') | |||||
def readbatch(self, pointers, tostore): | |||||
"""Batch download from remote to local blostore.""" | |||||
self._batch(pointers, tostore, 'download') | |||||
def _batchrequest(self, pointers, action): | |||||
"""Get metadata about objects pointed by pointers for given action | |||||
Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} | |||||
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md | |||||
""" | |||||
objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] | |||||
requestdata = json.dumps({ | |||||
'objects': objects, | |||||
'operation': action, | |||||
}) | |||||
batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl, | |||||
data=requestdata) | |||||
batchreq.add_header('Accept', 'application/vnd.git-lfs+json') | |||||
batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') | |||||
try: | |||||
rawjson = self.urlopener.open(batchreq).read() | |||||
except util.urlerr.httperror as ex: | |||||
raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)') | |||||
% (ex, action)) | |||||
try: | |||||
response = json.loads(rawjson) | |||||
except ValueError: | |||||
raise LfsRemoteError(_('LFS server returns invalid JSON: %s') | |||||
% rawjson) | |||||
return response | |||||
def _checkforservererror(self, pointers, responses): | |||||
"""Scans errors from objects | |||||
Returns LfsRemoteError if any objects has an error""" | |||||
for response in responses: | |||||
error = response.get('error') | |||||
if error: | |||||
ptrmap = {p.oid(): p for p in pointers} | |||||
p = ptrmap.get(response['oid'], None) | |||||
if error['code'] == 404 and p: | |||||
filename = getattr(p, 'filename', 'unknown') | |||||
raise LfsRemoteError( | |||||
_(('LFS server error. Remote object ' | |||||
'for file %s not found: %r')) % (filename, response)) | |||||
raise LfsRemoteError(_('LFS server error: %r') % response) | |||||
def _extractobjects(self, response, pointers, action): | |||||
"""extract objects from response of the batch API | |||||
response: parsed JSON object returned by batch API | |||||
return response['objects'] filtered by action | |||||
raise if any object has an error | |||||
""" | |||||
# Scan errors from objects - fail early | |||||
objects = response.get('objects', []) | |||||
self._checkforservererror(pointers, objects) | |||||
# Filter objects with given action. Practically, this skips uploading | |||||
# objects which exist in the server. | |||||
filteredobjects = [o for o in objects if action in o.get('actions', [])] | |||||
# But for downloading, we want all objects. Therefore missing objects | |||||
# should be considered an error. | |||||
if action == 'download': | |||||
if len(filteredobjects) < len(objects): | |||||
missing = [o.get('oid', '?') | |||||
for o in objects | |||||
if action not in o.get('actions', [])] | |||||
raise LfsRemoteError( | |||||
_('LFS server claims required objects do not exist:\n%s') | |||||
% '\n'.join(missing)) | |||||
return filteredobjects | |||||
def _basictransfer(self, obj, action, localstore): | |||||
"""Download or upload a single object using basic transfer protocol | |||||
obj: dict, an object description returned by batch API | |||||
action: string, one of ['upload', 'download'] | |||||
localstore: blobstore.local | |||||
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ | |||||
basic-transfers.md | |||||
""" | |||||
oid = str(obj['oid']) | |||||
href = str(obj['actions'][action].get('href')) | |||||
headers = obj['actions'][action].get('header', {}).items() | |||||
request = util.urlreq.request(href) | |||||
if action == 'upload': | |||||
# If uploading blobs, read data from local blobstore. | |||||
request.data = localstore.vfs(oid) | |||||
request.get_method = lambda: 'PUT' | |||||
for k, v in headers: | |||||
request.add_header(k, v) | |||||
response = b'' | |||||
try: | |||||
req = self.urlopener.open(request) | |||||
while True: | |||||
data = req.read(1048576) | |||||
if not data: | |||||
break | |||||
response += data | |||||
except util.urlerr.httperror as ex: | |||||
raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') | |||||
% (ex, oid, action)) | |||||
if action == 'download': | |||||
# If downloading blobs, store downloaded data to local blobstore | |||||
localstore.write(oid, response) | |||||
def _batch(self, pointers, localstore, action): | |||||
if action not in ['upload', 'download']: | |||||
raise error.ProgrammingError('invalid Git-LFS action: %s' % action) | |||||
response = self._batchrequest(pointers, action) | |||||
objects = self._extractobjects(response, pointers, action) | |||||
total = sum(x.get('size', 0) for x in objects) | |||||
sizes = {} | |||||
for obj in objects: | |||||
sizes[obj.get('oid')] = obj.get('size', 0) | |||||
topic = {'upload': _('lfs uploading'), | |||||
'download': _('lfs downloading')}[action] | |||||
if self.ui.verbose and len(objects) > 1: | |||||
self.ui.write(_('lfs: need to transfer %d objects (%s)\n') | |||||
% (len(objects), util.bytecount(total))) | |||||
self.ui.progress(topic, 0, total=total) | |||||
def transfer(chunk): | |||||
for obj in chunk: | |||||
objsize = obj.get('size', 0) | |||||
if self.ui.verbose: | |||||
if action == 'download': | |||||
msg = _('lfs: downloading %s (%s)\n') | |||||
elif action == 'upload': | |||||
msg = _('lfs: uploading %s (%s)\n') | |||||
self.ui.write(msg % (obj.get('oid'), | |||||
util.bytecount(objsize))) | |||||
retry = self.retry | |||||
while True: | |||||
try: | |||||
self._basictransfer(obj, action, localstore) | |||||
yield 1, obj.get('oid') | |||||
break | |||||
except Exception as ex: | |||||
if retry > 0: | |||||
if self.ui.verbose: | |||||
self.ui.write( | |||||
_('lfs: failed: %r (remaining retry %d)\n') | |||||
% (ex, retry)) | |||||
retry -= 1 | |||||
continue | |||||
raise | |||||
oids = worker.worker(self.ui, 0.1, transfer, (), | |||||
sorted(objects, key=lambda o: o.get('oid'))) | |||||
processed = 0 | |||||
for _one, oid in oids: | |||||
processed += sizes[oid] | |||||
self.ui.progress(topic, processed, total=total) | |||||
if self.ui.verbose: | |||||
self.ui.write(_('lfs: processed: %s\n') % oid) | |||||
self.ui.progress(topic, pos=None, total=total) | |||||
def __del__(self): | |||||
# copied from mercurial/httppeer.py | |||||
urlopener = getattr(self, 'urlopener', None) | |||||
if urlopener: | |||||
for h in urlopener.handlers: | |||||
h.close() | |||||
getattr(h, "close_all", lambda : None)() | |||||
class _dummyremote(object): | |||||
"""Dummy store storing blobs to temp directory.""" | |||||
def __init__(self, repo, url): | |||||
fullpath = repo.vfs.join('lfs', url.path) | |||||
self.vfs = lfsvfs(fullpath) | |||||
def writebatch(self, pointers, fromstore): | |||||
for p in pointers: | |||||
content = fromstore.read(p.oid()) | |||||
with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: | |||||
fp.write(content) | |||||
def readbatch(self, pointers, tostore): | |||||
for p in pointers: | |||||
content = self.vfs.read(p.oid()) | |||||
tostore.write(p.oid(), content) | |||||
class _nullremote(object): | |||||
"""Null store storing blobs to /dev/null.""" | |||||
def __init__(self, repo, url): | |||||
pass | |||||
def writebatch(self, pointers, fromstore): | |||||
pass | |||||
def readbatch(self, pointers, tostore): | |||||
pass | |||||
class _promptremote(object): | |||||
"""Prompt user to set lfs.url when accessed.""" | |||||
def __init__(self, repo, url): | |||||
pass | |||||
def writebatch(self, pointers, fromstore, ui=None): | |||||
self._prompt() | |||||
def readbatch(self, pointers, tostore, ui=None): | |||||
self._prompt() | |||||
def _prompt(self): | |||||
raise error.Abort(_('lfs.url needs to be configured')) | |||||
_storemap = { | |||||
'https': _gitlfsremote, | |||||
'http': _gitlfsremote, | |||||
'file': _dummyremote, | |||||
'null': _nullremote, | |||||
None: _promptremote, | |||||
} | |||||
def remote(repo): | |||||
"""remotestore factory. return a store in _storemap depending on config""" | |||||
defaulturl = '' | |||||
# convert deprecated configs to the new url. TODO: remove this if other | |||||
# places are migrated to the new url config. | |||||
# deprecated config: lfs.remotestore | |||||
deprecatedstore = repo.ui.config('lfs', 'remotestore') | |||||
if deprecatedstore == 'dummy': | |||||
# deprecated config: lfs.remotepath | |||||
defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath') | |||||
elif deprecatedstore == 'git-lfs': | |||||
# deprecated config: lfs.remoteurl | |||||
defaulturl = repo.ui.config('lfs', 'remoteurl') | |||||
elif deprecatedstore == 'null': | |||||
defaulturl = 'null://' | |||||
url = util.url(repo.ui.config('lfs', 'url', defaulturl)) | |||||
scheme = url.scheme | |||||
if scheme not in _storemap: | |||||
raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) | |||||
return _storemap[scheme](repo, url) | |||||
class LfsRemoteError(error.RevlogError): | |||||
pass |
# pointer.py - Git-LFS pointer serialization | |||||
# | |||||
# Copyright 2017 Facebook, Inc. | |||||
# | |||||
# This software may be used and distributed according to the terms of the | |||||
# GNU General Public License version 2 or any later version. | |||||
from __future__ import absolute_import | |||||
import re | |||||
from mercurial import ( | |||||
error, | |||||
) | |||||
from mercurial.i18n import _ | |||||
class InvalidPointer(error.RevlogError): | |||||
pass | |||||
class gitlfspointer(dict): | |||||
VERSION = 'https://git-lfs.github.com/spec/v1' | |||||
def __init__(self, *args, **kwargs): | |||||
self['version'] = self.VERSION | |||||
super(gitlfspointer, self).__init__(*args, **kwargs) | |||||
@classmethod | |||||
def deserialize(cls, text): | |||||
try: | |||||
return cls(l.split(' ', 1) for l in text.splitlines()).validate() | |||||
except ValueError: # l.split returns 1 item instead of 2 | |||||
raise InvalidPointer(_('cannot parse git-lfs text: %r') % text) | |||||
def serialize(self): | |||||
sortkeyfunc = lambda x: (x[0] != 'version', x) | |||||
items = sorted(self.validate().iteritems(), key=sortkeyfunc) | |||||
return ''.join('%s %s\n' % (k, v) for k, v in items) | |||||
def oid(self): | |||||
return self['oid'].split(':')[-1] | |||||
def size(self): | |||||
return int(self['size']) | |||||
# regular expressions used by _validate | |||||
# see https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md | |||||
_keyre = re.compile(r'\A[a-z0-9.-]+\Z') | |||||
_valuere = re.compile(r'\A[^\n]*\Z') | |||||
_requiredre = { | |||||
'size': re.compile(r'\A[0-9]+\Z'), | |||||
'oid': re.compile(r'\Asha256:[0-9a-f]{64}\Z'), | |||||
'version': re.compile(r'\A%s\Z' % re.escape(VERSION)), | |||||
} | |||||
def validate(self): | |||||
"""raise InvalidPointer on error. return self if there is no error""" | |||||
requiredcount = 0 | |||||
for k, v in self.iteritems(): | |||||
if k in self._requiredre: | |||||
if not self._requiredre[k].match(v): | |||||
raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) | |||||
requiredcount += 1 | |||||
elif not self._keyre.match(k): | |||||
raise InvalidPointer(_('unexpected key: %s') % k) | |||||
if not self._valuere.match(v): | |||||
raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) | |||||
if len(self._requiredre) != requiredcount: | |||||
miss = sorted(set(self._requiredre.keys()).difference(self.keys())) | |||||
raise InvalidPointer(_('missed keys: %s') % ', '.join(miss)) | |||||
return self | |||||
deserialize = gitlfspointer.deserialize |
# wrapper.py - methods wrapping core mercurial logic | |||||
# | |||||
# Copyright 2017 Facebook, Inc. | |||||
# | |||||
# This software may be used and distributed according to the terms of the | |||||
# GNU General Public License version 2 or any later version. | |||||
from __future__ import absolute_import | |||||
import hashlib | |||||
from mercurial import ( | |||||
error, | |||||
filelog, | |||||
revlog, | |||||
util, | |||||
) | |||||
from mercurial.i18n import _ | |||||
from mercurial.node import bin, nullid, short | |||||
from mercurial.utils import stringutil | |||||
from . import ( | |||||
blobstore, | |||||
pointer, | |||||
) | |||||
if util.safehasattr(revlog, 'parsemeta'): | |||||
# Since 0596d274 | |||||
_parsemeta = revlog.parsemeta | |||||
_packmeta = revlog.packmeta | |||||
else: | |||||
_parsemeta = filelog.parsemeta | |||||
_packmeta = filelog.packmeta | |||||
def supportedoutgoingversions(orig, repo): | |||||
versions = orig(repo) | |||||
versions.discard('01') | |||||
versions.discard('02') | |||||
versions.add('03') | |||||
return versions | |||||
def allsupportedversions(orig, ui): | |||||
versions = orig(ui) | |||||
versions.add('03') | |||||
return versions | |||||
def bypasscheckhash(self, text): | |||||
return False | |||||
def readfromstore(self, text): | |||||
"""Read filelog content from local blobstore transform for flagprocessor. | |||||
Default tranform for flagprocessor, returning contents from blobstore. | |||||
Returns a 2-typle (text, validatehash) where validatehash is True as the | |||||
contents of the blobstore should be checked using checkhash. | |||||
""" | |||||
p = pointer.deserialize(text) | |||||
oid = p.oid() | |||||
store = self.opener.lfslocalblobstore | |||||
if not store.has(oid): | |||||
p.filename = getattr(self, 'indexfile', None) | |||||
self.opener.lfsremoteblobstore.readbatch([p], store) | |||||
text = store.read(oid) | |||||
# pack hg filelog metadata | |||||
hgmeta = {} | |||||
for k in p.keys(): | |||||
if k.startswith('x-hg-'): | |||||
name = k[len('x-hg-'):] | |||||
hgmeta[name] = p[k] | |||||
if hgmeta or text.startswith('\1\n'): | |||||
text = _packmeta(hgmeta, text) | |||||
return (text, True) | |||||
def writetostore(self, text): | |||||
# hg filelog metadata (includes rename, etc) | |||||
hgmeta, offset = _parsemeta(text) | |||||
if offset and offset > 0: | |||||
# lfs blob does not contain hg filelog metadata | |||||
text = text[offset:] | |||||
# git-lfs only supports sha256 | |||||
oid = hashlib.sha256(text).hexdigest() | |||||
self.opener.lfslocalblobstore.write(oid, text) | |||||
# replace contents with metadata | |||||
longoid = 'sha256:%s' % oid | |||||
metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text))) | |||||
# by default, we expect the content to be binary. however, LFS could also | |||||
# be used for non-binary content. add a special entry for non-binary data. | |||||
# this will be used by filectx.isbinary(). | |||||
if not stringutil.binary(text): | |||||
# not hg filelog metadata (affecting commit hash), no "x-hg-" prefix | |||||
metadata['x-is-binary'] = '0' | |||||
# translate hg filelog metadata to lfs metadata with "x-hg-" prefix | |||||
if hgmeta is not None: | |||||
for k, v in hgmeta.iteritems(): | |||||
metadata['x-hg-%s' % k] = v | |||||
rawtext = metadata.serialize() | |||||
return (rawtext, False) | |||||
def _islfs(rlog, node=None, rev=None): | |||||
if rev is None: | |||||
if node is None: | |||||
# both None - likely working copy content where node is not ready | |||||
return False | |||||
rev = rlog.rev(node) | |||||
else: | |||||
node = rlog.node(rev) | |||||
if node == nullid: | |||||
return False | |||||
flags = rlog.flags(rev) | |||||
return bool(flags & revlog.REVIDX_EXTSTORED) | |||||
def filelogaddrevision(orig, self, text, transaction, link, p1, p2, | |||||
cachedelta=None, node=None, | |||||
flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds): | |||||
threshold = self.opener.options['lfsthreshold'] | |||||
textlen = len(text) | |||||
# exclude hg rename meta from file size | |||||
meta, offset = _parsemeta(text) | |||||
if offset: | |||||
textlen -= offset | |||||
if threshold and textlen > threshold: | |||||
flags |= revlog.REVIDX_EXTSTORED | |||||
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta, | |||||
node=node, flags=flags, **kwds) | |||||
def filelogrenamed(orig, self, node): | |||||
if _islfs(self, node): | |||||
rawtext = self.revision(node, raw=True) | |||||
if not rawtext: | |||||
return False | |||||
metadata = pointer.deserialize(rawtext) | |||||
if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata: | |||||
return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev']) | |||||
else: | |||||
return False | |||||
return orig(self, node) | |||||
def filelogsize(orig, self, rev): | |||||
if _islfs(self, rev=rev): | |||||
# fast path: use lfs metadata to answer size | |||||
rawtext = self.revision(rev, raw=True) | |||||
metadata = pointer.deserialize(rawtext) | |||||
return int(metadata['size']) | |||||
return orig(self, rev) | |||||
def filectxcmp(orig, self, fctx): | |||||
"""returns True if text is different than fctx""" | |||||
# some fctx (ex. hg-git) is not based on basefilectx and do not have islfs | |||||
if self.islfs() and getattr(fctx, 'islfs', lambda: False)(): | |||||
# fast path: check LFS oid | |||||
p1 = pointer.deserialize(self.rawdata()) | |||||
p2 = pointer.deserialize(fctx.rawdata()) | |||||
return p1.oid() != p2.oid() | |||||
return orig(self, fctx) | |||||
def filectxisbinary(orig, self): | |||||
if self.islfs(): | |||||
# fast path: use lfs metadata to answer isbinary | |||||
metadata = pointer.deserialize(self.rawdata()) | |||||
# if lfs metadata says nothing, assume it's binary by default | |||||
return bool(int(metadata.get('x-is-binary', 1))) | |||||
return orig(self) | |||||
def filectxislfs(self): | |||||
return _islfs(self.filelog(), self.filenode()) | |||||
def vfsinit(orig, self, othervfs): | |||||
orig(self, othervfs) | |||||
# copy lfs related options | |||||
for k, v in othervfs.options.items(): | |||||
if k.startswith('lfs'): | |||||
self.options[k] = v | |||||
# also copy lfs blobstores. note: this can run before reposetup, so lfs | |||||
# blobstore attributes are not always ready at this time. | |||||
for name in ['lfslocalblobstore', 'lfsremoteblobstore']: | |||||
if util.safehasattr(othervfs, name): | |||||
setattr(self, name, getattr(othervfs, name)) | |||||
def _canskipupload(repo): | |||||
# if remotestore is a null store, upload is a no-op and can be skipped | |||||
return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) | |||||
def candownload(repo): | |||||
# if remotestore is a null store, downloads will lead to nothing | |||||
return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) | |||||
def uploadblobsfromrevs(repo, revs): | |||||
'''upload lfs blobs introduced by revs | |||||
Note: also used by other extensions e. g. infinitepush. avoid renaming. | |||||
''' | |||||
if _canskipupload(repo): | |||||
return | |||||
pointers = extractpointers(repo, revs) | |||||
uploadblobs(repo, pointers) | |||||
def prepush(pushop): | |||||
"""Prepush hook. | |||||
Read through the revisions to push, looking for filelog entries that can be | |||||
deserialized into metadata so that we can block the push on their upload to | |||||
the remote blobstore. | |||||
""" | |||||
return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing) | |||||
def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing, | |||||
*args, **kwargs): | |||||
"""upload LFS blobs added by outgoing revisions on 'hg bundle'""" | |||||
uploadblobsfromrevs(repo, outgoing.missing) | |||||
return orig(ui, repo, source, filename, bundletype, outgoing, *args, | |||||
**kwargs) | |||||
def extractpointers(repo, revs): | |||||
"""return a list of lfs pointers added by given revs""" | |||||
ui = repo.ui | |||||
if ui.debugflag: | |||||
ui.write(_('lfs: computing set of blobs to upload\n')) | |||||
pointers = {} | |||||
for r in revs: | |||||
ctx = repo[r] | |||||
for p in pointersfromctx(ctx).values(): | |||||
pointers[p.oid()] = p | |||||
return pointers.values() | |||||
def pointersfromctx(ctx): | |||||
"""return a dict {path: pointer} for given single changectx""" | |||||
result = {} | |||||
for f in ctx.files(): | |||||
if f not in ctx: | |||||
continue | |||||
fctx = ctx[f] | |||||
if not _islfs(fctx.filelog(), fctx.filenode()): | |||||
continue | |||||
try: | |||||
result[f] = pointer.deserialize(fctx.rawdata()) | |||||
except pointer.InvalidPointer as ex: | |||||
raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n') | |||||
% (f, short(ctx.node()), ex)) | |||||
return result | |||||
def uploadblobs(repo, pointers): | |||||
"""upload given pointers from local blobstore""" | |||||
if not pointers: | |||||
return | |||||
remoteblob = repo.svfs.lfsremoteblobstore | |||||
remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore) |
from __future__ import absolute_import, print_function | |||||
import os | |||||
import sys | |||||
# make it runnable using python directly without run-tests.py | |||||
sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')] | |||||
from hgext3rd.lfs import pointer | |||||
def tryparse(text): | |||||
r = {} | |||||
try: | |||||
r = pointer.deserialize(text) | |||||
print('ok') | |||||
except Exception as ex: | |||||
print(ex) | |||||
if r: | |||||
text2 = r.serialize() | |||||
if text2 != text: | |||||
print('reconstructed text differs') | |||||
return r | |||||
t = ('version https://git-lfs.github.com/spec/v1\n' | |||||
'oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1' | |||||
'258daaa5e2ca24d17e2393\n' | |||||
'size 12345\n' | |||||
'x-foo extra-information\n') | |||||
tryparse('') | |||||
tryparse(t) | |||||
tryparse(t.replace('git-lfs', 'unknown')) | |||||
tryparse(t.replace('v1\n', 'v1\n\n')) | |||||
tryparse(t.replace('sha256', 'ahs256')) | |||||
tryparse(t.replace('sha256:', '')) | |||||
tryparse(t.replace('12345', '0x12345')) | |||||
tryparse(t.replace('extra-information', 'extra\0information')) | |||||
tryparse(t.replace('extra-information', 'extra\ninformation')) | |||||
tryparse(t.replace('x-foo', 'x_foo')) | |||||
tryparse(t.replace('oid', 'blobid')) | |||||
tryparse(t.replace('size', 'size-bytes').replace('oid', 'object-id')) |
missed keys: oid, size | |||||
ok | |||||
unexpected value: version='https://unknown.github.com/spec/v1' | |||||
cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\n\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra-information\n' | |||||
unexpected value: oid='ahs256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' | |||||
unexpected value: oid='4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' | |||||
unexpected value: size='0x12345' | |||||
ok | |||||
cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra\ninformation\n' | |||||
unexpected key: x_foo | |||||
missed keys: oid | |||||
missed keys: oid, size |
Require lfs-test-server (https://github.com/git-lfs/lfs-test-server) | |||||
$ hash lfs-test-server || { echo 'skipped: missing lfs-test-server'; exit 80; } | |||||
$ LFS_LISTEN="tcp://:$HGPORT" | |||||
$ LFS_HOST="localhost:$HGPORT" | |||||
$ LFS_PUBLIC=1 | |||||
$ export LFS_LISTEN LFS_HOST LFS_PUBLIC | |||||
$ lfs-test-server &> lfs-server.log & | |||||
$ echo $! >> $DAEMON_PIDS | |||||
$ cat >> $HGRCPATH <<EOF | |||||
> [extensions] | |||||
> lfs=$TESTDIR/../hgext3rd/lfs | |||||
> [lfs] | |||||
> url=http://foo:bar@$LFS_HOST/ | |||||
> threshold=1 | |||||
> EOF | |||||
$ hg init repo1 | |||||
$ cd repo1 | |||||
$ echo THIS-IS-LFS > a | |||||
$ hg commit -m a -A a | |||||
$ hg init ../repo2 | |||||
$ hg push ../repo2 -v | |||||
pushing to ../repo2 | |||||
searching for changes | |||||
lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) | |||||
1 changesets found | |||||
uncompressed size of bundle content: | |||||
* (changelog) (glob) | |||||
* (manifests) (glob) | |||||
* a (glob) | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 1 changesets with 1 changes to 1 files | |||||
$ cd ../repo2 | |||||
$ hg update tip -v | |||||
resolving manifests | |||||
getting a | |||||
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) | |||||
1 files updated, 0 files merged, 0 files removed, 0 files unresolved | |||||
When the server has some blobs already | |||||
$ hg mv a b | |||||
$ echo ANOTHER-LARGE-FILE > c | |||||
$ echo ANOTHER-LARGE-FILE2 > d | |||||
$ hg commit -m b-and-c -A b c d | |||||
$ hg push ../repo1 -v | grep -v '^ ' | |||||
pushing to ../repo1 | |||||
searching for changes | |||||
lfs: need to transfer 2 objects (39 bytes) | |||||
lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) | |||||
lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) | |||||
1 changesets found | |||||
uncompressed size of bundle content: | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 1 changesets with 3 changes to 3 files | |||||
$ hg --repo ../repo1 update tip -v | |||||
resolving manifests | |||||
getting b | |||||
getting c | |||||
lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) | |||||
getting d | |||||
lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) | |||||
3 files updated, 0 files merged, 0 files removed, 0 files unresolved | |||||
Check error message when the remote missed a blob: | |||||
$ echo FFFFF > b | |||||
$ hg commit -m b -A b | |||||
$ echo FFFFF >> b | |||||
$ hg commit -m b b | |||||
$ rm -rf .hg/store/lfs | |||||
$ hg update -C '.^' | |||||
abort: LFS server claims required objects do not exist: | |||||
8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13! | |||||
[255] | |||||
Check error message when object does not exist: | |||||
$ hg init test && cd test | |||||
$ echo "[extensions]" >> .hg/hgrc | |||||
$ echo "lfs=" >> .hg/hgrc | |||||
$ echo "[lfs]" >> .hg/hgrc | |||||
$ echo "threshold=1" >> .hg/hgrc | |||||
$ echo a > a | |||||
$ hg add a | |||||
$ hg commit -m 'test' | |||||
$ echo aaaaa > a | |||||
$ hg commit -m 'largefile' | |||||
$ hg debugdata .hg/store/data/a.i 1 # verify this is no the file content but includes "oid", the LFS "pointer". | |||||
version https://git-lfs.github.com/spec/v1 | |||||
oid sha256:bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a | |||||
size 6 | |||||
x-is-binary 0 | |||||
$ cd .. | |||||
$ hg --config 'lfs.url=https://dewey-lfs.vip.facebook.com/lfs' clone test test2 | |||||
updating to branch default | |||||
abort: LFS server error. Remote object for file data/a.i not found:(.*)! (re) | |||||
[255] |
# Initial setup | |||||
$ cat >> $HGRCPATH << EOF | |||||
> [extensions] | |||||
> lfs=$TESTDIR/../hgext3rd/lfs/ | |||||
> [lfs] | |||||
> threshold=1000B | |||||
> EOF | |||||
$ LONG=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC | |||||
# Prepare server and enable extension | |||||
$ hg init server | |||||
$ hg clone -q server client | |||||
$ cd client | |||||
# Commit small file | |||||
$ echo s > smallfile | |||||
$ hg commit -Aqm "add small file" | |||||
# Commit large file | |||||
$ echo $LONG > largefile | |||||
$ hg commit --traceback -Aqm "add large file" | |||||
# Ensure metadata is stored | |||||
$ hg debugdata largefile 0 | |||||
version https://git-lfs.github.com/spec/v1 | |||||
oid sha256:f11e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b | |||||
size 1501 | |||||
x-is-binary 0 | |||||
# Check the blobstore is populated | |||||
$ find .hg/store/lfs/objects | sort | |||||
.hg/store/lfs/objects | |||||
.hg/store/lfs/objects/f1 | |||||
.hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b | |||||
# Check the blob stored contains the actual contents of the file | |||||
$ cat .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b | |||||
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC | |||||
# Push changes to the server | |||||
$ hg push | |||||
pushing to $TESTTMP/server (glob) | |||||
searching for changes | |||||
abort: lfs.url needs to be configured | |||||
[255] | |||||
$ cat >> $HGRCPATH << EOF | |||||
> [lfs] | |||||
> url=file:$TESTTMP/dummy-remote/ | |||||
> EOF | |||||
$ hg push -v | egrep -v '^(uncompressed| )' | |||||
pushing to $TESTTMP/server (glob) | |||||
searching for changes | |||||
2 changesets found | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 2 changesets with 2 changes to 2 files | |||||
# Unknown URL scheme | |||||
$ hg push --config lfs.url=ftp://foobar | |||||
abort: lfs: unknown url scheme: ftp | |||||
[255] | |||||
$ cd ../ | |||||
# Initialize new client (not cloning) and setup extension | |||||
$ hg init client2 | |||||
$ cd client2 | |||||
$ cat >> .hg/hgrc <<EOF | |||||
> [paths] | |||||
> default = $TESTTMP/server | |||||
> EOF | |||||
# Pull from server | |||||
$ hg pull default | |||||
pulling from $TESTTMP/server (glob) | |||||
requesting all changes | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 2 changesets with 2 changes to 2 files | |||||
new changesets b29ba743f89d:00c137947d30 | |||||
(run 'hg update' to get a working copy) | |||||
# Check the blobstore is not yet populated | |||||
$ [ -d .hg/store/lfs/objects ] | |||||
[1] | |||||
# Update to the last revision containing the large file | |||||
$ hg update | |||||
2 files updated, 0 files merged, 0 files removed, 0 files unresolved | |||||
# Check the blobstore has been populated on update | |||||
$ find .hg/store/lfs/objects | sort | |||||
.hg/store/lfs/objects | |||||
.hg/store/lfs/objects/f1 | |||||
.hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b | |||||
# Check the contents of the file are fetched from blobstore when requested | |||||
$ hg cat -r . largefile | |||||
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC | |||||
# Check the file has been copied in the working copy | |||||
$ cat largefile | |||||
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC | |||||
$ cd .. | |||||
# Check rename, and switch between large and small files | |||||
$ hg init repo3 | |||||
$ cd repo3 | |||||
$ cat >> .hg/hgrc << EOF | |||||
> [lfs] | |||||
> threshold=10B | |||||
> EOF | |||||
$ echo LONGER-THAN-TEN-BYTES-WILL-TRIGGER-LFS > large | |||||
$ echo SHORTER > small | |||||
$ hg add . -q | |||||
$ hg commit -m 'commit with lfs content' | |||||
$ hg mv large l | |||||
$ hg mv small s | |||||
$ hg commit -m 'renames' | |||||
$ echo SHORT > l | |||||
$ echo BECOME-LARGER-FROM-SHORTER > s | |||||
$ hg commit -m 'large to small, small to large' | |||||
$ echo 1 >> l | |||||
$ echo 2 >> s | |||||
$ hg commit -m 'random modifications' | |||||
$ echo RESTORE-TO-BE-LARGE > l | |||||
$ echo SHORTER > s | |||||
$ hg commit -m 'switch large and small again' | |||||
# Test lfs_files template | |||||
$ hg log -r 'all()' -T '{rev} {join(lfs_files, ", ")}\n' | |||||
0 large | |||||
1 l | |||||
2 s | |||||
3 s | |||||
4 l | |||||
# Push and pull the above repo | |||||
$ hg --cwd .. init repo4 | |||||
$ hg push ../repo4 | |||||
pushing to ../repo4 | |||||
searching for changes | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 5 changesets with 10 changes to 4 files | |||||
$ hg --cwd .. init repo5 | |||||
$ hg --cwd ../repo5 pull ../repo3 | |||||
pulling from ../repo3 | |||||
requesting all changes | |||||
adding changesets | |||||
adding manifests | |||||
adding file changes | |||||
added 5 changesets with 10 changes to 4 files | |||||
new changesets fd47a419c4f7:5adf850972b9 | |||||
(run 'hg update' to get a working copy) | |||||
$ cd .. | |||||
# Test clone | |||||
$ hg init repo6 | |||||
$ cd repo6 | |||||
$ cat >> .hg/hgrc << EOF | |||||
> [lfs] | |||||
> threshold=30B | |||||
> EOF | |||||
$ echo LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES > large | |||||
$ echo SMALL > small | |||||
$ hg commit -Aqm 'create a lfs file' large small | |||||
$ hg debuglfsupload -r 'all()' -v | |||||
$ cd .. | |||||
$ hg clone repo6 repo7 | |||||
updating to branch default | |||||
2 files updated, 0 files merged, 0 files removed, 0 files unresolved | |||||
$ cd repo7 | |||||
$ cat large | |||||
LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES | |||||
$ cat small | |||||
SMALL | |||||
$ cd .. | |||||
# Test rename and status | |||||
$ hg init repo8 | |||||
$ cd repo8 | |||||
$ cat >> .hg/hgrc << EOF | |||||
> [lfs] | |||||
> threshold=10B | |||||
> EOF | |||||
$ echo THIS-IS-LFS-BECAUSE-10-BYTES > a1 | |||||
$ echo SMALL > a2 | |||||
$ hg commit -m a -A a1 a2 | |||||
$ hg status | |||||
$ hg mv a1 b1 | |||||
$ hg mv a2 a1 | |||||
$ hg mv b1 a2 | |||||
$ hg commit -m b | |||||
$ hg status | |||||
$ HEADER=$'\1\n' | |||||
$ printf '%sSTART-WITH-HG-FILELOG-METADATA' "$HEADER" > a2 | |||||
$ printf '%sMETA\n' "$HEADER" > a1 | |||||
$ hg commit -m meta | |||||
$ hg status | |||||
$ hg log -T '{rev}: {file_copies} | {file_dels} | {file_adds}\n' | |||||
2: | | | |||||
1: a1 (a2)a2 (a1) | | | |||||
0: | | a1 a2 | |||||
$ for n in a1 a2; do | |||||
> for r in 0 1 2; do | |||||
> printf '\n%s @ %s\n' $n $r | |||||
> hg debugdata $n $r | |||||
> done | |||||
> done | |||||
a1 @ 0 | |||||
version https://git-lfs.github.com/spec/v1 | |||||
oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 | |||||
size 29 | |||||
x-is-binary 0 | |||||
a1 @ 1 | |||||
\x01 (esc) | |||||
copy: a2 | |||||
copyrev: 50470ad23cf937b1f4b9f80bfe54df38e65b50d9 | |||||
\x01 (esc) | |||||
SMALL | |||||
a1 @ 2 | |||||
\x01 (esc) | |||||
\x01 (esc) | |||||
\x01 (esc) | |||||
META | |||||
a2 @ 0 | |||||
SMALL | |||||
a2 @ 1 | |||||
version https://git-lfs.github.com/spec/v1 | |||||
oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 | |||||
size 29 | |||||
x-hg-copy a1 | |||||
x-hg-copyrev be23af27908a582af43e5cda209a5a9b319de8d4 | |||||
x-is-binary 0 | |||||
a2 @ 2 | |||||
version https://git-lfs.github.com/spec/v1 | |||||
oid sha256:876dadc86a8542f9798048f2c47f51dbf8e4359aed883e8ec80c5db825f0d943 | |||||
size 32 | |||||
x-is-binary 0 | |||||
# Verify commit hashes include rename metadata | |||||
$ hg log -T '{rev}:{node|short} {desc}\n' | |||||
2:0fae949de7fa meta | |||||
1:9cd6bdffdac0 b | |||||
0:7f96794915f7 a | |||||
$ cd .. | |||||
# Test bundle | |||||
$ hg init repo9 | |||||
$ cd repo9 | |||||
$ cat >> .hg/hgrc << EOF | |||||
> [lfs] | |||||
> threshold=10B | |||||
> [diff] | |||||
> git=1 | |||||
> EOF | |||||
$ for i in 0 single two three 4; do | |||||
> echo 'THIS-IS-LFS-'$i > a | |||||
> hg commit -m a-$i -A a | |||||
> done | |||||
$ hg update 2 -q | |||||
$ echo 'THIS-IS-LFS-2-CHILD' > a | |||||
$ hg commit -m branching -q | |||||
$ hg bundle --base 1 bundle.hg -v | |||||
4 changesets found | |||||
uncompressed size of bundle content: | |||||
* (changelog) (glob) | |||||
* (manifests) (glob) | |||||
* a (glob) | |||||
$ hg --config extensions.strip= strip -r 2 --no-backup --force -q | |||||
$ hg -R bundle.hg log -p -T '{rev} {desc}\n' a | |||||
5 branching | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-two | |||||
+THIS-IS-LFS-2-CHILD | |||||
4 a-4 | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-three | |||||
+THIS-IS-LFS-4 | |||||
3 a-three | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-two | |||||
+THIS-IS-LFS-three | |||||
2 a-two | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-single | |||||
+THIS-IS-LFS-two | |||||
1 a-single | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-0 | |||||
+THIS-IS-LFS-single | |||||
0 a-0 | |||||
diff --git a/a b/a | |||||
new file mode 100644 | |||||
--- /dev/null | |||||
+++ b/a | |||||
@@ -0,0 +1,1 @@ | |||||
+THIS-IS-LFS-0 | |||||
$ hg bundle -R bundle.hg --base 1 bundle-again.hg -q | |||||
$ hg -R bundle-again.hg log -p -T '{rev} {desc}\n' a | |||||
5 branching | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-two | |||||
+THIS-IS-LFS-2-CHILD | |||||
4 a-4 | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-three | |||||
+THIS-IS-LFS-4 | |||||
3 a-three | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-two | |||||
+THIS-IS-LFS-three | |||||
2 a-two | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-single | |||||
+THIS-IS-LFS-two | |||||
1 a-single | |||||
diff --git a/a b/a | |||||
--- a/a | |||||
+++ b/a | |||||
@@ -1,1 +1,1 @@ | |||||
-THIS-IS-LFS-0 | |||||
+THIS-IS-LFS-single | |||||
0 a-0 | |||||
diff --git a/a b/a | |||||
new file mode 100644 | |||||
--- /dev/null | |||||
+++ b/a | |||||
@@ -0,0 +1,1 @@ | |||||
+THIS-IS-LFS-0 | |||||
$ cd .. | |||||
# Test isbinary | |||||
$ hg init repo10 | |||||
$ cd repo10 | |||||
$ cat >> .hg/hgrc << EOF | |||||
> [extensions] | |||||
> lfs=$TESTDIR/../hgext3rd/lfs/ | |||||
> [lfs] | |||||
> threshold=1 | |||||
> EOF | |||||
$ $PYTHON <<'EOF' | |||||
> def write(path, content): | |||||
> with open(path, 'wb') as f: | |||||
> f.write(content) | |||||
> write('a', b'\0\0') | |||||
> write('b', b'\1\n') | |||||
> write('c', b'\1\n\0') | |||||
> write('d', b'xx') | |||||
> EOF | |||||
$ hg add a b c d | |||||
$ hg diff --stat | |||||
a | Bin | |||||
b | 1 + | |||||
c | Bin | |||||
d | 1 + | |||||
4 files changed, 2 insertions(+), 0 deletions(-) | |||||
$ hg commit -m binarytest | |||||
$ cat > $TESTTMP/dumpbinary.py << EOF | |||||
> def reposetup(ui, repo): | |||||
> for n in 'abcd': | |||||
> ui.write(('%s: binary=%s\n') % (n, repo['.'][n].isbinary())) | |||||
> EOF | |||||
$ hg --config extensions.dumpbinary=$TESTTMP/dumpbinary.py id --trace | |||||
a: binary=True | |||||
b: binary=False | |||||
c: binary=True | |||||
d: binary=False | |||||
b55353847f02 tip | |||||
$ cd .. | |||||
# Test fctx.cmp fastpath - diff without LFS blobs | |||||
$ hg init repo11 | |||||
$ cd repo11 | |||||
$ cat >> .hg/hgrc <<EOF | |||||
> [lfs] | |||||
> threshold=1 | |||||
> EOF | |||||
$ for i in 1 2 3; do | |||||
> cp ../repo10/a a | |||||
> if [ $i = 3 ]; then | |||||
> # make a content-only change | |||||
> chmod +x a | |||||
> i=2 | |||||
> fi | |||||
> echo $i >> a | |||||
> hg commit -m $i -A a | |||||
> done | |||||
$ [ -d .hg/store/lfs/objects ] | |||||
$ cd .. | |||||
$ hg clone repo11 repo12 --noupdate | |||||
$ cd repo12 | |||||
$ hg log --removed -p a -T '{desc}\n' --config diff.nobinary=1 --git | |||||
2 | |||||
diff --git a/a b/a | |||||
old mode 100644 | |||||
new mode 100755 | |||||
2 | |||||
diff --git a/a b/a | |||||
Binary file a has changed | |||||
1 | |||||
diff --git a/a b/a | |||||
new file mode 100644 | |||||
Binary file a has changed | |||||
$ [ -d .hg/store/lfs/objects ] | |||||
[1] | |||||
$ cd .. | |||||
# Verify the repos | |||||
$ cat > $TESTTMP/dumpflog.py << EOF | |||||
> # print raw revision sizes, flags, and hashes for certain files | |||||
> import hashlib | |||||
> from mercurial import revlog | |||||
> from mercurial.node import short | |||||
> def hash(rawtext): | |||||
> h = hashlib.sha512() | |||||
> h.update(rawtext) | |||||
> return h.hexdigest()[:4] | |||||
> def reposetup(ui, repo): | |||||
> # these 2 files are interesting | |||||
> for name in ['l', 's']: | |||||
> fl = repo.file(name) | |||||
> if len(fl) == 0: | |||||
> continue | |||||
> sizes = [revlog.revlog.rawsize(fl, i) for i in fl] | |||||
> texts = [fl.revision(i, raw=True) for i in fl] | |||||
> flags = [fl.flags(i) for i in fl] | |||||
> hashes = [hash(t) for t in texts] | |||||
> print(' %s: rawsizes=%r flags=%r hashes=%r' | |||||
> % (name, sizes, flags, hashes)) | |||||
> EOF | |||||
$ for i in client client2 server repo3 repo4 repo5 repo6 repo7 repo8 repo9 \ | |||||
> repo10; do | |||||
> echo 'repo:' $i | |||||
> hg --cwd $i verify --config extensions.dumpflog=$TESTTMP/dumpflog.py -q | |||||
> done | |||||
repo: client | |||||
repo: client2 | |||||
repo: server | |||||
repo: repo3 | |||||
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] | |||||
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] | |||||
repo: repo4 | |||||
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] | |||||
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] | |||||
repo: repo5 | |||||
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] | |||||
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] | |||||
repo: repo6 | |||||
repo: repo7 | |||||
repo: repo8 | |||||
repo: repo9 | |||||
repo: repo10 |