diff --git a/hgext3rd/lfs/__init__.py b/hgext3rd/lfs/__init__.py deleted file mode 100644 --- a/hgext3rd/lfs/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# lfs - hash-preserving large file support using Git-LFS protocol -# -# Copyright 2017 Facebook, Inc. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -"""lfs - large file support - -Configs:: - - [lfs] - # Remote endpoint. Multiple protocols are supported: - # - http(s)://user:pass@example.com/path - # git-lfs endpoint - # - file:///tmp/path - # local filesystem, usually for testing - # if unset, lfs will prompt setting this when it must use this value. - # (default: unset) - url = https://example.com/lfs - - # size of a file to make it use LFS - threshold = 10M - - # how many times to retry before giving up on transferring an object - retry = 5 -""" - -from __future__ import absolute_import - -from mercurial import ( - bundle2, - changegroup, - context, - exchange, - extensions, - filelog, - registrar, - revlog, - scmutil, - vfs as vfsmod, -) -from mercurial.i18n import _ - -from . import ( - blobstore, - wrapper, -) - -cmdtable = {} -command = registrar.command(cmdtable) - -templatekeyword = registrar.templatekeyword() - -def reposetup(ui, repo): - # Nothing to do with a remote repo - if not repo.local(): - return - - threshold = repo.ui.configbytes('lfs', 'threshold', None) - - repo.svfs.options['lfsthreshold'] = threshold - repo.svfs.lfslocalblobstore = blobstore.local(repo) - repo.svfs.lfsremoteblobstore = blobstore.remote(repo) - - # Push hook - repo.prepushoutgoinghooks.add('lfs', wrapper.prepush) - -def wrapfilelog(filelog): - wrapfunction = extensions.wrapfunction - - wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) - wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) - wrapfunction(filelog, 'size', wrapper.filelogsize) - -def extsetup(ui): - wrapfilelog(filelog.filelog) - - wrapfunction = extensions.wrapfunction - wrapfunction(changegroup, - 'supportedoutgoingversions', - wrapper.supportedoutgoingversions) - wrapfunction(changegroup, - 'allsupportedversions', - wrapper.allsupportedversions) - - wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp) - wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary) - context.basefilectx.islfs = wrapper.filectxislfs - - revlog.addflagprocessor( - revlog.REVIDX_EXTSTORED, - ( - wrapper.readfromstore, - wrapper.writetostore, - wrapper.bypasscheckhash, - ), - ) - - # Make bundle choose changegroup3 instead of changegroup2. This affects - # "hg bundle" command. Note: it does not cover all bundle formats like - # "packed1". Using "packed1" with lfs will likely cause trouble. - names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02'] - for k in names: - exchange._bundlespeccgversions[k] = '03' - - # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs - # options and blob stores are passed from othervfs to the new readonlyvfs. - wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit) - - # when writing a bundle via "hg bundle" command, upload related LFS blobs - wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle) - -@templatekeyword('lfs_files') -def lfsfiles(repo, ctx, **args): - """List of strings. LFS files added or modified by the changeset.""" - pointers = wrapper.pointersfromctx(ctx) # {path: pointer} - return sorted(pointers.keys()) - -@command('debuglfsupload', - [('r', 'rev', [], _('upload large files introduced by REV'))]) -def debuglfsupload(ui, repo, **opts): - """upload lfs blobs added by the working copy parent or given revisions""" - revs = opts.get('rev', []) - pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs)) - wrapper.uploadblobs(repo, pointers) diff --git a/hgext3rd/lfs/blobstore.py b/hgext3rd/lfs/blobstore.py deleted file mode 100644 --- a/hgext3rd/lfs/blobstore.py +++ /dev/null @@ -1,351 +0,0 @@ -# blobstore.py - local and remote (speaking Git-LFS protocol) blob storages -# -# Copyright 2017 Facebook, Inc. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -from __future__ import absolute_import - -import json -import os -import re - -from mercurial import ( - error, - url as urlmod, - util, - vfs as vfsmod, - worker, -) -from mercurial.i18n import _ - -# 64 bytes for SHA256 -_lfsre = re.compile(r'\A[a-f0-9]{64}\Z') - -class lfsvfs(vfsmod.vfs): - def join(self, path): - """split the path at first two characters, like: XX/XXXXX...""" - if not _lfsre.match(path): - raise error.ProgrammingError('unexpected lfs path: %s' % path) - return super(lfsvfs, self).join(path[0:2], path[2:]) - -class filewithprogress(object): - """a file-like object that supports __len__ and read. - - Useful to provide progress information for how many bytes are read. - """ - - def __init__(self, fp, callback): - self._fp = fp - self._callback = callback # func(readsize) - fp.seek(0, os.SEEK_END) - self._len = fp.tell() - fp.seek(0) - - def __len__(self): - return self._len - - def read(self, size): - if self._fp is None: - return b'' - data = self._fp.read(size) - if data: - if self._callback: - self._callback(len(data)) - else: - self._fp.close() - self._fp = None - return data - -class local(object): - """Local blobstore for large file contents. - - This blobstore is used both as a cache and as a staging area for large blobs - to be uploaded to the remote blobstore. - """ - - def __init__(self, repo): - fullpath = repo.svfs.join('lfs/objects') - self.vfs = lfsvfs(fullpath) - - def write(self, oid, data): - """Write blob to local blobstore.""" - with self.vfs(oid, 'wb', atomictemp=True) as fp: - fp.write(data) - - def read(self, oid): - """Read blob from local blobstore.""" - return self.vfs.read(oid) - - def has(self, oid): - """Returns True if the local blobstore contains the requested blob, - False otherwise.""" - return self.vfs.exists(oid) - -class _gitlfsremote(object): - - def __init__(self, repo, url): - ui = repo.ui - self.ui = ui - baseurl, authinfo = url.authinfo() - self.baseurl = baseurl.rstrip('/') - self.urlopener = urlmod.opener(ui, authinfo) - self.retry = ui.configint('lfs', 'retry') or 5 - - def writebatch(self, pointers, fromstore): - """Batch upload from local to remote blobstore.""" - self._batch(pointers, fromstore, 'upload') - - def readbatch(self, pointers, tostore): - """Batch download from remote to local blostore.""" - self._batch(pointers, tostore, 'download') - - def _batchrequest(self, pointers, action): - """Get metadata about objects pointed by pointers for given action - - Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} - See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md - """ - objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] - requestdata = json.dumps({ - 'objects': objects, - 'operation': action, - }) - batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl, - data=requestdata) - batchreq.add_header('Accept', 'application/vnd.git-lfs+json') - batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') - try: - rawjson = self.urlopener.open(batchreq).read() - except util.urlerr.httperror as ex: - raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)') - % (ex, action)) - try: - response = json.loads(rawjson) - except ValueError: - raise LfsRemoteError(_('LFS server returns invalid JSON: %s') - % rawjson) - return response - - def _checkforservererror(self, pointers, responses): - """Scans errors from objects - - Returns LfsRemoteError if any objects has an error""" - for response in responses: - error = response.get('error') - if error: - ptrmap = {p.oid(): p for p in pointers} - p = ptrmap.get(response['oid'], None) - if error['code'] == 404 and p: - filename = getattr(p, 'filename', 'unknown') - raise LfsRemoteError( - _(('LFS server error. Remote object ' - 'for file %s not found: %r')) % (filename, response)) - raise LfsRemoteError(_('LFS server error: %r') % response) - - def _extractobjects(self, response, pointers, action): - """extract objects from response of the batch API - - response: parsed JSON object returned by batch API - return response['objects'] filtered by action - raise if any object has an error - """ - # Scan errors from objects - fail early - objects = response.get('objects', []) - self._checkforservererror(pointers, objects) - - # Filter objects with given action. Practically, this skips uploading - # objects which exist in the server. - filteredobjects = [o for o in objects if action in o.get('actions', [])] - # But for downloading, we want all objects. Therefore missing objects - # should be considered an error. - if action == 'download': - if len(filteredobjects) < len(objects): - missing = [o.get('oid', '?') - for o in objects - if action not in o.get('actions', [])] - raise LfsRemoteError( - _('LFS server claims required objects do not exist:\n%s') - % '\n'.join(missing)) - - return filteredobjects - - def _basictransfer(self, obj, action, localstore): - """Download or upload a single object using basic transfer protocol - - obj: dict, an object description returned by batch API - action: string, one of ['upload', 'download'] - localstore: blobstore.local - - See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ - basic-transfers.md - """ - oid = str(obj['oid']) - - href = str(obj['actions'][action].get('href')) - headers = obj['actions'][action].get('header', {}).items() - - request = util.urlreq.request(href) - if action == 'upload': - # If uploading blobs, read data from local blobstore. - request.data = localstore.vfs(oid) - request.get_method = lambda: 'PUT' - - for k, v in headers: - request.add_header(k, v) - - response = b'' - try: - req = self.urlopener.open(request) - while True: - data = req.read(1048576) - if not data: - break - response += data - except util.urlerr.httperror as ex: - raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') - % (ex, oid, action)) - - if action == 'download': - # If downloading blobs, store downloaded data to local blobstore - localstore.write(oid, response) - - def _batch(self, pointers, localstore, action): - if action not in ['upload', 'download']: - raise error.ProgrammingError('invalid Git-LFS action: %s' % action) - - response = self._batchrequest(pointers, action) - objects = self._extractobjects(response, pointers, action) - total = sum(x.get('size', 0) for x in objects) - sizes = {} - for obj in objects: - sizes[obj.get('oid')] = obj.get('size', 0) - topic = {'upload': _('lfs uploading'), - 'download': _('lfs downloading')}[action] - if self.ui.verbose and len(objects) > 1: - self.ui.write(_('lfs: need to transfer %d objects (%s)\n') - % (len(objects), util.bytecount(total))) - self.ui.progress(topic, 0, total=total) - def transfer(chunk): - for obj in chunk: - objsize = obj.get('size', 0) - if self.ui.verbose: - if action == 'download': - msg = _('lfs: downloading %s (%s)\n') - elif action == 'upload': - msg = _('lfs: uploading %s (%s)\n') - self.ui.write(msg % (obj.get('oid'), - util.bytecount(objsize))) - retry = self.retry - while True: - try: - self._basictransfer(obj, action, localstore) - yield 1, obj.get('oid') - break - except Exception as ex: - if retry > 0: - if self.ui.verbose: - self.ui.write( - _('lfs: failed: %r (remaining retry %d)\n') - % (ex, retry)) - retry -= 1 - continue - raise - - oids = worker.worker(self.ui, 0.1, transfer, (), - sorted(objects, key=lambda o: o.get('oid'))) - processed = 0 - for _one, oid in oids: - processed += sizes[oid] - self.ui.progress(topic, processed, total=total) - if self.ui.verbose: - self.ui.write(_('lfs: processed: %s\n') % oid) - self.ui.progress(topic, pos=None, total=total) - - def __del__(self): - # copied from mercurial/httppeer.py - urlopener = getattr(self, 'urlopener', None) - if urlopener: - for h in urlopener.handlers: - h.close() - getattr(h, "close_all", lambda : None)() - -class _dummyremote(object): - """Dummy store storing blobs to temp directory.""" - - def __init__(self, repo, url): - fullpath = repo.vfs.join('lfs', url.path) - self.vfs = lfsvfs(fullpath) - - def writebatch(self, pointers, fromstore): - for p in pointers: - content = fromstore.read(p.oid()) - with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: - fp.write(content) - - def readbatch(self, pointers, tostore): - for p in pointers: - content = self.vfs.read(p.oid()) - tostore.write(p.oid(), content) - -class _nullremote(object): - """Null store storing blobs to /dev/null.""" - - def __init__(self, repo, url): - pass - - def writebatch(self, pointers, fromstore): - pass - - def readbatch(self, pointers, tostore): - pass - -class _promptremote(object): - """Prompt user to set lfs.url when accessed.""" - - def __init__(self, repo, url): - pass - - def writebatch(self, pointers, fromstore, ui=None): - self._prompt() - - def readbatch(self, pointers, tostore, ui=None): - self._prompt() - - def _prompt(self): - raise error.Abort(_('lfs.url needs to be configured')) - -_storemap = { - 'https': _gitlfsremote, - 'http': _gitlfsremote, - 'file': _dummyremote, - 'null': _nullremote, - None: _promptremote, -} - -def remote(repo): - """remotestore factory. return a store in _storemap depending on config""" - defaulturl = '' - - # convert deprecated configs to the new url. TODO: remove this if other - # places are migrated to the new url config. - # deprecated config: lfs.remotestore - deprecatedstore = repo.ui.config('lfs', 'remotestore') - if deprecatedstore == 'dummy': - # deprecated config: lfs.remotepath - defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath') - elif deprecatedstore == 'git-lfs': - # deprecated config: lfs.remoteurl - defaulturl = repo.ui.config('lfs', 'remoteurl') - elif deprecatedstore == 'null': - defaulturl = 'null://' - - url = util.url(repo.ui.config('lfs', 'url', defaulturl)) - scheme = url.scheme - if scheme not in _storemap: - raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) - return _storemap[scheme](repo, url) - -class LfsRemoteError(error.RevlogError): - pass diff --git a/hgext3rd/lfs/pointer.py b/hgext3rd/lfs/pointer.py deleted file mode 100644 --- a/hgext3rd/lfs/pointer.py +++ /dev/null @@ -1,72 +0,0 @@ -# pointer.py - Git-LFS pointer serialization -# -# Copyright 2017 Facebook, Inc. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -from __future__ import absolute_import - -import re - -from mercurial import ( - error, -) -from mercurial.i18n import _ - -class InvalidPointer(error.RevlogError): - pass - -class gitlfspointer(dict): - VERSION = 'https://git-lfs.github.com/spec/v1' - - def __init__(self, *args, **kwargs): - self['version'] = self.VERSION - super(gitlfspointer, self).__init__(*args, **kwargs) - - @classmethod - def deserialize(cls, text): - try: - return cls(l.split(' ', 1) for l in text.splitlines()).validate() - except ValueError: # l.split returns 1 item instead of 2 - raise InvalidPointer(_('cannot parse git-lfs text: %r') % text) - - def serialize(self): - sortkeyfunc = lambda x: (x[0] != 'version', x) - items = sorted(self.validate().iteritems(), key=sortkeyfunc) - return ''.join('%s %s\n' % (k, v) for k, v in items) - - def oid(self): - return self['oid'].split(':')[-1] - - def size(self): - return int(self['size']) - - # regular expressions used by _validate - # see https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - _keyre = re.compile(r'\A[a-z0-9.-]+\Z') - _valuere = re.compile(r'\A[^\n]*\Z') - _requiredre = { - 'size': re.compile(r'\A[0-9]+\Z'), - 'oid': re.compile(r'\Asha256:[0-9a-f]{64}\Z'), - 'version': re.compile(r'\A%s\Z' % re.escape(VERSION)), - } - - def validate(self): - """raise InvalidPointer on error. return self if there is no error""" - requiredcount = 0 - for k, v in self.iteritems(): - if k in self._requiredre: - if not self._requiredre[k].match(v): - raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) - requiredcount += 1 - elif not self._keyre.match(k): - raise InvalidPointer(_('unexpected key: %s') % k) - if not self._valuere.match(v): - raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) - if len(self._requiredre) != requiredcount: - miss = sorted(set(self._requiredre.keys()).difference(self.keys())) - raise InvalidPointer(_('missed keys: %s') % ', '.join(miss)) - return self - -deserialize = gitlfspointer.deserialize diff --git a/hgext3rd/lfs/wrapper.py b/hgext3rd/lfs/wrapper.py deleted file mode 100644 --- a/hgext3rd/lfs/wrapper.py +++ /dev/null @@ -1,256 +0,0 @@ -# wrapper.py - methods wrapping core mercurial logic -# -# Copyright 2017 Facebook, Inc. -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -from __future__ import absolute_import - -import hashlib - -from mercurial import ( - error, - filelog, - revlog, - util, -) -from mercurial.i18n import _ -from mercurial.node import bin, nullid, short -from mercurial.utils import stringutil - -from . import ( - blobstore, - pointer, -) - -if util.safehasattr(revlog, 'parsemeta'): - # Since 0596d274 - _parsemeta = revlog.parsemeta - _packmeta = revlog.packmeta -else: - _parsemeta = filelog.parsemeta - _packmeta = filelog.packmeta - -def supportedoutgoingversions(orig, repo): - versions = orig(repo) - versions.discard('01') - versions.discard('02') - versions.add('03') - return versions - -def allsupportedversions(orig, ui): - versions = orig(ui) - versions.add('03') - return versions - -def bypasscheckhash(self, text): - return False - -def readfromstore(self, text): - """Read filelog content from local blobstore transform for flagprocessor. - - Default tranform for flagprocessor, returning contents from blobstore. - Returns a 2-typle (text, validatehash) where validatehash is True as the - contents of the blobstore should be checked using checkhash. - """ - p = pointer.deserialize(text) - oid = p.oid() - store = self.opener.lfslocalblobstore - if not store.has(oid): - p.filename = getattr(self, 'indexfile', None) - self.opener.lfsremoteblobstore.readbatch([p], store) - text = store.read(oid) - - # pack hg filelog metadata - hgmeta = {} - for k in p.keys(): - if k.startswith('x-hg-'): - name = k[len('x-hg-'):] - hgmeta[name] = p[k] - if hgmeta or text.startswith('\1\n'): - text = _packmeta(hgmeta, text) - - return (text, True) - -def writetostore(self, text): - # hg filelog metadata (includes rename, etc) - hgmeta, offset = _parsemeta(text) - if offset and offset > 0: - # lfs blob does not contain hg filelog metadata - text = text[offset:] - - # git-lfs only supports sha256 - oid = hashlib.sha256(text).hexdigest() - self.opener.lfslocalblobstore.write(oid, text) - - # replace contents with metadata - longoid = 'sha256:%s' % oid - metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text))) - - # by default, we expect the content to be binary. however, LFS could also - # be used for non-binary content. add a special entry for non-binary data. - # this will be used by filectx.isbinary(). - if not stringutil.binary(text): - # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix - metadata['x-is-binary'] = '0' - - # translate hg filelog metadata to lfs metadata with "x-hg-" prefix - if hgmeta is not None: - for k, v in hgmeta.iteritems(): - metadata['x-hg-%s' % k] = v - - rawtext = metadata.serialize() - return (rawtext, False) - -def _islfs(rlog, node=None, rev=None): - if rev is None: - if node is None: - # both None - likely working copy content where node is not ready - return False - rev = rlog.rev(node) - else: - node = rlog.node(rev) - if node == nullid: - return False - flags = rlog.flags(rev) - return bool(flags & revlog.REVIDX_EXTSTORED) - -def filelogaddrevision(orig, self, text, transaction, link, p1, p2, - cachedelta=None, node=None, - flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds): - threshold = self.opener.options['lfsthreshold'] - textlen = len(text) - # exclude hg rename meta from file size - meta, offset = _parsemeta(text) - if offset: - textlen -= offset - - if threshold and textlen > threshold: - flags |= revlog.REVIDX_EXTSTORED - - return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta, - node=node, flags=flags, **kwds) - -def filelogrenamed(orig, self, node): - if _islfs(self, node): - rawtext = self.revision(node, raw=True) - if not rawtext: - return False - metadata = pointer.deserialize(rawtext) - if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata: - return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev']) - else: - return False - return orig(self, node) - -def filelogsize(orig, self, rev): - if _islfs(self, rev=rev): - # fast path: use lfs metadata to answer size - rawtext = self.revision(rev, raw=True) - metadata = pointer.deserialize(rawtext) - return int(metadata['size']) - return orig(self, rev) - -def filectxcmp(orig, self, fctx): - """returns True if text is different than fctx""" - # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs - if self.islfs() and getattr(fctx, 'islfs', lambda: False)(): - # fast path: check LFS oid - p1 = pointer.deserialize(self.rawdata()) - p2 = pointer.deserialize(fctx.rawdata()) - return p1.oid() != p2.oid() - return orig(self, fctx) - -def filectxisbinary(orig, self): - if self.islfs(): - # fast path: use lfs metadata to answer isbinary - metadata = pointer.deserialize(self.rawdata()) - # if lfs metadata says nothing, assume it's binary by default - return bool(int(metadata.get('x-is-binary', 1))) - return orig(self) - -def filectxislfs(self): - return _islfs(self.filelog(), self.filenode()) - -def vfsinit(orig, self, othervfs): - orig(self, othervfs) - # copy lfs related options - for k, v in othervfs.options.items(): - if k.startswith('lfs'): - self.options[k] = v - # also copy lfs blobstores. note: this can run before reposetup, so lfs - # blobstore attributes are not always ready at this time. - for name in ['lfslocalblobstore', 'lfsremoteblobstore']: - if util.safehasattr(othervfs, name): - setattr(self, name, getattr(othervfs, name)) - -def _canskipupload(repo): - # if remotestore is a null store, upload is a no-op and can be skipped - return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) - -def candownload(repo): - # if remotestore is a null store, downloads will lead to nothing - return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) - -def uploadblobsfromrevs(repo, revs): - '''upload lfs blobs introduced by revs - - Note: also used by other extensions e. g. infinitepush. avoid renaming. - ''' - if _canskipupload(repo): - return - pointers = extractpointers(repo, revs) - uploadblobs(repo, pointers) - -def prepush(pushop): - """Prepush hook. - - Read through the revisions to push, looking for filelog entries that can be - deserialized into metadata so that we can block the push on their upload to - the remote blobstore. - """ - return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing) - -def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing, - *args, **kwargs): - """upload LFS blobs added by outgoing revisions on 'hg bundle'""" - uploadblobsfromrevs(repo, outgoing.missing) - return orig(ui, repo, source, filename, bundletype, outgoing, *args, - **kwargs) - -def extractpointers(repo, revs): - """return a list of lfs pointers added by given revs""" - ui = repo.ui - if ui.debugflag: - ui.write(_('lfs: computing set of blobs to upload\n')) - pointers = {} - for r in revs: - ctx = repo[r] - for p in pointersfromctx(ctx).values(): - pointers[p.oid()] = p - return pointers.values() - -def pointersfromctx(ctx): - """return a dict {path: pointer} for given single changectx""" - result = {} - for f in ctx.files(): - if f not in ctx: - continue - fctx = ctx[f] - if not _islfs(fctx.filelog(), fctx.filenode()): - continue - try: - result[f] = pointer.deserialize(fctx.rawdata()) - except pointer.InvalidPointer as ex: - raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n') - % (f, short(ctx.node()), ex)) - return result - -def uploadblobs(repo, pointers): - """upload given pointers from local blobstore""" - if not pointers: - return - - remoteblob = repo.svfs.lfsremoteblobstore - remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore) diff --git a/tests/test-lfs-pointer.py b/tests/test-lfs-pointer.py deleted file mode 100644 --- a/tests/test-lfs-pointer.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import absolute_import, print_function - -import os -import sys - -# make it runnable using python directly without run-tests.py -sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')] - -from hgext3rd.lfs import pointer - -def tryparse(text): - r = {} - try: - r = pointer.deserialize(text) - print('ok') - except Exception as ex: - print(ex) - if r: - text2 = r.serialize() - if text2 != text: - print('reconstructed text differs') - return r - -t = ('version https://git-lfs.github.com/spec/v1\n' - 'oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1' - '258daaa5e2ca24d17e2393\n' - 'size 12345\n' - 'x-foo extra-information\n') - -tryparse('') -tryparse(t) -tryparse(t.replace('git-lfs', 'unknown')) -tryparse(t.replace('v1\n', 'v1\n\n')) -tryparse(t.replace('sha256', 'ahs256')) -tryparse(t.replace('sha256:', '')) -tryparse(t.replace('12345', '0x12345')) -tryparse(t.replace('extra-information', 'extra\0information')) -tryparse(t.replace('extra-information', 'extra\ninformation')) -tryparse(t.replace('x-foo', 'x_foo')) -tryparse(t.replace('oid', 'blobid')) -tryparse(t.replace('size', 'size-bytes').replace('oid', 'object-id')) diff --git a/tests/test-lfs-pointer.py.out b/tests/test-lfs-pointer.py.out deleted file mode 100644 --- a/tests/test-lfs-pointer.py.out +++ /dev/null @@ -1,12 +0,0 @@ -missed keys: oid, size -ok -unexpected value: version='https://unknown.github.com/spec/v1' -cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\n\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra-information\n' -unexpected value: oid='ahs256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' -unexpected value: oid='4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' -unexpected value: size='0x12345' -ok -cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra\ninformation\n' -unexpected key: x_foo -missed keys: oid -missed keys: oid, size diff --git a/tests/test-lfs-test-server.t b/tests/test-lfs-test-server.t deleted file mode 100644 --- a/tests/test-lfs-test-server.t +++ /dev/null @@ -1,108 +0,0 @@ -Require lfs-test-server (https://github.com/git-lfs/lfs-test-server) - - $ hash lfs-test-server || { echo 'skipped: missing lfs-test-server'; exit 80; } - - $ LFS_LISTEN="tcp://:$HGPORT" - $ LFS_HOST="localhost:$HGPORT" - $ LFS_PUBLIC=1 - $ export LFS_LISTEN LFS_HOST LFS_PUBLIC - $ lfs-test-server &> lfs-server.log & - $ echo $! >> $DAEMON_PIDS - - $ cat >> $HGRCPATH < [extensions] - > lfs=$TESTDIR/../hgext3rd/lfs - > [lfs] - > url=http://foo:bar@$LFS_HOST/ - > threshold=1 - > EOF - - $ hg init repo1 - $ cd repo1 - $ echo THIS-IS-LFS > a - $ hg commit -m a -A a - - $ hg init ../repo2 - $ hg push ../repo2 -v - pushing to ../repo2 - searching for changes - lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) - 1 changesets found - uncompressed size of bundle content: - * (changelog) (glob) - * (manifests) (glob) - * a (glob) - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - - $ cd ../repo2 - $ hg update tip -v - resolving manifests - getting a - lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - -When the server has some blobs already - - $ hg mv a b - $ echo ANOTHER-LARGE-FILE > c - $ echo ANOTHER-LARGE-FILE2 > d - $ hg commit -m b-and-c -A b c d - $ hg push ../repo1 -v | grep -v '^ ' - pushing to ../repo1 - searching for changes - lfs: need to transfer 2 objects (39 bytes) - lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) - lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) - 1 changesets found - uncompressed size of bundle content: - adding changesets - adding manifests - adding file changes - added 1 changesets with 3 changes to 3 files - - $ hg --repo ../repo1 update tip -v - resolving manifests - getting b - getting c - lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) - getting d - lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) - 3 files updated, 0 files merged, 0 files removed, 0 files unresolved - -Check error message when the remote missed a blob: - - $ echo FFFFF > b - $ hg commit -m b -A b - $ echo FFFFF >> b - $ hg commit -m b b - $ rm -rf .hg/store/lfs - $ hg update -C '.^' - abort: LFS server claims required objects do not exist: - 8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13! - [255] - -Check error message when object does not exist: - - $ hg init test && cd test - $ echo "[extensions]" >> .hg/hgrc - $ echo "lfs=" >> .hg/hgrc - $ echo "[lfs]" >> .hg/hgrc - $ echo "threshold=1" >> .hg/hgrc - $ echo a > a - $ hg add a - $ hg commit -m 'test' - $ echo aaaaa > a - $ hg commit -m 'largefile' - $ hg debugdata .hg/store/data/a.i 1 # verify this is no the file content but includes "oid", the LFS "pointer". - version https://git-lfs.github.com/spec/v1 - oid sha256:bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a - size 6 - x-is-binary 0 - $ cd .. - $ hg --config 'lfs.url=https://dewey-lfs.vip.facebook.com/lfs' clone test test2 - updating to branch default - abort: LFS server error. Remote object for file data/a.i not found:(.*)! (re) - [255] diff --git a/tests/test-lfs.t b/tests/test-lfs.t deleted file mode 100644 --- a/tests/test-lfs.t +++ /dev/null @@ -1,544 +0,0 @@ -# Initial setup - - $ cat >> $HGRCPATH << EOF - > [extensions] - > lfs=$TESTDIR/../hgext3rd/lfs/ - > [lfs] - > threshold=1000B - > EOF - - $ LONG=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC - -# Prepare server and enable extension - $ hg init server - $ hg clone -q server client - $ cd client - -# Commit small file - $ echo s > smallfile - $ hg commit -Aqm "add small file" - -# Commit large file - $ echo $LONG > largefile - $ hg commit --traceback -Aqm "add large file" - -# Ensure metadata is stored - $ hg debugdata largefile 0 - version https://git-lfs.github.com/spec/v1 - oid sha256:f11e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b - size 1501 - x-is-binary 0 - -# Check the blobstore is populated - $ find .hg/store/lfs/objects | sort - .hg/store/lfs/objects - .hg/store/lfs/objects/f1 - .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b - -# Check the blob stored contains the actual contents of the file - $ cat .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC - -# Push changes to the server - - $ hg push - pushing to $TESTTMP/server (glob) - searching for changes - abort: lfs.url needs to be configured - [255] - - $ cat >> $HGRCPATH << EOF - > [lfs] - > url=file:$TESTTMP/dummy-remote/ - > EOF - - $ hg push -v | egrep -v '^(uncompressed| )' - pushing to $TESTTMP/server (glob) - searching for changes - 2 changesets found - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 2 files - -# Unknown URL scheme - - $ hg push --config lfs.url=ftp://foobar - abort: lfs: unknown url scheme: ftp - [255] - - $ cd ../ - -# Initialize new client (not cloning) and setup extension - $ hg init client2 - $ cd client2 - $ cat >> .hg/hgrc < [paths] - > default = $TESTTMP/server - > EOF - -# Pull from server - $ hg pull default - pulling from $TESTTMP/server (glob) - requesting all changes - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 2 files - new changesets b29ba743f89d:00c137947d30 - (run 'hg update' to get a working copy) - -# Check the blobstore is not yet populated - $ [ -d .hg/store/lfs/objects ] - [1] - -# Update to the last revision containing the large file - $ hg update - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved - -# Check the blobstore has been populated on update - $ find .hg/store/lfs/objects | sort - .hg/store/lfs/objects - .hg/store/lfs/objects/f1 - .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b - -# Check the contents of the file are fetched from blobstore when requested - $ hg cat -r . largefile - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC - -# Check the file has been copied in the working copy - $ cat largefile - AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC - - $ cd .. - -# Check rename, and switch between large and small files - - $ hg init repo3 - $ cd repo3 - $ cat >> .hg/hgrc << EOF - > [lfs] - > threshold=10B - > EOF - - $ echo LONGER-THAN-TEN-BYTES-WILL-TRIGGER-LFS > large - $ echo SHORTER > small - $ hg add . -q - $ hg commit -m 'commit with lfs content' - - $ hg mv large l - $ hg mv small s - $ hg commit -m 'renames' - - $ echo SHORT > l - $ echo BECOME-LARGER-FROM-SHORTER > s - $ hg commit -m 'large to small, small to large' - - $ echo 1 >> l - $ echo 2 >> s - $ hg commit -m 'random modifications' - - $ echo RESTORE-TO-BE-LARGE > l - $ echo SHORTER > s - $ hg commit -m 'switch large and small again' - -# Test lfs_files template - - $ hg log -r 'all()' -T '{rev} {join(lfs_files, ", ")}\n' - 0 large - 1 l - 2 s - 3 s - 4 l - -# Push and pull the above repo - - $ hg --cwd .. init repo4 - $ hg push ../repo4 - pushing to ../repo4 - searching for changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 10 changes to 4 files - - $ hg --cwd .. init repo5 - $ hg --cwd ../repo5 pull ../repo3 - pulling from ../repo3 - requesting all changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 10 changes to 4 files - new changesets fd47a419c4f7:5adf850972b9 - (run 'hg update' to get a working copy) - - $ cd .. - -# Test clone - - $ hg init repo6 - $ cd repo6 - $ cat >> .hg/hgrc << EOF - > [lfs] - > threshold=30B - > EOF - - $ echo LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES > large - $ echo SMALL > small - $ hg commit -Aqm 'create a lfs file' large small - $ hg debuglfsupload -r 'all()' -v - - $ cd .. - - $ hg clone repo6 repo7 - updating to branch default - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ cd repo7 - $ cat large - LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES - $ cat small - SMALL - - $ cd .. - -# Test rename and status - - $ hg init repo8 - $ cd repo8 - $ cat >> .hg/hgrc << EOF - > [lfs] - > threshold=10B - > EOF - - $ echo THIS-IS-LFS-BECAUSE-10-BYTES > a1 - $ echo SMALL > a2 - $ hg commit -m a -A a1 a2 - $ hg status - $ hg mv a1 b1 - $ hg mv a2 a1 - $ hg mv b1 a2 - $ hg commit -m b - $ hg status - $ HEADER=$'\1\n' - $ printf '%sSTART-WITH-HG-FILELOG-METADATA' "$HEADER" > a2 - $ printf '%sMETA\n' "$HEADER" > a1 - $ hg commit -m meta - $ hg status - $ hg log -T '{rev}: {file_copies} | {file_dels} | {file_adds}\n' - 2: | | - 1: a1 (a2)a2 (a1) | | - 0: | | a1 a2 - - $ for n in a1 a2; do - > for r in 0 1 2; do - > printf '\n%s @ %s\n' $n $r - > hg debugdata $n $r - > done - > done - - a1 @ 0 - version https://git-lfs.github.com/spec/v1 - oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 - size 29 - x-is-binary 0 - - a1 @ 1 - \x01 (esc) - copy: a2 - copyrev: 50470ad23cf937b1f4b9f80bfe54df38e65b50d9 - \x01 (esc) - SMALL - - a1 @ 2 - \x01 (esc) - \x01 (esc) - \x01 (esc) - META - - a2 @ 0 - SMALL - - a2 @ 1 - version https://git-lfs.github.com/spec/v1 - oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 - size 29 - x-hg-copy a1 - x-hg-copyrev be23af27908a582af43e5cda209a5a9b319de8d4 - x-is-binary 0 - - a2 @ 2 - version https://git-lfs.github.com/spec/v1 - oid sha256:876dadc86a8542f9798048f2c47f51dbf8e4359aed883e8ec80c5db825f0d943 - size 32 - x-is-binary 0 - -# Verify commit hashes include rename metadata - - $ hg log -T '{rev}:{node|short} {desc}\n' - 2:0fae949de7fa meta - 1:9cd6bdffdac0 b - 0:7f96794915f7 a - - $ cd .. - -# Test bundle - - $ hg init repo9 - $ cd repo9 - $ cat >> .hg/hgrc << EOF - > [lfs] - > threshold=10B - > [diff] - > git=1 - > EOF - - $ for i in 0 single two three 4; do - > echo 'THIS-IS-LFS-'$i > a - > hg commit -m a-$i -A a - > done - - $ hg update 2 -q - $ echo 'THIS-IS-LFS-2-CHILD' > a - $ hg commit -m branching -q - - $ hg bundle --base 1 bundle.hg -v - 4 changesets found - uncompressed size of bundle content: - * (changelog) (glob) - * (manifests) (glob) - * a (glob) - $ hg --config extensions.strip= strip -r 2 --no-backup --force -q - $ hg -R bundle.hg log -p -T '{rev} {desc}\n' a - 5 branching - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-two - +THIS-IS-LFS-2-CHILD - - 4 a-4 - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-three - +THIS-IS-LFS-4 - - 3 a-three - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-two - +THIS-IS-LFS-three - - 2 a-two - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-single - +THIS-IS-LFS-two - - 1 a-single - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-0 - +THIS-IS-LFS-single - - 0 a-0 - diff --git a/a b/a - new file mode 100644 - --- /dev/null - +++ b/a - @@ -0,0 +1,1 @@ - +THIS-IS-LFS-0 - - $ hg bundle -R bundle.hg --base 1 bundle-again.hg -q - $ hg -R bundle-again.hg log -p -T '{rev} {desc}\n' a - 5 branching - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-two - +THIS-IS-LFS-2-CHILD - - 4 a-4 - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-three - +THIS-IS-LFS-4 - - 3 a-three - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-two - +THIS-IS-LFS-three - - 2 a-two - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-single - +THIS-IS-LFS-two - - 1 a-single - diff --git a/a b/a - --- a/a - +++ b/a - @@ -1,1 +1,1 @@ - -THIS-IS-LFS-0 - +THIS-IS-LFS-single - - 0 a-0 - diff --git a/a b/a - new file mode 100644 - --- /dev/null - +++ b/a - @@ -0,0 +1,1 @@ - +THIS-IS-LFS-0 - - $ cd .. - -# Test isbinary - - $ hg init repo10 - $ cd repo10 - $ cat >> .hg/hgrc << EOF - > [extensions] - > lfs=$TESTDIR/../hgext3rd/lfs/ - > [lfs] - > threshold=1 - > EOF - $ $PYTHON <<'EOF' - > def write(path, content): - > with open(path, 'wb') as f: - > f.write(content) - > write('a', b'\0\0') - > write('b', b'\1\n') - > write('c', b'\1\n\0') - > write('d', b'xx') - > EOF - $ hg add a b c d - $ hg diff --stat - a | Bin - b | 1 + - c | Bin - d | 1 + - 4 files changed, 2 insertions(+), 0 deletions(-) - $ hg commit -m binarytest - $ cat > $TESTTMP/dumpbinary.py << EOF - > def reposetup(ui, repo): - > for n in 'abcd': - > ui.write(('%s: binary=%s\n') % (n, repo['.'][n].isbinary())) - > EOF - $ hg --config extensions.dumpbinary=$TESTTMP/dumpbinary.py id --trace - a: binary=True - b: binary=False - c: binary=True - d: binary=False - b55353847f02 tip - - $ cd .. - -# Test fctx.cmp fastpath - diff without LFS blobs - - $ hg init repo11 - $ cd repo11 - $ cat >> .hg/hgrc < [lfs] - > threshold=1 - > EOF - $ for i in 1 2 3; do - > cp ../repo10/a a - > if [ $i = 3 ]; then - > # make a content-only change - > chmod +x a - > i=2 - > fi - > echo $i >> a - > hg commit -m $i -A a - > done - $ [ -d .hg/store/lfs/objects ] - - $ cd .. - - $ hg clone repo11 repo12 --noupdate - $ cd repo12 - $ hg log --removed -p a -T '{desc}\n' --config diff.nobinary=1 --git - 2 - diff --git a/a b/a - old mode 100644 - new mode 100755 - - 2 - diff --git a/a b/a - Binary file a has changed - - 1 - diff --git a/a b/a - new file mode 100644 - Binary file a has changed - - $ [ -d .hg/store/lfs/objects ] - [1] - - $ cd .. - -# Verify the repos - - $ cat > $TESTTMP/dumpflog.py << EOF - > # print raw revision sizes, flags, and hashes for certain files - > import hashlib - > from mercurial import revlog - > from mercurial.node import short - > def hash(rawtext): - > h = hashlib.sha512() - > h.update(rawtext) - > return h.hexdigest()[:4] - > def reposetup(ui, repo): - > # these 2 files are interesting - > for name in ['l', 's']: - > fl = repo.file(name) - > if len(fl) == 0: - > continue - > sizes = [revlog.revlog.rawsize(fl, i) for i in fl] - > texts = [fl.revision(i, raw=True) for i in fl] - > flags = [fl.flags(i) for i in fl] - > hashes = [hash(t) for t in texts] - > print(' %s: rawsizes=%r flags=%r hashes=%r' - > % (name, sizes, flags, hashes)) - > EOF - - $ for i in client client2 server repo3 repo4 repo5 repo6 repo7 repo8 repo9 \ - > repo10; do - > echo 'repo:' $i - > hg --cwd $i verify --config extensions.dumpflog=$TESTTMP/dumpflog.py -q - > done - repo: client - repo: client2 - repo: server - repo: repo3 - l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] - s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] - repo: repo4 - l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] - s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] - repo: repo5 - l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] - s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] - repo: repo6 - repo: repo7 - repo: repo8 - repo: repo9 - repo: repo10