This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3914

lfs: delete now that it's in upstream
ClosedPublic

Authored by martinvonz on Jul 10 2018, 5:07 PM.
Tags
None
Subscribers
None

Details

Reviewers
quark
Group Reviewers
Restricted Project
Commits
rFBHGX04b9aa619610: lfs: delete now that it's in upstream
Summary

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

martinvonz created this revision.Jul 10 2018, 5:07 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptJul 10 2018, 5:07 PM
quark accepted this revision.Jul 10 2018, 5:55 PM
This revision is now accepted and ready to land.Jul 10 2018, 5:55 PM
Closed by commit rFBHGX04b9aa619610: lfs: delete now that it's in upstream (authored by martinvonz). · Explain WhyJul 10 2018, 6:46 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 9516

hgext3rd/lfs/__init__.py

  • This file was deleted.
# lfs - hash-preserving large file support using Git-LFS protocol
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""lfs - large file support
Configs::
[lfs]
# Remote endpoint. Multiple protocols are supported:
# - http(s)://user:pass@example.com/path
# git-lfs endpoint
# - file:///tmp/path
# local filesystem, usually for testing
# if unset, lfs will prompt setting this when it must use this value.
# (default: unset)
url = https://example.com/lfs
# size of a file to make it use LFS
threshold = 10M
# how many times to retry before giving up on transferring an object
retry = 5
"""
from __future__ import absolute_import
from mercurial import (
bundle2,
changegroup,
context,
exchange,
extensions,
filelog,
registrar,
revlog,
scmutil,
vfs as vfsmod,
)
from mercurial.i18n import _
from . import (
blobstore,
wrapper,
)
cmdtable = {}
command = registrar.command(cmdtable)
templatekeyword = registrar.templatekeyword()
def reposetup(ui, repo):
# Nothing to do with a remote repo
if not repo.local():
return
threshold = repo.ui.configbytes('lfs', 'threshold', None)
repo.svfs.options['lfsthreshold'] = threshold
repo.svfs.lfslocalblobstore = blobstore.local(repo)
repo.svfs.lfsremoteblobstore = blobstore.remote(repo)
# Push hook
repo.prepushoutgoinghooks.add('lfs', wrapper.prepush)
def wrapfilelog(filelog):
wrapfunction = extensions.wrapfunction
wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision)
wrapfunction(filelog, 'renamed', wrapper.filelogrenamed)
wrapfunction(filelog, 'size', wrapper.filelogsize)
def extsetup(ui):
wrapfilelog(filelog.filelog)
wrapfunction = extensions.wrapfunction
wrapfunction(changegroup,
'supportedoutgoingversions',
wrapper.supportedoutgoingversions)
wrapfunction(changegroup,
'allsupportedversions',
wrapper.allsupportedversions)
wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp)
wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary)
context.basefilectx.islfs = wrapper.filectxislfs
revlog.addflagprocessor(
revlog.REVIDX_EXTSTORED,
(
wrapper.readfromstore,
wrapper.writetostore,
wrapper.bypasscheckhash,
),
)
# Make bundle choose changegroup3 instead of changegroup2. This affects
# "hg bundle" command. Note: it does not cover all bundle formats like
# "packed1". Using "packed1" with lfs will likely cause trouble.
names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02']
for k in names:
exchange._bundlespeccgversions[k] = '03'
# bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs
# options and blob stores are passed from othervfs to the new readonlyvfs.
wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit)
# when writing a bundle via "hg bundle" command, upload related LFS blobs
wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle)
@templatekeyword('lfs_files')
def lfsfiles(repo, ctx, **args):
"""List of strings. LFS files added or modified by the changeset."""
pointers = wrapper.pointersfromctx(ctx) # {path: pointer}
return sorted(pointers.keys())
@command('debuglfsupload',
[('r', 'rev', [], _('upload large files introduced by REV'))])
def debuglfsupload(ui, repo, **opts):
"""upload lfs blobs added by the working copy parent or given revisions"""
revs = opts.get('rev', [])
pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs))
wrapper.uploadblobs(repo, pointers)

hgext3rd/lfs/blobstore.py

  • This file was deleted.
# blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import json
import os
import re
from mercurial import (
error,
url as urlmod,
util,
vfs as vfsmod,
worker,
)
from mercurial.i18n import _
# 64 bytes for SHA256
_lfsre = re.compile(r'\A[a-f0-9]{64}\Z')
class lfsvfs(vfsmod.vfs):
def join(self, path):
"""split the path at first two characters, like: XX/XXXXX..."""
if not _lfsre.match(path):
raise error.ProgrammingError('unexpected lfs path: %s' % path)
return super(lfsvfs, self).join(path[0:2], path[2:])
class filewithprogress(object):
"""a file-like object that supports __len__ and read.
Useful to provide progress information for how many bytes are read.
"""
def __init__(self, fp, callback):
self._fp = fp
self._callback = callback # func(readsize)
fp.seek(0, os.SEEK_END)
self._len = fp.tell()
fp.seek(0)
def __len__(self):
return self._len
def read(self, size):
if self._fp is None:
return b''
data = self._fp.read(size)
if data:
if self._callback:
self._callback(len(data))
else:
self._fp.close()
self._fp = None
return data
class local(object):
"""Local blobstore for large file contents.
This blobstore is used both as a cache and as a staging area for large blobs
to be uploaded to the remote blobstore.
"""
def __init__(self, repo):
fullpath = repo.svfs.join('lfs/objects')
self.vfs = lfsvfs(fullpath)
def write(self, oid, data):
"""Write blob to local blobstore."""
with self.vfs(oid, 'wb', atomictemp=True) as fp:
fp.write(data)
def read(self, oid):
"""Read blob from local blobstore."""
return self.vfs.read(oid)
def has(self, oid):
"""Returns True if the local blobstore contains the requested blob,
False otherwise."""
return self.vfs.exists(oid)
class _gitlfsremote(object):
def __init__(self, repo, url):
ui = repo.ui
self.ui = ui
baseurl, authinfo = url.authinfo()
self.baseurl = baseurl.rstrip('/')
self.urlopener = urlmod.opener(ui, authinfo)
self.retry = ui.configint('lfs', 'retry') or 5
def writebatch(self, pointers, fromstore):
"""Batch upload from local to remote blobstore."""
self._batch(pointers, fromstore, 'upload')
def readbatch(self, pointers, tostore):
"""Batch download from remote to local blostore."""
self._batch(pointers, tostore, 'download')
def _batchrequest(self, pointers, action):
"""Get metadata about objects pointed by pointers for given action
Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
"""
objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers]
requestdata = json.dumps({
'objects': objects,
'operation': action,
})
batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
data=requestdata)
batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
try:
rawjson = self.urlopener.open(batchreq).read()
except util.urlerr.httperror as ex:
raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
% (ex, action))
try:
response = json.loads(rawjson)
except ValueError:
raise LfsRemoteError(_('LFS server returns invalid JSON: %s')
% rawjson)
return response
def _checkforservererror(self, pointers, responses):
"""Scans errors from objects
Returns LfsRemoteError if any objects has an error"""
for response in responses:
error = response.get('error')
if error:
ptrmap = {p.oid(): p for p in pointers}
p = ptrmap.get(response['oid'], None)
if error['code'] == 404 and p:
filename = getattr(p, 'filename', 'unknown')
raise LfsRemoteError(
_(('LFS server error. Remote object '
'for file %s not found: %r')) % (filename, response))
raise LfsRemoteError(_('LFS server error: %r') % response)
def _extractobjects(self, response, pointers, action):
"""extract objects from response of the batch API
response: parsed JSON object returned by batch API
return response['objects'] filtered by action
raise if any object has an error
"""
# Scan errors from objects - fail early
objects = response.get('objects', [])
self._checkforservererror(pointers, objects)
# Filter objects with given action. Practically, this skips uploading
# objects which exist in the server.
filteredobjects = [o for o in objects if action in o.get('actions', [])]
# But for downloading, we want all objects. Therefore missing objects
# should be considered an error.
if action == 'download':
if len(filteredobjects) < len(objects):
missing = [o.get('oid', '?')
for o in objects
if action not in o.get('actions', [])]
raise LfsRemoteError(
_('LFS server claims required objects do not exist:\n%s')
% '\n'.join(missing))
return filteredobjects
def _basictransfer(self, obj, action, localstore):
"""Download or upload a single object using basic transfer protocol
obj: dict, an object description returned by batch API
action: string, one of ['upload', 'download']
localstore: blobstore.local
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
basic-transfers.md
"""
oid = str(obj['oid'])
href = str(obj['actions'][action].get('href'))
headers = obj['actions'][action].get('header', {}).items()
request = util.urlreq.request(href)
if action == 'upload':
# If uploading blobs, read data from local blobstore.
request.data = localstore.vfs(oid)
request.get_method = lambda: 'PUT'
for k, v in headers:
request.add_header(k, v)
response = b''
try:
req = self.urlopener.open(request)
while True:
data = req.read(1048576)
if not data:
break
response += data
except util.urlerr.httperror as ex:
raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
% (ex, oid, action))
if action == 'download':
# If downloading blobs, store downloaded data to local blobstore
localstore.write(oid, response)
def _batch(self, pointers, localstore, action):
if action not in ['upload', 'download']:
raise error.ProgrammingError('invalid Git-LFS action: %s' % action)
response = self._batchrequest(pointers, action)
objects = self._extractobjects(response, pointers, action)
total = sum(x.get('size', 0) for x in objects)
sizes = {}
for obj in objects:
sizes[obj.get('oid')] = obj.get('size', 0)
topic = {'upload': _('lfs uploading'),
'download': _('lfs downloading')}[action]
if self.ui.verbose and len(objects) > 1:
self.ui.write(_('lfs: need to transfer %d objects (%s)\n')
% (len(objects), util.bytecount(total)))
self.ui.progress(topic, 0, total=total)
def transfer(chunk):
for obj in chunk:
objsize = obj.get('size', 0)
if self.ui.verbose:
if action == 'download':
msg = _('lfs: downloading %s (%s)\n')
elif action == 'upload':
msg = _('lfs: uploading %s (%s)\n')
self.ui.write(msg % (obj.get('oid'),
util.bytecount(objsize)))
retry = self.retry
while True:
try:
self._basictransfer(obj, action, localstore)
yield 1, obj.get('oid')
break
except Exception as ex:
if retry > 0:
if self.ui.verbose:
self.ui.write(
_('lfs: failed: %r (remaining retry %d)\n')
% (ex, retry))
retry -= 1
continue
raise
oids = worker.worker(self.ui, 0.1, transfer, (),
sorted(objects, key=lambda o: o.get('oid')))
processed = 0
for _one, oid in oids:
processed += sizes[oid]
self.ui.progress(topic, processed, total=total)
if self.ui.verbose:
self.ui.write(_('lfs: processed: %s\n') % oid)
self.ui.progress(topic, pos=None, total=total)
def __del__(self):
# copied from mercurial/httppeer.py
urlopener = getattr(self, 'urlopener', None)
if urlopener:
for h in urlopener.handlers:
h.close()
getattr(h, "close_all", lambda : None)()
class _dummyremote(object):
"""Dummy store storing blobs to temp directory."""
def __init__(self, repo, url):
fullpath = repo.vfs.join('lfs', url.path)
self.vfs = lfsvfs(fullpath)
def writebatch(self, pointers, fromstore):
for p in pointers:
content = fromstore.read(p.oid())
with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
fp.write(content)
def readbatch(self, pointers, tostore):
for p in pointers:
content = self.vfs.read(p.oid())
tostore.write(p.oid(), content)
class _nullremote(object):
"""Null store storing blobs to /dev/null."""
def __init__(self, repo, url):
pass
def writebatch(self, pointers, fromstore):
pass
def readbatch(self, pointers, tostore):
pass
class _promptremote(object):
"""Prompt user to set lfs.url when accessed."""
def __init__(self, repo, url):
pass
def writebatch(self, pointers, fromstore, ui=None):
self._prompt()
def readbatch(self, pointers, tostore, ui=None):
self._prompt()
def _prompt(self):
raise error.Abort(_('lfs.url needs to be configured'))
_storemap = {
'https': _gitlfsremote,
'http': _gitlfsremote,
'file': _dummyremote,
'null': _nullremote,
None: _promptremote,
}
def remote(repo):
"""remotestore factory. return a store in _storemap depending on config"""
defaulturl = ''
# convert deprecated configs to the new url. TODO: remove this if other
# places are migrated to the new url config.
# deprecated config: lfs.remotestore
deprecatedstore = repo.ui.config('lfs', 'remotestore')
if deprecatedstore == 'dummy':
# deprecated config: lfs.remotepath
defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath')
elif deprecatedstore == 'git-lfs':
# deprecated config: lfs.remoteurl
defaulturl = repo.ui.config('lfs', 'remoteurl')
elif deprecatedstore == 'null':
defaulturl = 'null://'
url = util.url(repo.ui.config('lfs', 'url', defaulturl))
scheme = url.scheme
if scheme not in _storemap:
raise error.Abort(_('lfs: unknown url scheme: %s') % scheme)
return _storemap[scheme](repo, url)
class LfsRemoteError(error.RevlogError):
pass

hgext3rd/lfs/pointer.py

  • This file was deleted.
# pointer.py - Git-LFS pointer serialization
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import re
from mercurial import (
error,
)
from mercurial.i18n import _
class InvalidPointer(error.RevlogError):
pass
class gitlfspointer(dict):
VERSION = 'https://git-lfs.github.com/spec/v1'
def __init__(self, *args, **kwargs):
self['version'] = self.VERSION
super(gitlfspointer, self).__init__(*args, **kwargs)
@classmethod
def deserialize(cls, text):
try:
return cls(l.split(' ', 1) for l in text.splitlines()).validate()
except ValueError: # l.split returns 1 item instead of 2
raise InvalidPointer(_('cannot parse git-lfs text: %r') % text)
def serialize(self):
sortkeyfunc = lambda x: (x[0] != 'version', x)
items = sorted(self.validate().iteritems(), key=sortkeyfunc)
return ''.join('%s %s\n' % (k, v) for k, v in items)
def oid(self):
return self['oid'].split(':')[-1]
def size(self):
return int(self['size'])
# regular expressions used by _validate
# see https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
_keyre = re.compile(r'\A[a-z0-9.-]+\Z')
_valuere = re.compile(r'\A[^\n]*\Z')
_requiredre = {
'size': re.compile(r'\A[0-9]+\Z'),
'oid': re.compile(r'\Asha256:[0-9a-f]{64}\Z'),
'version': re.compile(r'\A%s\Z' % re.escape(VERSION)),
}
def validate(self):
"""raise InvalidPointer on error. return self if there is no error"""
requiredcount = 0
for k, v in self.iteritems():
if k in self._requiredre:
if not self._requiredre[k].match(v):
raise InvalidPointer(_('unexpected value: %s=%r') % (k, v))
requiredcount += 1
elif not self._keyre.match(k):
raise InvalidPointer(_('unexpected key: %s') % k)
if not self._valuere.match(v):
raise InvalidPointer(_('unexpected value: %s=%r') % (k, v))
if len(self._requiredre) != requiredcount:
miss = sorted(set(self._requiredre.keys()).difference(self.keys()))
raise InvalidPointer(_('missed keys: %s') % ', '.join(miss))
return self
deserialize = gitlfspointer.deserialize

hgext3rd/lfs/wrapper.py

  • This file was deleted.
# wrapper.py - methods wrapping core mercurial logic
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import hashlib
from mercurial import (
error,
filelog,
revlog,
util,
)
from mercurial.i18n import _
from mercurial.node import bin, nullid, short
from mercurial.utils import stringutil
from . import (
blobstore,
pointer,
)
if util.safehasattr(revlog, 'parsemeta'):
# Since 0596d274
_parsemeta = revlog.parsemeta
_packmeta = revlog.packmeta
else:
_parsemeta = filelog.parsemeta
_packmeta = filelog.packmeta
def supportedoutgoingversions(orig, repo):
versions = orig(repo)
versions.discard('01')
versions.discard('02')
versions.add('03')
return versions
def allsupportedversions(orig, ui):
versions = orig(ui)
versions.add('03')
return versions
def bypasscheckhash(self, text):
return False
def readfromstore(self, text):
"""Read filelog content from local blobstore transform for flagprocessor.
Default tranform for flagprocessor, returning contents from blobstore.
Returns a 2-typle (text, validatehash) where validatehash is True as the
contents of the blobstore should be checked using checkhash.
"""
p = pointer.deserialize(text)
oid = p.oid()
store = self.opener.lfslocalblobstore
if not store.has(oid):
p.filename = getattr(self, 'indexfile', None)
self.opener.lfsremoteblobstore.readbatch([p], store)
text = store.read(oid)
# pack hg filelog metadata
hgmeta = {}
for k in p.keys():
if k.startswith('x-hg-'):
name = k[len('x-hg-'):]
hgmeta[name] = p[k]
if hgmeta or text.startswith('\1\n'):
text = _packmeta(hgmeta, text)
return (text, True)
def writetostore(self, text):
# hg filelog metadata (includes rename, etc)
hgmeta, offset = _parsemeta(text)
if offset and offset > 0:
# lfs blob does not contain hg filelog metadata
text = text[offset:]
# git-lfs only supports sha256
oid = hashlib.sha256(text).hexdigest()
self.opener.lfslocalblobstore.write(oid, text)
# replace contents with metadata
longoid = 'sha256:%s' % oid
metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text)))
# by default, we expect the content to be binary. however, LFS could also
# be used for non-binary content. add a special entry for non-binary data.
# this will be used by filectx.isbinary().
if not stringutil.binary(text):
# not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
metadata['x-is-binary'] = '0'
# translate hg filelog metadata to lfs metadata with "x-hg-" prefix
if hgmeta is not None:
for k, v in hgmeta.iteritems():
metadata['x-hg-%s' % k] = v
rawtext = metadata.serialize()
return (rawtext, False)
def _islfs(rlog, node=None, rev=None):
if rev is None:
if node is None:
# both None - likely working copy content where node is not ready
return False
rev = rlog.rev(node)
else:
node = rlog.node(rev)
if node == nullid:
return False
flags = rlog.flags(rev)
return bool(flags & revlog.REVIDX_EXTSTORED)
def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
cachedelta=None, node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
threshold = self.opener.options['lfsthreshold']
textlen = len(text)
# exclude hg rename meta from file size
meta, offset = _parsemeta(text)
if offset:
textlen -= offset
if threshold and textlen > threshold:
flags |= revlog.REVIDX_EXTSTORED
return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
node=node, flags=flags, **kwds)
def filelogrenamed(orig, self, node):
if _islfs(self, node):
rawtext = self.revision(node, raw=True)
if not rawtext:
return False
metadata = pointer.deserialize(rawtext)
if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
else:
return False
return orig(self, node)
def filelogsize(orig, self, rev):
if _islfs(self, rev=rev):
# fast path: use lfs metadata to answer size
rawtext = self.revision(rev, raw=True)
metadata = pointer.deserialize(rawtext)
return int(metadata['size'])
return orig(self, rev)
def filectxcmp(orig, self, fctx):
"""returns True if text is different than fctx"""
# some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
# fast path: check LFS oid
p1 = pointer.deserialize(self.rawdata())
p2 = pointer.deserialize(fctx.rawdata())
return p1.oid() != p2.oid()
return orig(self, fctx)
def filectxisbinary(orig, self):
if self.islfs():
# fast path: use lfs metadata to answer isbinary
metadata = pointer.deserialize(self.rawdata())
# if lfs metadata says nothing, assume it's binary by default
return bool(int(metadata.get('x-is-binary', 1)))
return orig(self)
def filectxislfs(self):
return _islfs(self.filelog(), self.filenode())
def vfsinit(orig, self, othervfs):
orig(self, othervfs)
# copy lfs related options
for k, v in othervfs.options.items():
if k.startswith('lfs'):
self.options[k] = v
# also copy lfs blobstores. note: this can run before reposetup, so lfs
# blobstore attributes are not always ready at this time.
for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
if util.safehasattr(othervfs, name):
setattr(self, name, getattr(othervfs, name))
def _canskipupload(repo):
# if remotestore is a null store, upload is a no-op and can be skipped
return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
def candownload(repo):
# if remotestore is a null store, downloads will lead to nothing
return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
def uploadblobsfromrevs(repo, revs):
'''upload lfs blobs introduced by revs
Note: also used by other extensions e. g. infinitepush. avoid renaming.
'''
if _canskipupload(repo):
return
pointers = extractpointers(repo, revs)
uploadblobs(repo, pointers)
def prepush(pushop):
"""Prepush hook.
Read through the revisions to push, looking for filelog entries that can be
deserialized into metadata so that we can block the push on their upload to
the remote blobstore.
"""
return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
*args, **kwargs):
"""upload LFS blobs added by outgoing revisions on 'hg bundle'"""
uploadblobsfromrevs(repo, outgoing.missing)
return orig(ui, repo, source, filename, bundletype, outgoing, *args,
**kwargs)
def extractpointers(repo, revs):
"""return a list of lfs pointers added by given revs"""
ui = repo.ui
if ui.debugflag:
ui.write(_('lfs: computing set of blobs to upload\n'))
pointers = {}
for r in revs:
ctx = repo[r]
for p in pointersfromctx(ctx).values():
pointers[p.oid()] = p
return pointers.values()
def pointersfromctx(ctx):
"""return a dict {path: pointer} for given single changectx"""
result = {}
for f in ctx.files():
if f not in ctx:
continue
fctx = ctx[f]
if not _islfs(fctx.filelog(), fctx.filenode()):
continue
try:
result[f] = pointer.deserialize(fctx.rawdata())
except pointer.InvalidPointer as ex:
raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
% (f, short(ctx.node()), ex))
return result
def uploadblobs(repo, pointers):
"""upload given pointers from local blobstore"""
if not pointers:
return
remoteblob = repo.svfs.lfsremoteblobstore
remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)

tests/test-lfs-pointer.py

  • This file was deleted.
from __future__ import absolute_import, print_function
import os
import sys
# make it runnable using python directly without run-tests.py
sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')]
from hgext3rd.lfs import pointer
def tryparse(text):
r = {}
try:
r = pointer.deserialize(text)
print('ok')
except Exception as ex:
print(ex)
if r:
text2 = r.serialize()
if text2 != text:
print('reconstructed text differs')
return r
t = ('version https://git-lfs.github.com/spec/v1\n'
'oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1'
'258daaa5e2ca24d17e2393\n'
'size 12345\n'
'x-foo extra-information\n')
tryparse('')
tryparse(t)
tryparse(t.replace('git-lfs', 'unknown'))
tryparse(t.replace('v1\n', 'v1\n\n'))
tryparse(t.replace('sha256', 'ahs256'))
tryparse(t.replace('sha256:', ''))
tryparse(t.replace('12345', '0x12345'))
tryparse(t.replace('extra-information', 'extra\0information'))
tryparse(t.replace('extra-information', 'extra\ninformation'))
tryparse(t.replace('x-foo', 'x_foo'))
tryparse(t.replace('oid', 'blobid'))
tryparse(t.replace('size', 'size-bytes').replace('oid', 'object-id'))

tests/test-lfs-pointer.py.out

  • This file was deleted.
missed keys: oid, size
ok
unexpected value: version='https://unknown.github.com/spec/v1'
cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\n\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra-information\n'
unexpected value: oid='ahs256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393'
unexpected value: oid='4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393'
unexpected value: size='0x12345'
ok
cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra\ninformation\n'
unexpected key: x_foo
missed keys: oid
missed keys: oid, size

tests/test-lfs-test-server.t

  • This file was deleted.
Require lfs-test-server (https://github.com/git-lfs/lfs-test-server)
$ hash lfs-test-server || { echo 'skipped: missing lfs-test-server'; exit 80; }
$ LFS_LISTEN="tcp://:$HGPORT"
$ LFS_HOST="localhost:$HGPORT"
$ LFS_PUBLIC=1
$ export LFS_LISTEN LFS_HOST LFS_PUBLIC
$ lfs-test-server &> lfs-server.log &
$ echo $! >> $DAEMON_PIDS
$ cat >> $HGRCPATH <<EOF
> [extensions]
> lfs=$TESTDIR/../hgext3rd/lfs
> [lfs]
> url=http://foo:bar@$LFS_HOST/
> threshold=1
> EOF
$ hg init repo1
$ cd repo1
$ echo THIS-IS-LFS > a
$ hg commit -m a -A a
$ hg init ../repo2
$ hg push ../repo2 -v
pushing to ../repo2
searching for changes
lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
1 changesets found
uncompressed size of bundle content:
* (changelog) (glob)
* (manifests) (glob)
* a (glob)
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
$ cd ../repo2
$ hg update tip -v
resolving manifests
getting a
lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
When the server has some blobs already
$ hg mv a b
$ echo ANOTHER-LARGE-FILE > c
$ echo ANOTHER-LARGE-FILE2 > d
$ hg commit -m b-and-c -A b c d
$ hg push ../repo1 -v | grep -v '^ '
pushing to ../repo1
searching for changes
lfs: need to transfer 2 objects (39 bytes)
lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
1 changesets found
uncompressed size of bundle content:
adding changesets
adding manifests
adding file changes
added 1 changesets with 3 changes to 3 files
$ hg --repo ../repo1 update tip -v
resolving manifests
getting b
getting c
lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
getting d
lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
3 files updated, 0 files merged, 0 files removed, 0 files unresolved
Check error message when the remote missed a blob:
$ echo FFFFF > b
$ hg commit -m b -A b
$ echo FFFFF >> b
$ hg commit -m b b
$ rm -rf .hg/store/lfs
$ hg update -C '.^'
abort: LFS server claims required objects do not exist:
8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13!
[255]
Check error message when object does not exist:
$ hg init test && cd test
$ echo "[extensions]" >> .hg/hgrc
$ echo "lfs=" >> .hg/hgrc
$ echo "[lfs]" >> .hg/hgrc
$ echo "threshold=1" >> .hg/hgrc
$ echo a > a
$ hg add a
$ hg commit -m 'test'
$ echo aaaaa > a
$ hg commit -m 'largefile'
$ hg debugdata .hg/store/data/a.i 1 # verify this is no the file content but includes "oid", the LFS "pointer".
version https://git-lfs.github.com/spec/v1
oid sha256:bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a
size 6
x-is-binary 0
$ cd ..
$ hg --config 'lfs.url=https://dewey-lfs.vip.facebook.com/lfs' clone test test2
updating to branch default
abort: LFS server error. Remote object for file data/a.i not found:(.*)! (re)
[255]

tests/test-lfs.t

  • This file was deleted.
# Initial setup
$ cat >> $HGRCPATH << EOF
> [extensions]
> lfs=$TESTDIR/../hgext3rd/lfs/
> [lfs]
> threshold=1000B
> EOF
$ LONG=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
# Prepare server and enable extension
$ hg init server
$ hg clone -q server client
$ cd client
# Commit small file
$ echo s > smallfile
$ hg commit -Aqm "add small file"
# Commit large file
$ echo $LONG > largefile
$ hg commit --traceback -Aqm "add large file"
# Ensure metadata is stored
$ hg debugdata largefile 0
version https://git-lfs.github.com/spec/v1
oid sha256:f11e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b
size 1501
x-is-binary 0
# Check the blobstore is populated
$ find .hg/store/lfs/objects | sort
.hg/store/lfs/objects
.hg/store/lfs/objects/f1
.hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b
# Check the blob stored contains the actual contents of the file
$ cat .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
# Push changes to the server
$ hg push
pushing to $TESTTMP/server (glob)
searching for changes
abort: lfs.url needs to be configured
[255]
$ cat >> $HGRCPATH << EOF
> [lfs]
> url=file:$TESTTMP/dummy-remote/
> EOF
$ hg push -v | egrep -v '^(uncompressed| )'
pushing to $TESTTMP/server (glob)
searching for changes
2 changesets found
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
# Unknown URL scheme
$ hg push --config lfs.url=ftp://foobar
abort: lfs: unknown url scheme: ftp
[255]
$ cd ../
# Initialize new client (not cloning) and setup extension
$ hg init client2
$ cd client2
$ cat >> .hg/hgrc <<EOF
> [paths]
> default = $TESTTMP/server
> EOF
# Pull from server
$ hg pull default
pulling from $TESTTMP/server (glob)
requesting all changes
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 2 files
new changesets b29ba743f89d:00c137947d30
(run 'hg update' to get a working copy)
# Check the blobstore is not yet populated
$ [ -d .hg/store/lfs/objects ]
[1]
# Update to the last revision containing the large file
$ hg update
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
# Check the blobstore has been populated on update
$ find .hg/store/lfs/objects | sort
.hg/store/lfs/objects
.hg/store/lfs/objects/f1
.hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b
# Check the contents of the file are fetched from blobstore when requested
$ hg cat -r . largefile
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
# Check the file has been copied in the working copy
$ cat largefile
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC
$ cd ..
# Check rename, and switch between large and small files
$ hg init repo3
$ cd repo3
$ cat >> .hg/hgrc << EOF
> [lfs]
> threshold=10B
> EOF
$ echo LONGER-THAN-TEN-BYTES-WILL-TRIGGER-LFS > large
$ echo SHORTER > small
$ hg add . -q
$ hg commit -m 'commit with lfs content'
$ hg mv large l
$ hg mv small s
$ hg commit -m 'renames'
$ echo SHORT > l
$ echo BECOME-LARGER-FROM-SHORTER > s
$ hg commit -m 'large to small, small to large'
$ echo 1 >> l
$ echo 2 >> s
$ hg commit -m 'random modifications'
$ echo RESTORE-TO-BE-LARGE > l
$ echo SHORTER > s
$ hg commit -m 'switch large and small again'
# Test lfs_files template
$ hg log -r 'all()' -T '{rev} {join(lfs_files, ", ")}\n'
0 large
1 l
2 s
3 s
4 l
# Push and pull the above repo
$ hg --cwd .. init repo4
$ hg push ../repo4
pushing to ../repo4
searching for changes
adding changesets
adding manifests
adding file changes
added 5 changesets with 10 changes to 4 files
$ hg --cwd .. init repo5
$ hg --cwd ../repo5 pull ../repo3
pulling from ../repo3
requesting all changes
adding changesets
adding manifests
adding file changes
added 5 changesets with 10 changes to 4 files
new changesets fd47a419c4f7:5adf850972b9
(run 'hg update' to get a working copy)
$ cd ..
# Test clone
$ hg init repo6
$ cd repo6
$ cat >> .hg/hgrc << EOF
> [lfs]
> threshold=30B
> EOF
$ echo LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES > large
$ echo SMALL > small
$ hg commit -Aqm 'create a lfs file' large small
$ hg debuglfsupload -r 'all()' -v
$ cd ..
$ hg clone repo6 repo7
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ cd repo7
$ cat large
LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES
$ cat small
SMALL
$ cd ..
# Test rename and status
$ hg init repo8
$ cd repo8
$ cat >> .hg/hgrc << EOF
> [lfs]
> threshold=10B
> EOF
$ echo THIS-IS-LFS-BECAUSE-10-BYTES > a1
$ echo SMALL > a2
$ hg commit -m a -A a1 a2
$ hg status
$ hg mv a1 b1
$ hg mv a2 a1
$ hg mv b1 a2
$ hg commit -m b
$ hg status
$ HEADER=$'\1\n'
$ printf '%sSTART-WITH-HG-FILELOG-METADATA' "$HEADER" > a2
$ printf '%sMETA\n' "$HEADER" > a1
$ hg commit -m meta
$ hg status
$ hg log -T '{rev}: {file_copies} | {file_dels} | {file_adds}\n'
2: | |
1: a1 (a2)a2 (a1) | |
0: | | a1 a2
$ for n in a1 a2; do
> for r in 0 1 2; do
> printf '\n%s @ %s\n' $n $r
> hg debugdata $n $r
> done
> done
a1 @ 0
version https://git-lfs.github.com/spec/v1
oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024
size 29
x-is-binary 0
a1 @ 1
\x01 (esc)
copy: a2
copyrev: 50470ad23cf937b1f4b9f80bfe54df38e65b50d9
\x01 (esc)
SMALL
a1 @ 2
\x01 (esc)
\x01 (esc)
\x01 (esc)
META
a2 @ 0
SMALL
a2 @ 1
version https://git-lfs.github.com/spec/v1
oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024
size 29
x-hg-copy a1
x-hg-copyrev be23af27908a582af43e5cda209a5a9b319de8d4
x-is-binary 0
a2 @ 2
version https://git-lfs.github.com/spec/v1
oid sha256:876dadc86a8542f9798048f2c47f51dbf8e4359aed883e8ec80c5db825f0d943
size 32
x-is-binary 0
# Verify commit hashes include rename metadata
$ hg log -T '{rev}:{node|short} {desc}\n'
2:0fae949de7fa meta
1:9cd6bdffdac0 b
0:7f96794915f7 a
$ cd ..
# Test bundle
$ hg init repo9
$ cd repo9
$ cat >> .hg/hgrc << EOF
> [lfs]
> threshold=10B
> [diff]
> git=1
> EOF
$ for i in 0 single two three 4; do
> echo 'THIS-IS-LFS-'$i > a
> hg commit -m a-$i -A a
> done
$ hg update 2 -q
$ echo 'THIS-IS-LFS-2-CHILD' > a
$ hg commit -m branching -q
$ hg bundle --base 1 bundle.hg -v
4 changesets found
uncompressed size of bundle content:
* (changelog) (glob)
* (manifests) (glob)
* a (glob)
$ hg --config extensions.strip= strip -r 2 --no-backup --force -q
$ hg -R bundle.hg log -p -T '{rev} {desc}\n' a
5 branching
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-two
+THIS-IS-LFS-2-CHILD
4 a-4
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-three
+THIS-IS-LFS-4
3 a-three
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-two
+THIS-IS-LFS-three
2 a-two
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-single
+THIS-IS-LFS-two
1 a-single
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-0
+THIS-IS-LFS-single
0 a-0
diff --git a/a b/a
new file mode 100644
--- /dev/null
+++ b/a
@@ -0,0 +1,1 @@
+THIS-IS-LFS-0
$ hg bundle -R bundle.hg --base 1 bundle-again.hg -q
$ hg -R bundle-again.hg log -p -T '{rev} {desc}\n' a
5 branching
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-two
+THIS-IS-LFS-2-CHILD
4 a-4
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-three
+THIS-IS-LFS-4
3 a-three
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-two
+THIS-IS-LFS-three
2 a-two
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-single
+THIS-IS-LFS-two
1 a-single
diff --git a/a b/a
--- a/a
+++ b/a
@@ -1,1 +1,1 @@
-THIS-IS-LFS-0
+THIS-IS-LFS-single
0 a-0
diff --git a/a b/a
new file mode 100644
--- /dev/null
+++ b/a
@@ -0,0 +1,1 @@
+THIS-IS-LFS-0
$ cd ..
# Test isbinary
$ hg init repo10
$ cd repo10
$ cat >> .hg/hgrc << EOF
> [extensions]
> lfs=$TESTDIR/../hgext3rd/lfs/
> [lfs]
> threshold=1
> EOF
$ $PYTHON <<'EOF'
> def write(path, content):
> with open(path, 'wb') as f:
> f.write(content)
> write('a', b'\0\0')
> write('b', b'\1\n')
> write('c', b'\1\n\0')
> write('d', b'xx')
> EOF
$ hg add a b c d
$ hg diff --stat
a | Bin
b | 1 +
c | Bin
d | 1 +
4 files changed, 2 insertions(+), 0 deletions(-)
$ hg commit -m binarytest
$ cat > $TESTTMP/dumpbinary.py << EOF
> def reposetup(ui, repo):
> for n in 'abcd':
> ui.write(('%s: binary=%s\n') % (n, repo['.'][n].isbinary()))
> EOF
$ hg --config extensions.dumpbinary=$TESTTMP/dumpbinary.py id --trace
a: binary=True
b: binary=False
c: binary=True
d: binary=False
b55353847f02 tip
$ cd ..
# Test fctx.cmp fastpath - diff without LFS blobs
$ hg init repo11
$ cd repo11
$ cat >> .hg/hgrc <<EOF
> [lfs]
> threshold=1
> EOF
$ for i in 1 2 3; do
> cp ../repo10/a a
> if [ $i = 3 ]; then
> # make a content-only change
> chmod +x a
> i=2
> fi
> echo $i >> a
> hg commit -m $i -A a
> done
$ [ -d .hg/store/lfs/objects ]
$ cd ..
$ hg clone repo11 repo12 --noupdate
$ cd repo12
$ hg log --removed -p a -T '{desc}\n' --config diff.nobinary=1 --git
2
diff --git a/a b/a
old mode 100644
new mode 100755
2
diff --git a/a b/a
Binary file a has changed
1
diff --git a/a b/a
new file mode 100644
Binary file a has changed
$ [ -d .hg/store/lfs/objects ]
[1]
$ cd ..
# Verify the repos
$ cat > $TESTTMP/dumpflog.py << EOF
> # print raw revision sizes, flags, and hashes for certain files
> import hashlib
> from mercurial import revlog
> from mercurial.node import short
> def hash(rawtext):
> h = hashlib.sha512()
> h.update(rawtext)
> return h.hexdigest()[:4]
> def reposetup(ui, repo):
> # these 2 files are interesting
> for name in ['l', 's']:
> fl = repo.file(name)
> if len(fl) == 0:
> continue
> sizes = [revlog.revlog.rawsize(fl, i) for i in fl]
> texts = [fl.revision(i, raw=True) for i in fl]
> flags = [fl.flags(i) for i in fl]
> hashes = [hash(t) for t in texts]
> print(' %s: rawsizes=%r flags=%r hashes=%r'
> % (name, sizes, flags, hashes))
> EOF
$ for i in client client2 server repo3 repo4 repo5 repo6 repo7 repo8 repo9 \
> repo10; do
> echo 'repo:' $i
> hg --cwd $i verify --config extensions.dumpflog=$TESTTMP/dumpflog.py -q
> done
repo: client
repo: client2
repo: server
repo: repo3
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d']
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b']
repo: repo4
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d']
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b']
repo: repo5
l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d']
s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b']
repo: repo6
repo: repo7
repo: repo8
repo: repo9
repo: repo10
tests/test-lfs-pointer.py