This is an archive of the discontinued Mercurial Phabricator instance.

Differential D4943

remotefilelog: bump pack file formats to version 2
ClosedPublic

Authored by durin42 on Oct 11 2018, 8:21 AM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG10c10da14c5d: remotefilelog: bump pack file formats to version 2
Summary

Drop support for format 0 and 1 packs. Version 2 will be different in
that it'll use zlib compression instead of lz4, as the latter isn't
really suitable for core.

If I can make it work, I'll make compression engines pluggable, but
the logic for handling these files is spread out enough it's a little
tricky to get that right.

Test changes are only due to pack files being renamed. The contents of
the packs are the same.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Oct 11 2018, 8:21 AM
indygreg accepted this revision.Oct 11 2018, 9:43 AM
indygreg added a subscriber: indygreg.
indygreg added inline comments.
hgext/remotefilelog/historypack.py
17–19

Presumably we'll want to delete/rename these. Can be done as a follow-up.

This revision is now accepted and ready to land.Oct 11 2018, 9:43 AM
durin42 updated this revision to Diff 12181.Oct 16 2018, 3:18 PM
durin42 updated this revision to Diff 12406.Nov 5 2018, 2:45 PM
Closed by commit rHG10c10da14c5d: remotefilelog: bump pack file formats to version 2 (authored by durin42). · Explain WhyNov 5 2018, 2:53 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 12418

hgext/remotefilelog/basepack.py

newpacks.append(newpack) newpacks.append(newpack)
self.packs.add(newpack) self.packs.add(newpack)
return newpacks return newpacks
class versionmixin(object): class versionmixin(object):
# Mix-in for classes with multiple supported versions # Mix-in for classes with multiple supported versions
VERSION = None VERSION = None
SUPPORTED_VERSIONS = [0] SUPPORTED_VERSIONS = [2]
def _checkversion(self, version): def _checkversion(self, version):
if version in self.SUPPORTED_VERSIONS: if version in self.SUPPORTED_VERSIONS:
if self.VERSION is None: if self.VERSION is None:
# only affect this instance # only affect this instance
self.VERSION = version self.VERSION = version
elif self.VERSION != version: elif self.VERSION != version:
raise RuntimeError('inconsistent version: %s' % version) raise RuntimeError('inconsistent version: %s' % version)
else: else:
raise RuntimeError('unsupported version: %s' % version) raise RuntimeError('unsupported version: %s' % version)
class basepack(versionmixin): class basepack(versionmixin):
# The maximum amount we should read via mmap before remmaping so the old # The maximum amount we should read via mmap before remmaping so the old
# pages can be released (100MB) # pages can be released (100MB)
MAXPAGEDIN = 100 * 1024**2 MAXPAGEDIN = 100 * 1024**2
SUPPORTED_VERSIONS = [0] SUPPORTED_VERSIONS = [2]
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
self.packpath = path + self.PACKSUFFIX self.packpath = path + self.PACKSUFFIX
self.indexpath = path + self.INDEXSUFFIX self.indexpath = path + self.INDEXSUFFIX
self.indexsize = os.stat(self.indexpath).st_size self.indexsize = os.stat(self.indexpath).st_size
self.datasize = os.stat(self.packpath).st_size self.datasize = os.stat(self.packpath).st_size
for i in pycompat.xrange(0, params.fanoutcount): for i in pycompat.xrange(0, params.fanoutcount):
loc = i * 4 loc = i * 4
fanoutentry = struct.unpack('!I', rawfanout[loc:loc + 4])[0] fanoutentry = struct.unpack('!I', rawfanout[loc:loc + 4])[0]
fanouttable.append(fanoutentry) fanouttable.append(fanoutentry)
return fanouttable return fanouttable
@util.propertycache @util.propertycache
def _indexend(self): def _indexend(self):
if self.VERSION == 0:
return self.indexsize
else:
nodecount = struct.unpack_from('!Q', self._index, nodecount = struct.unpack_from('!Q', self._index,
self.params.indexstart - 8)[0] self.params.indexstart - 8)[0]
return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
def freememory(self): def freememory(self):
"""Unmap and remap the memory to free it up after known expensive """Unmap and remap the memory to free it up after known expensive
operations. Return True if self._data and self._index were reloaded. operations. Return True if self._data and self._index were reloaded.
""" """
if self._index: if self._index:
if self._pagedin < self.MAXPAGEDIN: if self._pagedin < self.MAXPAGEDIN:
return False return False
def __iter__(self): def __iter__(self):
raise NotImplementedError() raise NotImplementedError()
def iterentries(self): def iterentries(self):
raise NotImplementedError() raise NotImplementedError()
class mutablebasepack(versionmixin): class mutablebasepack(versionmixin):
def __init__(self, ui, packdir, version=0): def __init__(self, ui, packdir, version=2):
self._checkversion(version) self._checkversion(version)
# TODO(augie): make this configurable
self._compressor = 'GZ'
opener = vfsmod.vfs(packdir) opener = vfsmod.vfs(packdir)
opener.createmode = 0o444 opener.createmode = 0o444
self.opener = opener self.opener = opener
self.entries = {} self.entries = {}
shallowutil.mkstickygroupdir(ui, packdir) shallowutil.mkstickygroupdir(ui, packdir)
self.packfp, self.packpath = opener.mkstemp( self.packfp, self.packpath = opener.mkstemp(
rawentrieslength = struct.pack('!Q', len(self.entries)) rawentrieslength = struct.pack('!Q', len(self.entries))
# The index offset is the it's location in the file. So after the 2 byte # The index offset is the it's location in the file. So after the 2 byte
# header and the fanouttable. # header and the fanouttable.
rawindex = self.createindex(locations, 2 + len(rawfanouttable)) rawindex = self.createindex(locations, 2 + len(rawfanouttable))
self._writeheader(params) self._writeheader(params)
self.idxfp.write(rawfanouttable) self.idxfp.write(rawfanouttable)
if self.VERSION == 1:
self.idxfp.write(rawentrieslength) self.idxfp.write(rawentrieslength)
self.idxfp.write(rawindex) self.idxfp.write(rawindex)
self.idxfp.close() self.idxfp.close()
def createindex(self, nodelocations): def createindex(self, nodelocations):
raise NotImplementedError() raise NotImplementedError()
def _writeheader(self, indexparams): def _writeheader(self, indexparams):
# Index header # Index header
# The number of fanout table entries # The number of fanout table entries
self.fanoutcount = 2**(prefixsize * 8) self.fanoutcount = 2**(prefixsize * 8)
# The total bytes used by the fanout table # The total bytes used by the fanout table
self.fanoutsize = self.fanoutcount * 4 self.fanoutsize = self.fanoutcount * 4
self.indexstart = FANOUTSTART + self.fanoutsize self.indexstart = FANOUTSTART + self.fanoutsize
if version == 1:
# Skip the index length # Skip the index length
self.indexstart += 8 self.indexstart += 8

hgext/remotefilelog/datapack.py

from __future__ import absolute_import from __future__ import absolute_import
import struct import struct
from mercurial.node import hex, nullid from mercurial.node import hex, nullid
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
error,
pycompat, pycompat,
util, util,
) )
from . import ( from . import (
basepack, basepack,
constants, constants,
lz4wrapper, lz4wrapper,
shallowutil, shallowutil,
INDEXSUFFIX = INDEXSUFFIX INDEXSUFFIX = INDEXSUFFIX
PACKSUFFIX = PACKSUFFIX PACKSUFFIX = PACKSUFFIX
# Format is <node><delta offset><pack data offset><pack data size> # Format is <node><delta offset><pack data offset><pack data size>
# See the mutabledatapack doccomment for more details. # See the mutabledatapack doccomment for more details.
INDEXFORMAT = '!20siQQ' INDEXFORMAT = '!20siQQ'
INDEXENTRYLENGTH = 40 INDEXENTRYLENGTH = 40
SUPPORTED_VERSIONS = [0, 1] SUPPORTED_VERSIONS = [2]
def getmissing(self, keys): def getmissing(self, keys):
missing = [] missing = []
for name, node in keys: for name, node in keys:
value = self._find(node) value = self._find(node)
if not value: if not value:
missing.append((name, node)) missing.append((name, node))
return missing return missing
def get(self, name, node): def get(self, name, node):
raise RuntimeError("must use getdeltachain with datapack (%s:%s)" raise RuntimeError("must use getdeltachain with datapack (%s:%s)"
% (name, hex(node))) % (name, hex(node)))
def getmeta(self, name, node): def getmeta(self, name, node):
value = self._find(node) value = self._find(node)
if value is None: if value is None:
raise KeyError((name, hex(node))) raise KeyError((name, hex(node)))
# version 0 does not support metadata
if self.VERSION == 0:
return {}
node, deltabaseoffset, offset, size = value node, deltabaseoffset, offset, size = value
rawentry = self._data[offset:offset + size] rawentry = self._data[offset:offset + size]
# see docstring of mutabledatapack for the format # see docstring of mutabledatapack for the format
offset = 0 offset = 0
offset += struct.unpack_from('!H', rawentry, offset)[0] + 2 # filename offset += struct.unpack_from('!H', rawentry, offset)[0] + 2 # filename
offset += 40 # node, deltabase node offset += 40 # node, deltabase node
offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8 # delta offset += struct.unpack_from('!Q', rawentry, offset)[0] + 8 # delta
deltastart = deltabasestart + NODELENGTH deltastart = deltabasestart + NODELENGTH
rawdeltalen = rawentry[deltastart:deltastart + 8] rawdeltalen = rawentry[deltastart:deltastart + 8]
deltalen = struct.unpack('!Q', rawdeltalen)[0] deltalen = struct.unpack('!Q', rawdeltalen)[0]
delta = rawentry[deltastart + 8:deltastart + 8 + deltalen] delta = rawentry[deltastart + 8:deltastart + 8 + deltalen]
delta = lz4wrapper.lz4decompress(delta) delta = lz4wrapper.lz4decompress(delta)
if getmeta: if getmeta:
if self.VERSION == 0:
meta = {}
else:
metastart = deltastart + 8 + deltalen metastart = deltastart + 8 + deltalen
metalen = struct.unpack_from('!I', rawentry, metastart)[0] metalen = struct.unpack_from('!I', rawentry, metastart)[0]
rawmeta = rawentry[metastart + 4:metastart + 4 + metalen] rawmeta = rawentry[metastart + 4:metastart + 4 + metalen]
meta = shallowutil.parsepackmeta(rawmeta) meta = shallowutil.parsepackmeta(rawmeta)
return filename, node, deltabasenode, delta, meta return filename, node, deltabasenode, delta, meta
else: else:
return filename, node, deltabasenode, delta return filename, node, deltabasenode, delta
def add(self, name, node, data): def add(self, name, node, data):
raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node)) raise RuntimeError("cannot add to datapack (%s:%s)" % (name, node))
def _find(self, node): def _find(self, node):
# it has to be at least long enough for the lz4 header. # it has to be at least long enough for the lz4 header.
assert deltalen >= 4 assert deltalen >= 4
# python-lz4 stores the length of the uncompressed field as a # python-lz4 stores the length of the uncompressed field as a
# little-endian 32-bit integer at the start of the data. # little-endian 32-bit integer at the start of the data.
uncompressedlen = struct.unpack('<I', data[offset:offset + 4])[0] uncompressedlen = struct.unpack('<I', data[offset:offset + 4])[0]
offset += deltalen offset += deltalen
if self.VERSION == 1:
# <4 byte len> + <metadata-list> # <4 byte len> + <metadata-list>
metalen = struct.unpack_from('!I', data, offset)[0] metalen = struct.unpack_from('!I', data, offset)[0]
offset += 4 + metalen offset += 4 + metalen
yield (filename, node, deltabase, uncompressedlen) yield (filename, node, deltabase, uncompressedlen)
# If we've read a lot of data from the mmap, free some memory. # If we've read a lot of data from the mmap, free some memory.
self._pagedin += offset - oldoffset self._pagedin += offset - oldoffset
if self.freememory(): if self.freememory():
data = self._data data = self._data
INDEXSUFFIX = INDEXSUFFIX INDEXSUFFIX = INDEXSUFFIX
PACKSUFFIX = PACKSUFFIX PACKSUFFIX = PACKSUFFIX
# v[01] index format: <node><delta offset><pack data offset><pack data size> # v[01] index format: <node><delta offset><pack data offset><pack data size>
INDEXFORMAT = datapack.INDEXFORMAT INDEXFORMAT = datapack.INDEXFORMAT
INDEXENTRYLENGTH = datapack.INDEXENTRYLENGTH INDEXENTRYLENGTH = datapack.INDEXENTRYLENGTH
# v1 has metadata support # v1 has metadata support
SUPPORTED_VERSIONS = [0, 1] SUPPORTED_VERSIONS = [2]
def add(self, name, node, deltabasenode, delta, metadata=None): def add(self, name, node, deltabasenode, delta, metadata=None):
# metadata is a dict, ex. {METAKEYFLAG: flag} # metadata is a dict, ex. {METAKEYFLAG: flag}
if len(name) > 2**16: if len(name) > 2**16:
raise RuntimeError(_("name too long %s") % name) raise RuntimeError(_("name too long %s") % name)
if len(node) != 20: if len(node) != 20:
raise RuntimeError(_("node should be 20 bytes %s") % node) raise RuntimeError(_("node should be 20 bytes %s") % node)
if node in self.entries: if node in self.entries:
# The revision has already been added # The revision has already been added
return return
# TODO: allow configurable compression # TODO: allow configurable compression
delta = lz4wrapper.lz4compress(delta) delta = lz4wrapper.lz4compress(delta)
rawdata = ''.join(( rawdata = ''.join((
struct.pack('!H', len(name)), # unsigned 2 byte int struct.pack('!H', len(name)), # unsigned 2 byte int
name, name,
node, node,
deltabasenode, deltabasenode,
struct.pack('!Q', len(delta)), # unsigned 8 byte int struct.pack('!Q', len(delta)), # unsigned 8 byte int
delta, delta,
)) ))
if self.VERSION == 1:
# v1 support metadata # v1 support metadata
rawmeta = shallowutil.buildpackmeta(metadata) rawmeta = shallowutil.buildpackmeta(metadata)
rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte rawdata += struct.pack('!I', len(rawmeta)) # unsigned 4 byte
rawdata += rawmeta rawdata += rawmeta
else:
# v0 cannot store metadata, raise if metadata contains flag
if metadata and metadata.get(constants.METAKEYFLAG, 0) != 0:
raise error.ProgrammingError('v0 pack cannot store flags')
offset = self.packfp.tell() offset = self.packfp.tell()
size = len(rawdata) size = len(rawdata)
self.entries[node] = (deltabasenode, offset, size) self.entries[node] = (deltabasenode, offset, size)
self.writeraw(rawdata) self.writeraw(rawdata)

hgext/remotefilelog/historypack.py

from __future__ import absolute_import from __future__ import absolute_import
import hashlib import hashlib
import struct import struct
from mercurial.node import hex, nullid from mercurial.node import hex, nullid
from mercurial import ( from mercurial import (
pycompat, pycompat,
util, util,
) )
from . import ( from . import (
basepack, basepack,
constants, constants,
shallowutil, shallowutil,
) )
# (filename hash, offset, size) # (filename hash, offset, size)
INDEXFORMAT0 = '!20sQQ' INDEXFORMAT2 = '!20sQQII'
INDEXENTRYLENGTH0 = struct.calcsize(INDEXFORMAT0) INDEXENTRYLENGTH2 = struct.calcsize(INDEXFORMAT2)
indygregUnsubmitted
Not Done

Presumably we'll want to delete/rename these. Can be done as a follow-up.

indygreg: Presumably we'll want to delete/rename these. Can be done as a follow-up.
INDEXFORMAT1 = '!20sQQII'
INDEXENTRYLENGTH1 = struct.calcsize(INDEXFORMAT1)
NODELENGTH = 20 NODELENGTH = 20
NODEINDEXFORMAT = '!20sQ' NODEINDEXFORMAT = '!20sQ'
NODEINDEXENTRYLENGTH = struct.calcsize(NODEINDEXFORMAT) NODEINDEXENTRYLENGTH = struct.calcsize(NODEINDEXFORMAT)
# (node, p1, p2, linknode) # (node, p1, p2, linknode)
PACKFORMAT = "!20s20s20s20sH" PACKFORMAT = "!20s20s20s20sH"
PACKENTRYLENGTH = 82 PACKENTRYLENGTH = 82
def add(self, filename, node, p1, p2, linknode, copyfrom): def add(self, filename, node, p1, p2, linknode, copyfrom):
raise RuntimeError("cannot add to historypackstore (%s:%s)" raise RuntimeError("cannot add to historypackstore (%s:%s)"
% (filename, hex(node))) % (filename, hex(node)))
class historypack(basepack.basepack): class historypack(basepack.basepack):
INDEXSUFFIX = INDEXSUFFIX INDEXSUFFIX = INDEXSUFFIX
PACKSUFFIX = PACKSUFFIX PACKSUFFIX = PACKSUFFIX
SUPPORTED_VERSIONS = [0, 1] SUPPORTED_VERSIONS = [2]
def __init__(self, path): def __init__(self, path):
super(historypack, self).__init__(path) super(historypack, self).__init__(path)
self.INDEXFORMAT = INDEXFORMAT2
if self.VERSION == 0: self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
self.INDEXFORMAT = INDEXFORMAT0
self.INDEXENTRYLENGTH = INDEXENTRYLENGTH0
else:
self.INDEXFORMAT = INDEXFORMAT1
self.INDEXENTRYLENGTH = INDEXENTRYLENGTH1
def getmissing(self, keys): def getmissing(self, keys):
missing = [] missing = []
for name, node in keys: for name, node in keys:
try: try:
self._findnode(name, node) self._findnode(name, node)
except KeyError: except KeyError:
missing.append((name, node)) missing.append((name, node))
else: else:
end = indexend end = indexend
entry = self._bisect(namehash, start, end, self.INDEXENTRYLENGTH) entry = self._bisect(namehash, start, end, self.INDEXENTRYLENGTH)
if not entry: if not entry:
raise KeyError(name) raise KeyError(name)
rawentry = struct.unpack(self.INDEXFORMAT, entry) rawentry = struct.unpack(self.INDEXFORMAT, entry)
if self.VERSION == 0:
x, offset, size = rawentry
nodeindexoffset = None
nodeindexsize = None
else:
x, offset, size, nodeindexoffset, nodeindexsize = rawentry x, offset, size, nodeindexoffset, nodeindexsize = rawentry
rawnamelen = self._index[nodeindexoffset:nodeindexoffset + rawnamelen = self._index[nodeindexoffset:nodeindexoffset +
constants.FILENAMESIZE] constants.FILENAMESIZE]
actualnamelen = struct.unpack('!H', rawnamelen)[0] actualnamelen = struct.unpack('!H', rawnamelen)[0]
nodeindexoffset += constants.FILENAMESIZE nodeindexoffset += constants.FILENAMESIZE
actualname = self._index[nodeindexoffset:nodeindexoffset + actualname = self._index[nodeindexoffset:nodeindexoffset +
actualnamelen] actualnamelen]
if actualname != name: if actualname != name:
raise KeyError("found file name %s when looking for %s" % raise KeyError("found file name %s when looking for %s" %
(actualname, name)) (actualname, name))
nodeindexoffset += actualnamelen nodeindexoffset += actualnamelen
filenamelength = struct.unpack('!H', self._data[offset:offset + filenamelength = struct.unpack('!H', self._data[offset:offset +
constants.FILENAMESIZE])[0] constants.FILENAMESIZE])[0]
offset += constants.FILENAMESIZE offset += constants.FILENAMESIZE
actualname = self._data[offset:offset + filenamelength] actualname = self._data[offset:offset + filenamelength]
offset += filenamelength offset += filenamelength
nodeindexentry = <node: 20 byte> [1] nodeindexentry = <node: 20 byte> [1]
<pack file node offset: 8 byte unsigned int> [1] <pack file node offset: 8 byte unsigned int> [1]
[1]: new in version 1. [1]: new in version 1.
""" """
INDEXSUFFIX = INDEXSUFFIX INDEXSUFFIX = INDEXSUFFIX
PACKSUFFIX = PACKSUFFIX PACKSUFFIX = PACKSUFFIX
SUPPORTED_VERSIONS = [0, 1] SUPPORTED_VERSIONS = [2]
def __init__(self, ui, packpath, version=0):
# internal config: remotefilelog.historypackv1
if version == 0 and ui.configbool('remotefilelog', 'historypackv1'):
version = 1
def __init__(self, ui, packpath, version=2):
super(mutablehistorypack, self).__init__(ui, packpath, version=version) super(mutablehistorypack, self).__init__(ui, packpath, version=version)
self.files = {} self.files = {}
self.entrylocations = {} self.entrylocations = {}
self.fileentries = {} self.fileentries = {}
if version == 0: self.INDEXFORMAT = INDEXFORMAT2
self.INDEXFORMAT = INDEXFORMAT0 self.INDEXENTRYLENGTH = INDEXENTRYLENGTH2
self.INDEXENTRYLENGTH = INDEXENTRYLENGTH0
else:
self.INDEXFORMAT = INDEXFORMAT1
self.INDEXENTRYLENGTH = INDEXENTRYLENGTH1
self.NODEINDEXFORMAT = NODEINDEXFORMAT self.NODEINDEXFORMAT = NODEINDEXFORMAT
self.NODEINDEXENTRYLENGTH = NODEINDEXENTRYLENGTH self.NODEINDEXENTRYLENGTH = NODEINDEXENTRYLENGTH
def add(self, filename, node, p1, p2, linknode, copyfrom): def add(self, filename, node, p1, p2, linknode, copyfrom):
copyfrom = copyfrom or '' copyfrom = copyfrom or ''
copyfromlen = struct.pack('!H', len(copyfrom)) copyfromlen = struct.pack('!H', len(copyfrom))
self.fileentries.setdefault(filename, []).append((node, p1, p2, self.fileentries.setdefault(filename, []).append((node, p1, p2,
return super(mutablehistorypack, self).close(ledger=ledger) return super(mutablehistorypack, self).close(ledger=ledger)
def createindex(self, nodelocations, indexoffset): def createindex(self, nodelocations, indexoffset):
fileindexformat = self.INDEXFORMAT fileindexformat = self.INDEXFORMAT
fileindexlength = self.INDEXENTRYLENGTH fileindexlength = self.INDEXENTRYLENGTH
nodeindexformat = self.NODEINDEXFORMAT nodeindexformat = self.NODEINDEXFORMAT
nodeindexlength = self.NODEINDEXENTRYLENGTH nodeindexlength = self.NODEINDEXENTRYLENGTH
version = self.VERSION
files = ((hashlib.sha1(filename).digest(), filename, offset, size) files = ((hashlib.sha1(filename).digest(), filename, offset, size)
for filename, (offset, size) in self.files.iteritems()) for filename, (offset, size) in self.files.iteritems())
files = sorted(files) files = sorted(files)
# node index is after file index size, file index, and node index size # node index is after file index size, file index, and node index size
indexlensize = struct.calcsize('!Q') indexlensize = struct.calcsize('!Q')
nodeindexoffset = (indexoffset + indexlensize + nodeindexoffset = (indexoffset + indexlensize +
(len(files) * fileindexlength) + indexlensize) (len(files) * fileindexlength) + indexlensize)
fileindexentries = [] fileindexentries = []
nodeindexentries = [] nodeindexentries = []
nodecount = 0 nodecount = 0
for namehash, filename, offset, size in files: for namehash, filename, offset, size in files:
# File section index # File section index
if version == 0:
rawentry = struct.pack(fileindexformat, namehash, offset, size)
else:
nodelocations = self.entrylocations[filename] nodelocations = self.entrylocations[filename]
nodeindexsize = len(nodelocations) * nodeindexlength nodeindexsize = len(nodelocations) * nodeindexlength
rawentry = struct.pack(fileindexformat, namehash, offset, size, rawentry = struct.pack(fileindexformat, namehash, offset, size,
nodeindexoffset, nodeindexsize) nodeindexoffset, nodeindexsize)
# Node index # Node index
nodeindexentries.append(struct.pack(constants.FILENAMESTRUCT, nodeindexentries.append(struct.pack(constants.FILENAMESTRUCT,
len(filename)) + filename) len(filename)) + filename)
nodeindexoffset += constants.FILENAMESIZE + len(filename) nodeindexoffset += constants.FILENAMESIZE + len(filename)
for node, location in sorted(nodelocations.iteritems()): for node, location in sorted(nodelocations.iteritems()):
nodeindexentries.append(struct.pack(nodeindexformat, node, nodeindexentries.append(struct.pack(nodeindexformat, node,
location)) location))
nodecount += 1 nodecount += 1
nodeindexoffset += len(nodelocations) * nodeindexlength nodeindexoffset += len(nodelocations) * nodeindexlength
fileindexentries.append(rawentry) fileindexentries.append(rawentry)
nodecountraw = '' nodecountraw = ''
if version == 1:
nodecountraw = struct.pack('!Q', nodecount) nodecountraw = struct.pack('!Q', nodecount)
return (''.join(fileindexentries) + nodecountraw + return (''.join(fileindexentries) + nodecountraw +
''.join(nodeindexentries)) ''.join(nodeindexentries))

hgext/remotefilelog/repack.py

return filetime[0] < limit return filetime[0] < limit
garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack') garbagecollect = repo.ui.configbool('remotefilelog', 'gcrepack')
if not fullhistory: if not fullhistory:
fullhistory = history fullhistory = history
packer = repacker(repo, data, history, fullhistory, category, packer = repacker(repo, data, history, fullhistory, category,
gc=garbagecollect, isold=isold, options=options) gc=garbagecollect, isold=isold, options=options)
# internal config: remotefilelog.datapackversion with datapack.mutabledatapack(repo.ui, packpath, version=2) as dpack:
dv = repo.ui.configint('remotefilelog', 'datapackversion', 0)
with datapack.mutabledatapack(repo.ui, packpath, version=dv) as dpack:
with historypack.mutablehistorypack(repo.ui, packpath) as hpack: with historypack.mutablehistorypack(repo.ui, packpath) as hpack:
try: try:
packer.run(dpack, hpack) packer.run(dpack, hpack)
except error.LockHeld: except error.LockHeld:
raise RepackAlreadyRunning(_("skipping repack - another repack " raise RepackAlreadyRunning(_("skipping repack - another repack "
"is already running")) "is already running"))
def keepset(repo, keyfn, lastkeepkeys=None): def keepset(repo, keyfn, lastkeepkeys=None):

tests/test-remotefilelog-bgprefetch.t

new changesets 6b4b6f66ef8c new changesets 6b4b6f66ef8c
(run 'hg update' to get a working copy) (run 'hg update' to get a working copy)
prefetching file contents prefetching file contents
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonprefetch >/dev/null 2>%1 $ hg debugwaitonprefetch >/dev/null 2>%1
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/94d53eef9e622533aec1fc6d8053cb086e785d21.histidx $TESTTMP/hgcache/master/packs/3616094d229ed39e2593f79c772676d4ec00253a.dataidx
$TESTTMP/hgcache/master/packs/94d53eef9e622533aec1fc6d8053cb086e785d21.histpack $TESTTMP/hgcache/master/packs/3616094d229ed39e2593f79c772676d4ec00253a.datapack
$TESTTMP/hgcache/master/packs/f3644bc7773e8289deda7f765138120c838f4e6e.dataidx $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx
$TESTTMP/hgcache/master/packs/f3644bc7773e8289deda7f765138120c838f4e6e.datapack $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# background prefetch with repack on update when wcprevset configured # background prefetch with repack on update when wcprevset configured
$ clearcache $ clearcache
$ hg up -r 0 $ hg up -r 0
2 files updated, 0 files merged, 0 files removed, 0 files unresolved 2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg up -r 0 $ hg up -r 0
1 files updated, 0 files merged, 1 files removed, 0 files unresolved 1 files updated, 0 files merged, 1 files removed, 0 files unresolved
* files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob) * files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob)
$ sleep 1 $ sleep 1
$ hg debugwaitonprefetch >/dev/null 2>%1 $ hg debugwaitonprefetch >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that file 'w' was prefetched - it was not part of the update operation and therefore # Ensure that file 'w' was prefetched - it was not part of the update operation and therefore
# could only be downloaded by the background prefetch # could only be downloaded by the background prefetch
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0: $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
ef95c5376f34 000000000000 3 3 ef95c5376f34 000000000000 3 3
$ hg commit -qAm b $ hg commit -qAm b
* files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob)
$ hg bookmark temporary $ hg bookmark temporary
$ sleep 1 $ sleep 1
$ hg debugwaitonprefetch >/dev/null 2>%1 $ hg debugwaitonprefetch >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore # Ensure that file 'w' was prefetched - it was not part of the commit operation and therefore
# could only be downloaded by the background prefetch # could only be downloaded by the background prefetch
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0: $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
ef95c5376f34 000000000000 3 3 ef95c5376f34 000000000000 3 3
$ hg debugwaitonprefetch >/dev/null 2>%1 $ hg debugwaitonprefetch >/dev/null 2>%1
$ sleep 1 $ sleep 1
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
# Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore
# could only be downloaded by the background prefetch # could only be downloaded by the background prefetch
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0: $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
ef95c5376f34 000000000000 3 3 ef95c5376f34 000000000000 3 3
got lock after * seconds (glob) (?) got lock after * seconds (glob) (?)
(running background incremental repack) (running background incremental repack)
* files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?) * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that files were prefetched # Ensure that files were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0: $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
ef95c5376f34 000000000000 3 3 ef95c5376f34 000000000000 3 3
waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?) waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
got lock after * seconds (glob) (?) got lock after * seconds (glob) (?)
(running background incremental repack) (running background incremental repack)
* files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?) * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histidx $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
$TESTTMP/hgcache/master/packs/27c52c105a1ddf8c75143a6b279b04c24b1f4bee.histpack $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.dataidx $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.dataidx
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0.datapack $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Ensure that files were prefetched # Ensure that files were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/8299d5a1030f073f4adbb3b6bd2ad3bdcc276df0: $TESTTMP/hgcache/master/packs/ac19aff076286bebe3ff108c96c6445a0fe27c46:
w: w:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
bb6ccd5dceaa 000000000000 2 2 bb6ccd5dceaa 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
ef95c5376f34 000000000000 3 3 ef95c5376f34 000000000000 3 3

tests/test-remotefilelog-datapack.py

return tempdir return tempdir
def getHash(self, content): def getHash(self, content):
return hashlib.sha1(content).digest() return hashlib.sha1(content).digest()
def getFakeHash(self): def getFakeHash(self):
return ''.join(chr(random.randint(0, 255)) for _ in range(20)) return ''.join(chr(random.randint(0, 255)) for _ in range(20))
def createPack(self, revisions=None, packdir=None, version=0): def createPack(self, revisions=None, packdir=None):
if revisions is None: if revisions is None:
revisions = [("filename", self.getFakeHash(), nullid, "content")] revisions = [("filename", self.getFakeHash(), nullid, "content")]
if packdir is None: if packdir is None:
packdir = self.makeTempDir() packdir = self.makeTempDir()
packer = datapack.mutabledatapack( packer = datapack.mutabledatapack(uimod.ui(), packdir, version=2)
uimod.ui(), packdir, version=version)
for args in revisions: for args in revisions:
filename, node, base, content = args[0:4] filename, node, base, content = args[0:4]
# meta is optional # meta is optional
meta = None meta = None
if len(args) > 4: if len(args) > 4:
meta = args[4] meta = args[4]
packer.add(filename, node, base, content, metadata=meta) packer.add(filename, node, base, content, metadata=meta)
filename = '%s.txt' % i filename = '%s.txt' % i
content = 'put-something-here \n' * i content = 'put-something-here \n' * i
node = self.getHash(content) node = self.getHash(content)
meta = {constants.METAKEYFLAG: i ** 4, meta = {constants.METAKEYFLAG: i ** 4,
constants.METAKEYSIZE: len(content), constants.METAKEYSIZE: len(content),
'Z': 'random_string', 'Z': 'random_string',
'_': '\0' * i} '_': '\0' * i}
revisions.append((filename, node, nullid, content, meta)) revisions.append((filename, node, nullid, content, meta))
pack = self.createPack(revisions, version=1) pack = self.createPack(revisions)
for name, node, x, content, origmeta in revisions: for name, node, x, content, origmeta in revisions:
parsedmeta = pack.getmeta(name, node) parsedmeta = pack.getmeta(name, node)
# flag == 0 should be optimized out # flag == 0 should be optimized out
if origmeta[constants.METAKEYFLAG] == 0: if origmeta[constants.METAKEYFLAG] == 0:
del origmeta[constants.METAKEYFLAG] del origmeta[constants.METAKEYFLAG]
self.assertEquals(parsedmeta, origmeta) self.assertEquals(parsedmeta, origmeta)
def testPackMetadataThrows(self):
filename = '1'
content = '2'
node = self.getHash(content)
meta = {constants.METAKEYFLAG: 3}
revisions = [(filename, node, nullid, content, meta)]
try:
self.createPack(revisions, version=0)
self.assertTrue(False, "should throw if metadata is not supported")
except RuntimeError:
pass
def testGetMissing(self): def testGetMissing(self):
"""Test the getmissing() api. """Test the getmissing() api.
""" """
revisions = [] revisions = []
filename = "foo" filename = "foo"
lastnode = nullid lastnode = nullid
for i in range(10): for i in range(10):
content = "abcdef%s" % i content = "abcdef%s" % i

tests/test-remotefilelog-gc.t

$TESTTMP/hgcache/repos (glob) $TESTTMP/hgcache/repos (glob)
# Test that if garbage collection on repack and repack on hg gc flags are set then incremental repack with garbage collector is run # Test that if garbage collection on repack and repack on hg gc flags are set then incremental repack with garbage collector is run
$ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True $ hg gc --config remotefilelog.gcrepack=True --config remotefilelog.repackonhggc=True
# Ensure that loose files are repacked # Ensure that loose files are repacked
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/8d3499c65d926e4f107cf03c6b0df833222025b4.histidx $TESTTMP/hgcache/master/packs/173691d550fabb9d33db8da192f1c9bc62dd11a4.dataidx
$TESTTMP/hgcache/master/packs/8d3499c65d926e4f107cf03c6b0df833222025b4.histpack $TESTTMP/hgcache/master/packs/173691d550fabb9d33db8da192f1c9bc62dd11a4.datapack
$TESTTMP/hgcache/master/packs/9c7046f8cad0417c39aa7c03ce13e0ba991306c2.dataidx $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histidx
$TESTTMP/hgcache/master/packs/9c7046f8cad0417c39aa7c03ce13e0ba991306c2.datapack $TESTTMP/hgcache/master/packs/837b83c1ef6485a336eb4421ac5973c0ec130fbb.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that warning is displayed when there are no valid repos in repofile # Test that warning is displayed when there are no valid repos in repofile
$ cp $CACHEDIR/repos $CACHEDIR/repos.bak $ cp $CACHEDIR/repos $CACHEDIR/repos.bak
$ echo " " > $CACHEDIR/repos $ echo " " > $CACHEDIR/repos
$ hg gc $ hg gc

tests/test-remotefilelog-gcrepack.t

$ hg prefetch $ hg prefetch
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
$ hg repack $ hg repack
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR | sort | grep ".datapack\|.histpack" $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
$TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
# Ensure that all file versions were prefetched # Ensure that all file versions were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4: $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1406e7411862 000000000000 2 2 1406e7411862 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
y: y:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
50dbc4572b8e 000000000000 3 3 50dbc4572b8e 000000000000 3 3
> nodettl=86400 > nodettl=86400
> EOF > EOF
$ hg repack $ hg repack
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR | sort | grep ".datapack\|.histpack" $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
$TESTTMP/hgcache/master/packs/05baa499c6b07f2bf0ea3d2c8151da1cb86f5e33.datapack $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
$TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack $TESTTMP/hgcache/master/packs/b868298fad9bf477b4e9d9455226c440b0135fe6.datapack
# Ensure that file 'x' was garbage collected. It should be GCed because it is not in the keepset # Ensure that file 'x' was garbage collected. It should be GCed because it is not in the keepset
# and is old (commit date is 0.0 in tests). Ensure that file 'y' is present as it is in the keepset. # and is old (commit date is 0.0 in tests). Ensure that file 'y' is present as it is in the keepset.
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/05baa499c6b07f2bf0ea3d2c8151da1cb86f5e33: $TESTTMP/hgcache/master/packs/b868298fad9bf477b4e9d9455226c440b0135fe6:
y: y:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
50dbc4572b8e 000000000000 3 3 50dbc4572b8e 000000000000 3 3
Total: 3 3 (0.0% bigger) Total: 3 3 (0.0% bigger)
# Prefetch all data again and repack for later garbage collection # Prefetch all data again and repack for later garbage collection
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [remotefilelog] > [remotefilelog]
> bgprefetchrevs=all() > bgprefetchrevs=all()
> EOF > EOF
$ hg prefetch $ hg prefetch
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
$ hg repack $ hg repack
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR | sort | grep ".datapack\|.histpack" $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
$TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
# Ensure that all file versions were prefetched # Ensure that all file versions were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4: $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1406e7411862 000000000000 2 2 1406e7411862 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
y: y:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
50dbc4572b8e 000000000000 3 3 50dbc4572b8e 000000000000 3 3
> nodettl=$(($(date +%s) + 100)) > nodettl=$(($(date +%s) + 100))
> EOF > EOF
$ hg repack $ hg repack
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>%1 $ hg debugwaitonrepack >/dev/null 2>%1
$ find $CACHEDIR | sort | grep ".datapack\|.histpack" $ find $CACHEDIR | sort | grep ".datapack\|.histpack"
$TESTTMP/hgcache/master/packs/9a2ea858fe2967db9b6ea4c0ca238881cae9d6eb.histpack $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4.datapack $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1.datapack
# Ensure that all file versions were prefetched # Ensure that all file versions were prefetched
$ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1` $ hg debugdatapack `ls -ct $TESTTMP/hgcache/master/packs/*.datapack | head -n 1`
$TESTTMP/hgcache/master/packs/f7a942a6e4673d2c7b697fdd926ca2d153831ca4: $TESTTMP/hgcache/master/packs/be1bfed71a51645d8c3b9dc73e234e3a8ff06ac1:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1406e7411862 000000000000 2 2 1406e7411862 000000000000 2 2
Total: 2 2 (0.0% bigger) Total: 2 2 (0.0% bigger)
y: y:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
50dbc4572b8e 000000000000 3 3 50dbc4572b8e 000000000000 3 3
076f5e2225b3 50dbc4572b8e 14 2 076f5e2225b3 50dbc4572b8e 14 2
Total: 17 5 (240.0% bigger) Total: 17 5 (240.0% bigger)

tests/test-remotefilelog-histpack.py

node, p1node, p2node, and linknode. node, p1node, p2node, and linknode.
""" """
if revisions is None: if revisions is None:
revisions = [("filename", self.getFakeHash(), nullid, nullid, revisions = [("filename", self.getFakeHash(), nullid, nullid,
self.getFakeHash(), None)] self.getFakeHash(), None)]
packdir = self.makeTempDir() packdir = self.makeTempDir()
packer = historypack.mutablehistorypack(uimod.ui(), packdir, packer = historypack.mutablehistorypack(uimod.ui(), packdir,
version=1) version=2)
for filename, node, p1, p2, linknode, copyfrom in revisions: for filename, node, p1, p2, linknode, copyfrom in revisions:
packer.add(filename, node, p1, p2, linknode, copyfrom) packer.add(filename, node, p1, p2, linknode, copyfrom)
path = packer.close() path = packer.close()
return historypack.historypack(path) return historypack.historypack(path)
def testAddSingle(self): def testAddSingle(self):

tests/test-remotefilelog-repack-fast.t

$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack $ hg repack
$ find $CACHEDIR | sort $ find $CACHEDIR | sort
$TESTTMP/hgcache $TESTTMP/hgcache
$TESTTMP/hgcache/master $TESTTMP/hgcache/master
$TESTTMP/hgcache/master/packs $TESTTMP/hgcache/master/packs
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that the packs are readonly # Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs $ ls_l $CACHEDIR/master/packs
-r--r--r-- 1145 276d308429d0303762befa376788300f0310f90e.histidx -r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
-r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack -r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
-r--r--r-- 1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx -r--r--r-- 1074 add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
-r--r--r-- 69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack -r--r--r-- 69 add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
-rw-r--r-- 0 repacklock -rw-r--r-- 0 repacklock
# Test that the data in the new packs is accessible # Test that the data in the new packs is accessible
$ hg cat -r . x $ hg cat -r . x
x x
x x
# Test that adding new data and repacking it results in the loose data and the # Test that adding new data and repacking it results in the loose data and the
# old packs being combined. # old packs being combined.
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x3 $ hg commit -m x3
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --traceback $ hg repack --traceback
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Verify all the file data is still available # Verify all the file data is still available
$ hg cat -r . x $ hg cat -r . x
x x
x x
x x
$ hg cat -r '.^' x $ hg cat -r '.^' x
x x
x x
# Test that repacking again without new data does not delete the pack files # Test that repacking again without new data does not delete the pack files
# and did not change the pack names # and did not change the pack names
$ hg repack $ hg repack
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Run two repacks at once # Run two repacks at once
$ hg repack --config "hooks.prerepack=sleep 3" & $ hg repack --config "hooks.prerepack=sleep 3" &
$ sleep 1 $ sleep 1
$ hg repack $ hg repack
skipping repack - another repack is already running skipping repack - another repack is already running
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
# Run repack in the background # Run repack in the background
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x4 $ hg commit -m x4
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --background $ hg repack --background
(running background repack) (running background repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histidx $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.dataidx
$TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histpack $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.dataidx $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test debug commands # Test debug commands
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a d4a3ed9310e5 12 4 aee31534993a d4a3ed9310e5 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
$ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8
d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6
aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4 aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216 $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x x
Node Delta Base Delta SHA1 Delta Length Node Delta Base Delta SHA1 Delta Length
d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 77029ab56e83ea2115dd53ff87483682abe5d7ca 12 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 77029ab56e83ea2115dd53ff87483682abe5d7ca 12
Node Delta Base Delta SHA1 Delta Length Node Delta Base Delta SHA1 Delta Length
1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5 8 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5 8
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
$ hg revert x $ hg revert x
$ hg commit -m 'add y back without metadata' $ hg commit -m 'add y back without metadata'
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ ls $TESTTMP/hgcache/master/packs $ ls $TESTTMP/hgcache/master/packs
e8fdf7ae22b772dcc291f905b9c6e5f381d28739.dataidx 308a7aba9c54a0b71ae5adbbccd00c0aff20876e.dataidx
e8fdf7ae22b772dcc291f905b9c6e5f381d28739.datapack 308a7aba9c54a0b71ae5adbbccd00c0aff20876e.datapack
ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histidx bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histpack bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
repacklock repacklock
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
x x
Node P1 Node P2 Node Link Node Copy From Node P1 Node P2 Node Link Node Copy From
cd410a44d584 577959738234 000000000000 609547eda446 y cd410a44d584 577959738234 000000000000 609547eda446 y
1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7 1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7
d4a3ed9310e5 aee31534993a 000000000000 421535db10b6 d4a3ed9310e5 aee31534993a 000000000000 421535db10b6
$ rm -rf $TESTTMP/hgcache/master/packs $ rm -rf $TESTTMP/hgcache/master/packs
$ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
# Test repacking datapack without history # Test repacking datapack without history
$ rm -rf $CACHEDIR/master/packs/*hist* $ rm -rf $CACHEDIR/master/packs/*hist*
$ hg repack $ hg repack
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/a8d86ff8e1a11a77a85f5fea567f56a757583eda: $TESTTMP/hgcache/master/packs/ba4649b56263282b0699f9a6e7e34a4a2bac1638:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a d4a3ed9310e5 12 4 aee31534993a d4a3ed9310e5 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
y: y:
$ hg prefetch -r 0 $ hg prefetch -r 0
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
[1] [1]
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
[1] [1]
$ hg repack --incremental $ hg repack --incremental
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
$ hg prefetch -r 1 $ hg prefetch -r 1
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 2 $ hg prefetch -r 2
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 3 $ hg prefetch -r 3
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
$ hg repack --incremental $ hg repack --incremental
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack -r--r--r-- 225 06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack -r--r--r-- 336 604552d403a1381749faf656feca0ca265a6d52c.histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
1 gen3 pack, 1 gen0 pack - does nothing 1 gen3 pack, 1 gen0 pack - does nothing
$ hg repack --incremental $ hg repack --incremental
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 225 8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack -r--r--r-- 225 06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack -r--r--r-- 336 604552d403a1381749faf656feca0ca265a6d52c.histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
Pull should run background repack Pull should run background repack
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [remotefilelog] > [remotefilelog]
> backgroundrepack=True > backgroundrepack=True
> EOF > EOF
$ clearcache $ clearcache
$ hg prefetch -r 0 $ hg prefetch -r 0
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 1 $ hg prefetch -r 1
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 2 $ hg prefetch -r 2
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 3 $ hg prefetch -r 3
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg pull $ hg pull
pulling from ssh://user@dummy/master pulling from ssh://user@dummy/master
searching for changes searching for changes
no changes found no changes found
(running background incremental repack) (running background incremental repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 301 09b8bf49256b3fc2175977ba97d6402e91a9a604.datapack -r--r--r-- 301 671913bebdb7b95aae52a546662753eac7606e40.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack -r--r--r-- 336 604552d403a1381749faf656feca0ca265a6d52c.histpack
Test environment variable resolution Test environment variable resolution
$ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH' $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $TESTTMP/envcache | sort $ find $TESTTMP/envcache | sort
$TESTTMP/envcache $TESTTMP/envcache
$TESTTMP/envcache/master $TESTTMP/envcache/master
$TESTTMP/envcache/master/95 $TESTTMP/envcache/master/95

tests/test-remotefilelog-repack.t

$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack $ hg repack
$ find $CACHEDIR | sort $ find $CACHEDIR | sort
$TESTTMP/hgcache $TESTTMP/hgcache
$TESTTMP/hgcache/master $TESTTMP/hgcache/master
$TESTTMP/hgcache/master/packs $TESTTMP/hgcache/master/packs
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test that the packs are readonly # Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs $ ls_l $CACHEDIR/master/packs
-r--r--r-- 1145 276d308429d0303762befa376788300f0310f90e.histidx -r--r--r-- 1145 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
-r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack -r--r--r-- 172 1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
-r--r--r-- 1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx -r--r--r-- 1074 add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
-r--r--r-- 69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack -r--r--r-- 69 add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
-rw-r--r-- 0 repacklock -rw-r--r-- 0 repacklock
# Test that the data in the new packs is accessible # Test that the data in the new packs is accessible
$ hg cat -r . x $ hg cat -r . x
x x
x x
# Test that adding new data and repacking it results in the loose data and the # Test that adding new data and repacking it results in the loose data and the
# old packs being combined. # old packs being combined.
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x3 $ hg commit -m x3
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# First assert that with --packsonly, the loose object will be ignored: # First assert that with --packsonly, the loose object will be ignored:
$ hg repack --packsonly $ hg repack --packsonly
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histidx
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack $TESTTMP/hgcache/master/packs/1e91b207daf5d7b48f1be9c587d6b5ae654ce78c.histpack
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.dataidx
$TESTTMP/hgcache/master/packs/8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack $TESTTMP/hgcache/master/packs/add67cb28ae0a2962111588ce49467ca9ebb9195.datapack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --traceback $ hg repack --traceback
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Verify all the file data is still available # Verify all the file data is still available
$ hg cat -r . x $ hg cat -r . x
x x
x x
x x
$ hg cat -r '.^' x $ hg cat -r '.^' x
x x
x x
# Test that repacking again without new data does not delete the pack files # Test that repacking again without new data does not delete the pack files
# and did not change the pack names # and did not change the pack names
$ hg repack $ hg repack
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Run two repacks at once # Run two repacks at once
$ hg repack --config "hooks.prerepack=sleep 3" & $ hg repack --config "hooks.prerepack=sleep 3" &
$ sleep 1 $ sleep 1
$ hg repack $ hg repack
skipping repack - another repack is already running skipping repack - another repack is already running
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
# Run repack in the background # Run repack in the background
$ cd ../master $ cd ../master
$ echo x >> x $ echo x >> x
$ hg commit -m x4 $ hg commit -m x4
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72 $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1bb2e6237e035c8f8ef508e281f1ce075bc6db72
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histidx $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.dataidx
$TESTTMP/hgcache/master/packs/077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack $TESTTMP/hgcache/master/packs/1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.dataidx $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histidx
$TESTTMP/hgcache/master/packs/935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack $TESTTMP/hgcache/master/packs/8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
$ hg repack --background $ hg repack --background
(running background repack) (running background repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ find $CACHEDIR -type f | sort $ find $CACHEDIR -type f | sort
$TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histidx $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.dataidx
$TESTTMP/hgcache/master/packs/094b530486dad4427a0faf6bcbc031571b99ca24.histpack $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.dataidx $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histidx
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15.datapack $TESTTMP/hgcache/master/packs/604552d403a1381749faf656feca0ca265a6d52c.histpack
$TESTTMP/hgcache/master/packs/repacklock $TESTTMP/hgcache/master/packs/repacklock
$TESTTMP/hgcache/repos $TESTTMP/hgcache/repos
# Test debug commands # Test debug commands
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a d4a3ed9310e5 12 4 aee31534993a d4a3ed9310e5 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
$ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack --long $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 8 8
d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 12 6
aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4 aee31534993a501858fb6dd96a065671922e7d51 d4a3ed9310e5bd9887e3bf779da5077efab28216 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216 $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack --node d4a3ed9310e5bd9887e3bf779da5077efab28216
$TESTTMP/hgcache/master/packs/8fe685c56f6f7edf550bfcec74eeecc5f3c2ba15: $TESTTMP/hgcache/master/packs/06ae46494f0e3b9beda53eae8fc0e55139f13123:
x x
Node Delta Base Delta SHA1 Delta Length Node Delta Base Delta SHA1 Delta Length
d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 77029ab56e83ea2115dd53ff87483682abe5d7ca 12 d4a3ed9310e5bd9887e3bf779da5077efab28216 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 77029ab56e83ea2115dd53ff87483682abe5d7ca 12
Node Delta Base Delta SHA1 Delta Length Node Delta Base Delta SHA1 Delta Length
1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5 8 1bb2e6237e035c8f8ef508e281f1ce075bc6db72 0000000000000000000000000000000000000000 7ca8c71a64f7b56380e77573da2f7a5fdd2ecdb5 8
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
$ hg revert x $ hg revert x
$ hg commit -m 'add y back without metadata' $ hg commit -m 'add y back without metadata'
$ cd ../shallow $ cd ../shallow
$ hg pull -q $ hg pull -q
$ hg up -q tip $ hg up -q tip
2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ ls $TESTTMP/hgcache/master/packs $ ls $TESTTMP/hgcache/master/packs
e8fdf7ae22b772dcc291f905b9c6e5f381d28739.dataidx 308a7aba9c54a0b71ae5adbbccd00c0aff20876e.dataidx
e8fdf7ae22b772dcc291f905b9c6e5f381d28739.datapack 308a7aba9c54a0b71ae5adbbccd00c0aff20876e.datapack
ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histidx bfd60adb76018bb952e27cd23fc151bf94865d7d.histidx
ebbd7411e00456c0eec8d1150a77e2b3ef490f3f.histpack bfd60adb76018bb952e27cd23fc151bf94865d7d.histpack
repacklock repacklock
$ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx $ hg debughistorypack $TESTTMP/hgcache/master/packs/*.histidx
x x
Node P1 Node P2 Node Link Node Copy From Node P1 Node P2 Node Link Node Copy From
cd410a44d584 577959738234 000000000000 609547eda446 y cd410a44d584 577959738234 000000000000 609547eda446 y
1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7 1bb2e6237e03 d4a3ed9310e5 000000000000 0b03bbc9e1e7
d4a3ed9310e5 aee31534993a 000000000000 421535db10b6 d4a3ed9310e5 aee31534993a 000000000000 421535db10b6
$ rm -rf $TESTTMP/hgcache/master/packs $ rm -rf $TESTTMP/hgcache/master/packs
$ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs $ cp -R $TESTTMP/backuppacks $TESTTMP/hgcache/master/packs
# Test repacking datapack without history # Test repacking datapack without history
$ rm -rf $CACHEDIR/master/packs/*hist* $ rm -rf $CACHEDIR/master/packs/*hist*
$ hg repack $ hg repack
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/a8d86ff8e1a11a77a85f5fea567f56a757583eda: $TESTTMP/hgcache/master/packs/ba4649b56263282b0699f9a6e7e34a4a2bac1638:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a d4a3ed9310e5 12 4 aee31534993a d4a3ed9310e5 12 4
Total: 32 18 (77.8% bigger) Total: 32 18 (77.8% bigger)
y: y:
$ hg prefetch -r 0 $ hg prefetch -r 0
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
[1] [1]
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
[1] [1]
$ hg repack --incremental $ hg repack --incremental
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1 3 gen1 packs, 1 gen0 pack - packs 3 gen1 into 1
$ hg prefetch -r 1 $ hg prefetch -r 1
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 2 $ hg prefetch -r 2
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 38 $ hg prefetch -r 38
abort: unknown revision '38'! abort: unknown revision '38'!
[255] [255]
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
For the data packs, setting the limit for the repackmaxpacksize to be 64 such For the data packs, setting the limit for the repackmaxpacksize to be 64 such
that data pack with size 65 is more than the limit. This effectively ensures that data pack with size 65 is more than the limit. This effectively ensures
that no generation has 3 packs and therefore, no packs are chosen for the that no generation has 3 packs and therefore, no packs are chosen for the
incremental repacking. As for the history packs, setting repackmaxpacksize to be incremental repacking. As for the history packs, setting repackmaxpacksize to be
0 which should always result in no repacking. 0 which should always result in no repacking.
$ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=64 \ $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=64 \
> --config remotefilelog.history.repackmaxpacksize=0 > --config remotefilelog.history.repackmaxpacksize=0
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack -r--r--r-- 147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack -r--r--r-- 254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
Setting limit for the repackmaxpacksize to be the size of the biggest pack file Setting limit for the repackmaxpacksize to be the size of the biggest pack file
which ensures that it is effectively ignored in the incremental repacking. which ensures that it is effectively ignored in the incremental repacking.
$ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=65 \ $ hg repack --incremental --config remotefilelog.data.repackmaxpacksize=65 \
> --config remotefilelog.history.repackmaxpacksize=336 > --config remotefilelog.history.repackmaxpacksize=336
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack -r--r--r-- 147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack -r--r--r-- 254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
1 gen3 pack, 1 gen0 pack - does nothing 1 gen3 pack, 1 gen0 pack - does nothing
$ hg repack --incremental $ hg repack --incremental
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 147 935861cae0be6ce41a0d47a529e4d097e9e68a69.datapack -r--r--r-- 147 1bd27e610ee06450e5f3bb0cd3afb6870e4cf375.datapack
-r--r--r-- 67 b5a62f3496ccbd2479497cdbc7345f3304735f33.datapack -r--r--r-- 67 6409c5a1d61b251906689d4d1282ac44df6a7898.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 254 077e7ce5dfe862dc40cc8f3c9742d96a056865f2.histpack -r--r--r-- 254 8abe7889aae389337d12ebe6085d4ee13854c7c9.histpack
-r--r--r-- 90 c3399b56e035f73c3295276ed098235a08a0ed8c.histpack -r--r--r-- 90 955a622173324b2d8b53e1147f209f1cf125302e.histpack
Pull should run background repack Pull should run background repack
$ cat >> .hg/hgrc <<EOF $ cat >> .hg/hgrc <<EOF
> [remotefilelog] > [remotefilelog]
> backgroundrepack=True > backgroundrepack=True
> EOF > EOF
$ clearcache $ clearcache
$ hg prefetch -r 0 $ hg prefetch -r 0
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 1 $ hg prefetch -r 1
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 2 $ hg prefetch -r 2
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg prefetch -r 3 $ hg prefetch -r 3
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ hg pull $ hg pull
pulling from ssh://user@dummy/master pulling from ssh://user@dummy/master
searching for changes searching for changes
no changes found no changes found
(running background incremental repack) (running background incremental repack)
$ sleep 0.5 $ sleep 0.5
$ hg debugwaitonrepack >/dev/null 2>&1 $ hg debugwaitonrepack >/dev/null 2>&1
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
-r--r--r-- 301 09b8bf49256b3fc2175977ba97d6402e91a9a604.datapack -r--r--r-- 301 671913bebdb7b95aae52a546662753eac7606e40.datapack
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
-r--r--r-- 336 094b530486dad4427a0faf6bcbc031571b99ca24.histpack -r--r--r-- 336 604552d403a1381749faf656feca0ca265a6d52c.histpack
Test environment variable resolution Test environment variable resolution
$ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH' $ CACHEPATH=$TESTTMP/envcache hg prefetch --config 'remotefilelog.cachepath=$CACHEPATH'
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) 1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ find $TESTTMP/envcache | sort $ find $TESTTMP/envcache | sort
$TESTTMP/envcache $TESTTMP/envcache
$TESTTMP/envcache/master $TESTTMP/envcache/master
$TESTTMP/envcache/master/95 $TESTTMP/envcache/master/95
1bb2e6237e03 => d4a3ed9310e5 000000000000 0b03bbc9e1e7 1bb2e6237e03 => d4a3ed9310e5 000000000000 0b03bbc9e1e7
d4a3ed9310e5 => aee31534993a 000000000000 421535db10b6 d4a3ed9310e5 => aee31534993a 000000000000 421535db10b6
aee31534993a => 1406e7411862 000000000000 a89d614e2364 aee31534993a => 1406e7411862 000000000000 a89d614e2364
1406e7411862 => 000000000000 000000000000 b292c1e3311f 1406e7411862 => 000000000000 000000000000 b292c1e3311f
Test limiting the max delta chain length Test limiting the max delta chain length
$ hg repack --config packs.maxchainlen=1 $ hg repack --config packs.maxchainlen=1
$ hg debugdatapack $TESTTMP/hgcache/master/packs/*.dataidx $ hg debugdatapack $TESTTMP/hgcache/master/packs/*.dataidx
$TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742: $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a 000000000000 4 4 aee31534993a 000000000000 4 4
1406e7411862 aee31534993a 12 2 1406e7411862 aee31534993a 12 2
Total: 36 20 (80.0% bigger) Total: 36 20 (80.0% bigger)
y: y:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
577959738234 000000000000 70 8 577959738234 000000000000 70 8
Total: 70 8 (775.0% bigger) Total: 70 8 (775.0% bigger)
Test huge pack cleanup using different values of packs.maxpacksize: Test huge pack cleanup using different values of packs.maxpacksize:
$ hg repack --incremental --debug $ hg repack --incremental --debug
$ hg repack --incremental --debug --config packs.maxpacksize=512 $ hg repack --incremental --debug --config packs.maxpacksize=512
removing oversize packfile $TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742.datapack (426 bytes) removing oversize packfile $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918.datapack (426 bytes)
removing oversize packfile $TESTTMP/hgcache/master/packs/a8378d888fe4325250720e51311a65d2509be742.dataidx (1.21 KB) removing oversize packfile $TESTTMP/hgcache/master/packs/80f7c299aeb22849ebc28fbd11bb76078aa55918.dataidx (1.21 KB)
Do a repack where the new pack reuses a delta from the old pack Do a repack where the new pack reuses a delta from the old pack
$ clearcache $ clearcache
$ hg prefetch -r '2::3' $ hg prefetch -r '2::3'
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ hg debugdatapack $CACHEDIR/master/packs/*.datapack $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/abf210f6c3aa4dd0ecc7033633ad73591be16c95: $TESTTMP/hgcache/master/packs/90cfb1a45e2549500caace30add04b58a6b243af:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
Total: 20 14 (42.9% bigger) Total: 20 14 (42.9% bigger)
$ hg prefetch -r '0::1' $ hg prefetch -r '0::1'
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) 2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
$ hg repack $ hg repack
$ hg debugdatapack $CACHEDIR/master/packs/*.datapack $ hg debugdatapack $CACHEDIR/master/packs/*.datapack
$TESTTMP/hgcache/master/packs/09b8bf49256b3fc2175977ba97d6402e91a9a604: $TESTTMP/hgcache/master/packs/671913bebdb7b95aae52a546662753eac7606e40:
x: x:
Node Delta Base Delta Length Blob Size Node Delta Base Delta Length Blob Size
1bb2e6237e03 000000000000 8 8 1bb2e6237e03 000000000000 8 8
d4a3ed9310e5 1bb2e6237e03 12 6 d4a3ed9310e5 1bb2e6237e03 12 6
aee31534993a d4a3ed9310e5 12 4 aee31534993a d4a3ed9310e5 12 4
1406e7411862 aee31534993a 12 2 1406e7411862 aee31534993a 12 2
Total: 44 20 (120.0% bigger) Total: 44 20 (120.0% bigger)