( )⚙ D10650 revlog: introduce a plain compression mode

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10650

revlog: introduce a plain compression mode
ClosedPublic

Authored by marmoute on May 4 2021, 10:14 AM.

Details

Reviewers
indygreg
Alphare
Group Reviewers
hg-reviewers
Commits
rHGb876f0bf7366: revlog: introduce a plain compression mode
Summary

That mode is simple it means the chunk contains uncompressed data and can be
used directly.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 4 2021, 10:14 AM
joerg.sonnenberger added a subscriber: joerg.sonnenberger.May 4 2021, 1:02 PM

I don't understand this change. We already support none as compression method.

marmoute added a comment.May 5 2021, 3:47 PM
In D10650#161850, @joerg.sonnenberger wrote:

I don't understand this change. We already support none as compression method.

This is to read in the context of https://phab.mercurial-scm.org/D10646

Right now the compression information are "inline", each chunk starts with a byte that indicate the compression mode. One idea floating around for a while was to move more compression information in the index header to allow more streamlined handling of the chunk. (that would mostly matter for compiled language) This changeset is part of a series doing that.

I just pushed an update to the documentation of the constant to highlight that a bit. phabricator will be updated once the CI ran.

baymax updated this revision to Diff 27587.May 5 2021, 5:48 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

baymax updated this revision to Diff 27889.May 11 2021, 1:45 PM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

Alphare accepted this revision.May 17 2021, 5:49 PM
This revision is now accepted and ready to land.May 17 2021, 5:49 PM
marmoute added a commit: rHGb876f0bf7366: revlog: introduce a plain compression mode.May 17 2021, 5:55 PM
Closed by commit rHGb876f0bf7366: revlog: introduce a plain compression mode (authored by marmoute). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 28043

mercurial/revlog.py

short, short,
wdirrev, wdirrev,
) )
from .i18n import _ from .i18n import _
from .pycompat import getattr from .pycompat import getattr
from .revlogutils.constants import ( from .revlogutils.constants import (
ALL_KINDS, ALL_KINDS,
COMP_MODE_INLINE, COMP_MODE_INLINE,
COMP_MODE_PLAIN,
FEATURES_BY_VERSION, FEATURES_BY_VERSION,
FLAG_GENERALDELTA, FLAG_GENERALDELTA,
FLAG_INLINE_DATA, FLAG_INLINE_DATA,
INDEX_HEADER, INDEX_HEADER,
REVLOGV0, REVLOGV0,
REVLOGV1, REVLOGV1,
REVLOGV1_FLAGS, REVLOGV1_FLAGS,
REVLOGV2, REVLOGV2,
"""Obtain a single decompressed chunk for a revision. """Obtain a single decompressed chunk for a revision.
Accepts an integer revision and an optional already-open file handle Accepts an integer revision and an optional already-open file handle
to be used for reading. If used, the seek position of the file will not to be used for reading. If used, the seek position of the file will not
be preserved. be preserved.
Returns a str holding uncompressed data for the requested revision. Returns a str holding uncompressed data for the requested revision.
""" """
return self.decompress(self._getsegmentforrevs(rev, rev, df=df)[1]) compression_mode = self.index[rev][10]
data = self._getsegmentforrevs(rev, rev, df=df)[1]
if compression_mode == COMP_MODE_PLAIN:
return data
elif compression_mode == COMP_MODE_INLINE:
return self.decompress(data)
else:
msg = 'unknown compression mode %d'
msg %= compression_mode
raise error.RevlogError(msg)
def _chunks(self, revs, df=None, targetsize=None): def _chunks(self, revs, df=None, targetsize=None):
"""Obtain decompressed chunks for the specified revisions. """Obtain decompressed chunks for the specified revisions.
Accepts an iterable of numeric revisions that are assumed to be in Accepts an iterable of numeric revisions that are assumed to be in
ascending order. Also accepts an optional already-open file handle ascending order. Also accepts an optional already-open file handle
to be used for reading. If used, the seek position of the file will to be used for reading. If used, the seek position of the file will
not be preserved. not be preserved.
return [self._chunk(rev, df=df) for rev in revschunk] return [self._chunk(rev, df=df) for rev in revschunk]
decomp = self.decompress decomp = self.decompress
for rev in revschunk: for rev in revschunk:
chunkstart = start(rev) chunkstart = start(rev)
if inline: if inline:
chunkstart += (rev + 1) * iosize chunkstart += (rev + 1) * iosize
chunklength = length(rev) chunklength = length(rev)
comp_mode = self.index[rev][10]
c = buffer(data, chunkstart - offset, chunklength) c = buffer(data, chunkstart - offset, chunklength)
if comp_mode == COMP_MODE_PLAIN:
ladd(c)
elif comp_mode == COMP_MODE_INLINE:
ladd(decomp(c)) ladd(decomp(c))
else:
msg = 'unknown compression mode %d'
msg %= comp_mode
raise error.RevlogError(msg)
return l return l
def _chunkclear(self): def _chunkclear(self):
"""Clear the raw chunk cache.""" """Clear the raw chunk cache."""
self._chunkcache = (0, b'') self._chunkcache = (0, b'')
def deltaparent(self, rev): def deltaparent(self, rev):
if deltacomputer is None: if deltacomputer is None:
deltacomputer = deltautil.deltacomputer(self) deltacomputer = deltautil.deltacomputer(self)
revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags) revinfo = _revisioninfo(node, p1, p2, btext, textlen, cachedelta, flags)
deltainfo = deltacomputer.finddeltainfo(revinfo, fh) deltainfo = deltacomputer.finddeltainfo(revinfo, fh)
compression_mode = COMP_MODE_INLINE
if self._docket is not None:
h, d = deltainfo.data
if not h and not d:
# not data to store at all... declare them uncompressed
compression_mode = COMP_MODE_PLAIN
elif not h and d[0:1] == b'\0':
compression_mode = COMP_MODE_PLAIN
elif h == b'u':
# we have a more efficient way to declare uncompressed
h = b''
compression_mode = COMP_MODE_PLAIN
deltainfo = deltautil.drop_u_compression(deltainfo)
if sidedata and self.hassidedata: if sidedata and self.hassidedata:
serialized_sidedata = sidedatautil.serialize_sidedata(sidedata) serialized_sidedata = sidedatautil.serialize_sidedata(sidedata)
sidedata_offset = offset + deltainfo.deltalen sidedata_offset = offset + deltainfo.deltalen
else: else:
serialized_sidedata = b"" serialized_sidedata = b""
# Don't store the offset if the sidedata is empty, that way # Don't store the offset if the sidedata is empty, that way
# we can easily detect empty sidedata and they will be no different # we can easily detect empty sidedata and they will be no different
# than ones we manually add. # than ones we manually add.
sidedata_offset = 0 sidedata_offset = 0
e = ( e = (
offset_type(offset, flags), offset_type(offset, flags),
deltainfo.deltalen, deltainfo.deltalen,
textlen, textlen,
deltainfo.base, deltainfo.base,
link, link,
p1r, p1r,
p2r, p2r,
node, node,
sidedata_offset, sidedata_offset,
len(serialized_sidedata), len(serialized_sidedata),
COMP_MODE_INLINE, compression_mode,
) )
self.index.append(e) self.index.append(e)
entry = self.index.entry_binary(curr) entry = self.index.entry_binary(curr)
if curr == 0 and self._docket is None: if curr == 0 and self._docket is None:
header = self._format_flags | self._format_version header = self._format_flags | self._format_version
header = self.index.pack_header(header) header = self.index.pack_header(header)
entry = header + entry entry = header + entry

mercurial/revlogutils/constants.py

# bitmark for flags that could cause rawdata content change # bitmark for flags that could cause rawdata content change
REVIDX_RAWTEXT_CHANGING_FLAGS = REVIDX_ISCENSORED | REVIDX_EXTSTORED REVIDX_RAWTEXT_CHANGING_FLAGS = REVIDX_ISCENSORED | REVIDX_EXTSTORED
## chunk compression mode constants: ## chunk compression mode constants:
# These constants are used in revlog version >=2 to denote the compression used # These constants are used in revlog version >=2 to denote the compression used
# for a chunk. # for a chunk.
# Chunk use no compression, the data stored on disk can be directly use as
# chunk value. Without any header information prefixed.
COMP_MODE_PLAIN = 0
# Chunk use a compression mode stored "inline" at the start of the chunk # Chunk use a compression mode stored "inline" at the start of the chunk
# itself. This is the mode always used for revlog version "0" and "1" # itself. This is the mode always used for revlog version "0" and "1"
COMP_MODE_INLINE = 2 COMP_MODE_INLINE = 2
SUPPORTED_FLAGS = { SUPPORTED_FLAGS = {
REVLOGV0: REVLOGV0_FLAGS, REVLOGV0: REVLOGV0_FLAGS,
REVLOGV1: REVLOGV1_FLAGS, REVLOGV1: REVLOGV1_FLAGS,
REVLOGV2: REVLOGV2_FLAGS, REVLOGV2: REVLOGV2_FLAGS,

mercurial/revlogutils/deltas.py

data = attr.ib() data = attr.ib()
base = attr.ib() base = attr.ib()
chainbase = attr.ib() chainbase = attr.ib()
chainlen = attr.ib() chainlen = attr.ib()
compresseddeltalen = attr.ib() compresseddeltalen = attr.ib()
snapshotdepth = attr.ib() snapshotdepth = attr.ib()
def drop_u_compression(delta):
"""turn into a "u" (no-compression) into no-compression without header
This is useful for revlog format that has better compression method.
"""
assert delta.data[0] == b'u', delta.data[0]
return _deltainfo(
delta.distance,
delta.deltalen - 1,
(b'', delta.data[1]),
delta.base,
delta.chainbase,
delta.chainlen,
delta.compresseddeltalen,
delta.snapshotdepth,
)
def isgooddeltainfo(revlog, deltainfo, revinfo): def isgooddeltainfo(revlog, deltainfo, revinfo):
"""Returns True if the given delta is good. Good means that it is within """Returns True if the given delta is good. Good means that it is within
the disk span, disk size, and chain length bounds that we know to be the disk span, disk size, and chain length bounds that we know to be
performant.""" performant."""
if deltainfo is None: if deltainfo is None:
return False return False
# - 'deltainfo.distance' is the distance from the base revision -- # - 'deltainfo.distance' is the distance from the base revision --