| indygreg | |
| Alphare |
| hg-reviewers |
The revlog docker is now storing a default compression engine. When a chunk use
that compression, a dedicated mode is used in the revlog entry and we can
directly route it to the right decompressor.
We should probably make PLAIN and DEFAULT mode the only available mode for
revlogv2, but this is something for later.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/revlog.py (24 lines) | |||
| M | mercurial/revlogutils/constants.py (10 lines) | |||
| M | mercurial/revlogutils/docket.py (16 lines) |
| sha1nodeconstants, | sha1nodeconstants, | ||||
| short, | short, | ||||
| wdirrev, | wdirrev, | ||||
| ) | ) | ||||
| from .i18n import _ | from .i18n import _ | ||||
| from .pycompat import getattr | from .pycompat import getattr | ||||
| from .revlogutils.constants import ( | from .revlogutils.constants import ( | ||||
| ALL_KINDS, | ALL_KINDS, | ||||
| COMP_MODE_DEFAULT, | |||||
| COMP_MODE_INLINE, | COMP_MODE_INLINE, | ||||
| COMP_MODE_PLAIN, | COMP_MODE_PLAIN, | ||||
| FEATURES_BY_VERSION, | FEATURES_BY_VERSION, | ||||
| FLAG_GENERALDELTA, | FLAG_GENERALDELTA, | ||||
| FLAG_INLINE_DATA, | FLAG_INLINE_DATA, | ||||
| INDEX_HEADER, | INDEX_HEADER, | ||||
| REVLOGV0, | REVLOGV0, | ||||
| REVLOGV1, | REVLOGV1, | ||||
| ) | ) | ||||
| return compressor | return compressor | ||||
| @util.propertycache | @util.propertycache | ||||
| def _compressor(self): | def _compressor(self): | ||||
| engine = util.compengines[self._compengine] | engine = util.compengines[self._compengine] | ||||
| return engine.revlogcompressor(self._compengineopts) | return engine.revlogcompressor(self._compengineopts) | ||||
| @util.propertycache | |||||
| def _decompressor(self): | |||||
| """the default decompressor""" | |||||
| if self._docket is None: | |||||
| return None | |||||
| t = self._docket.default_compression_header | |||||
| c = self._get_decompressor(t) | |||||
| return c.decompress | |||||
| def _indexfp(self): | def _indexfp(self): | ||||
| """file object for the revlog's index file""" | """file object for the revlog's index file""" | ||||
| return self.opener(self._indexfile, mode=b"r") | return self.opener(self._indexfile, mode=b"r") | ||||
| def __index_write_fp(self): | def __index_write_fp(self): | ||||
| # You should not use this directly and use `_writing` instead | # You should not use this directly and use `_writing` instead | ||||
| try: | try: | ||||
| f = self.opener( | f = self.opener( | ||||
| be preserved. | be preserved. | ||||
| Returns a str holding uncompressed data for the requested revision. | Returns a str holding uncompressed data for the requested revision. | ||||
| """ | """ | ||||
| compression_mode = self.index[rev][10] | compression_mode = self.index[rev][10] | ||||
| data = self._getsegmentforrevs(rev, rev, df=df)[1] | data = self._getsegmentforrevs(rev, rev, df=df)[1] | ||||
| if compression_mode == COMP_MODE_PLAIN: | if compression_mode == COMP_MODE_PLAIN: | ||||
| return data | return data | ||||
| elif compression_mode == COMP_MODE_DEFAULT: | |||||
| return self._decompressor(data) | |||||
| elif compression_mode == COMP_MODE_INLINE: | elif compression_mode == COMP_MODE_INLINE: | ||||
| return self.decompress(data) | return self.decompress(data) | ||||
| else: | else: | ||||
| msg = 'unknown compression mode %d' | msg = 'unknown compression mode %d' | ||||
| msg %= compression_mode | msg %= compression_mode | ||||
| raise error.RevlogError(msg) | raise error.RevlogError(msg) | ||||
| def _chunks(self, revs, df=None, targetsize=None): | def _chunks(self, revs, df=None, targetsize=None): | ||||
| try: | try: | ||||
| offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df) | offset, data = self._getsegmentforrevs(firstrev, lastrev, df=df) | ||||
| except OverflowError: | except OverflowError: | ||||
| # issue4215 - we can't cache a run of chunks greater than | # issue4215 - we can't cache a run of chunks greater than | ||||
| # 2G on Windows | # 2G on Windows | ||||
| return [self._chunk(rev, df=df) for rev in revschunk] | return [self._chunk(rev, df=df) for rev in revschunk] | ||||
| decomp = self.decompress | decomp = self.decompress | ||||
| # self._decompressor might be None, but will not be used in that case | |||||
| def_decomp = self._decompressor | |||||
| for rev in revschunk: | for rev in revschunk: | ||||
| chunkstart = start(rev) | chunkstart = start(rev) | ||||
| if inline: | if inline: | ||||
| chunkstart += (rev + 1) * iosize | chunkstart += (rev + 1) * iosize | ||||
| chunklength = length(rev) | chunklength = length(rev) | ||||
| comp_mode = self.index[rev][10] | comp_mode = self.index[rev][10] | ||||
| c = buffer(data, chunkstart - offset, chunklength) | c = buffer(data, chunkstart - offset, chunklength) | ||||
| if comp_mode == COMP_MODE_PLAIN: | if comp_mode == COMP_MODE_PLAIN: | ||||
| ladd(c) | ladd(c) | ||||
| elif comp_mode == COMP_MODE_INLINE: | elif comp_mode == COMP_MODE_INLINE: | ||||
| ladd(decomp(c)) | ladd(decomp(c)) | ||||
| elif comp_mode == COMP_MODE_DEFAULT: | |||||
| ladd(def_decomp(c)) | |||||
| else: | else: | ||||
| msg = 'unknown compression mode %d' | msg = 'unknown compression mode %d' | ||||
| msg %= comp_mode | msg %= comp_mode | ||||
| raise error.RevlogError(msg) | raise error.RevlogError(msg) | ||||
| return l | return l | ||||
| def _chunkclear(self): | def _chunkclear(self): | ||||
| deltainfo = deltacomputer.finddeltainfo(revinfo, fh) | deltainfo = deltacomputer.finddeltainfo(revinfo, fh) | ||||
| compression_mode = COMP_MODE_INLINE | compression_mode = COMP_MODE_INLINE | ||||
| if self._docket is not None: | if self._docket is not None: | ||||
| h, d = deltainfo.data | h, d = deltainfo.data | ||||
| if not h and not d: | if not h and not d: | ||||
| # not data to store at all... declare them uncompressed | # not data to store at all... declare them uncompressed | ||||
| compression_mode = COMP_MODE_PLAIN | compression_mode = COMP_MODE_PLAIN | ||||
| elif not h and d[0:1] == b'\0': | elif not h: | ||||
| t = d[0:1] | |||||
| if t == b'\0': | |||||
| compression_mode = COMP_MODE_PLAIN | compression_mode = COMP_MODE_PLAIN | ||||
| elif t == self._docket.default_compression_header: | |||||
| compression_mode = COMP_MODE_DEFAULT | |||||
| elif h == b'u': | elif h == b'u': | ||||
| # we have a more efficient way to declare uncompressed | # we have a more efficient way to declare uncompressed | ||||
| h = b'' | h = b'' | ||||
| compression_mode = COMP_MODE_PLAIN | compression_mode = COMP_MODE_PLAIN | ||||
| deltainfo = deltautil.drop_u_compression(deltainfo) | deltainfo = deltautil.drop_u_compression(deltainfo) | ||||
| if sidedata and self.hassidedata: | if sidedata and self.hassidedata: | ||||
| serialized_sidedata = sidedatautil.serialize_sidedata(sidedata) | serialized_sidedata = sidedatautil.serialize_sidedata(sidedata) | ||||
| ## chunk compression mode constants: | ## chunk compression mode constants: | ||||
| # These constants are used in revlog version >=2 to denote the compression used | # These constants are used in revlog version >=2 to denote the compression used | ||||
| # for a chunk. | # for a chunk. | ||||
| # Chunk use no compression, the data stored on disk can be directly use as | # Chunk use no compression, the data stored on disk can be directly use as | ||||
| # chunk value. Without any header information prefixed. | # chunk value. Without any header information prefixed. | ||||
| COMP_MODE_PLAIN = 0 | COMP_MODE_PLAIN = 0 | ||||
| # Chunk use the "default compression" for the revlog (usually defined in the | |||||
| # revlog docket). A header is still used. | |||||
| # | |||||
| # XXX: keeping a header is probably not useful and we should probably drop it. | |||||
| # | |||||
| # XXX: The value of allow mixed type of compression in the revlog is unclear | |||||
| # and we should consider making PLAIN/DEFAULT the only available mode for | |||||
| # revlog v2, disallowing INLINE mode. | |||||
| COMP_MODE_DEFAULT = 1 | |||||
| # Chunk use a compression mode stored "inline" at the start of the chunk | # Chunk use a compression mode stored "inline" at the start of the chunk | ||||
| # itself. This is the mode always used for revlog version "0" and "1" | # itself. This is the mode always used for revlog version "0" and "1" | ||||
| COMP_MODE_INLINE = 2 | COMP_MODE_INLINE = 2 | ||||
| SUPPORTED_FLAGS = { | SUPPORTED_FLAGS = { | ||||
| REVLOGV0: REVLOGV0_FLAGS, | REVLOGV0: REVLOGV0_FLAGS, | ||||
| REVLOGV1: REVLOGV1_FLAGS, | REVLOGV1: REVLOGV1_FLAGS, | ||||
| REVLOGV2: REVLOGV2_FLAGS, | REVLOGV2: REVLOGV2_FLAGS, | ||||
| # * a data file, containing variable width data for these revisions, | # * a data file, containing variable width data for these revisions, | ||||
| from __future__ import absolute_import | from __future__ import absolute_import | ||||
| import struct | import struct | ||||
| from .. import ( | from .. import ( | ||||
| error, | error, | ||||
| util, | |||||
| ) | ) | ||||
| from . import ( | from . import ( | ||||
| constants, | constants, | ||||
| ) | ) | ||||
| # Docket format | # Docket format | ||||
| # | # | ||||
| # * 4 bytes: revlog version | # * 4 bytes: revlog version | ||||
| # | This is mandatory as docket must be compatible with the previous | # | This is mandatory as docket must be compatible with the previous | ||||
| # | revlog index header. | # | revlog index header. | ||||
| # * 8 bytes: size of index-data | # * 8 bytes: size of index-data | ||||
| # * 8 bytes: pending size of index-data | # * 8 bytes: pending size of index-data | ||||
| # * 8 bytes: size of data | # * 8 bytes: size of data | ||||
| # * 8 bytes: pending size of data | # * 8 bytes: pending size of data | ||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LLLL') | # * 1 bytes: default compression header | ||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LLLLc') | |||||
| class RevlogDocket(object): | class RevlogDocket(object): | ||||
| """metadata associated with revlog""" | """metadata associated with revlog""" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| revlog, | revlog, | ||||
| use_pending=False, | use_pending=False, | ||||
| version_header=None, | version_header=None, | ||||
| index_end=0, | index_end=0, | ||||
| pending_index_end=0, | pending_index_end=0, | ||||
| data_end=0, | data_end=0, | ||||
| pending_data_end=0, | pending_data_end=0, | ||||
| default_compression_header=None, | |||||
| ): | ): | ||||
| self._version_header = version_header | self._version_header = version_header | ||||
| self._read_only = bool(use_pending) | self._read_only = bool(use_pending) | ||||
| self._dirty = False | self._dirty = False | ||||
| self._radix = revlog.radix | self._radix = revlog.radix | ||||
| self._path = revlog._docket_file | self._path = revlog._docket_file | ||||
| self._opener = revlog.opener | self._opener = revlog.opener | ||||
| # thes asserts should be True as long as we have a single index filename | # thes asserts should be True as long as we have a single index filename | ||||
| assert index_end <= pending_index_end | assert index_end <= pending_index_end | ||||
| assert data_end <= pending_data_end | assert data_end <= pending_data_end | ||||
| self._initial_index_end = index_end | self._initial_index_end = index_end | ||||
| self._pending_index_end = pending_index_end | self._pending_index_end = pending_index_end | ||||
| self._initial_data_end = data_end | self._initial_data_end = data_end | ||||
| self._pending_data_end = pending_data_end | self._pending_data_end = pending_data_end | ||||
| if use_pending: | if use_pending: | ||||
| self._index_end = self._pending_index_end | self._index_end = self._pending_index_end | ||||
| self._data_end = self._pending_data_end | self._data_end = self._pending_data_end | ||||
| else: | else: | ||||
| self._index_end = self._initial_index_end | self._index_end = self._initial_index_end | ||||
| self._data_end = self._initial_data_end | self._data_end = self._initial_data_end | ||||
| self.default_compression_header = default_compression_header | |||||
| def index_filepath(self): | def index_filepath(self): | ||||
| """file path to the current index file associated to this docket""" | """file path to the current index file associated to this docket""" | ||||
| # very simplistic version at first | # very simplistic version at first | ||||
| return b"%s.idx" % self._radix | return b"%s.idx" % self._radix | ||||
| @property | @property | ||||
| def index_end(self): | def index_end(self): | ||||
| # this assert should be True as long as we have a single index filename | # this assert should be True as long as we have a single index filename | ||||
| assert official_data_end <= self._data_end | assert official_data_end <= self._data_end | ||||
| data = ( | data = ( | ||||
| self._version_header, | self._version_header, | ||||
| official_index_end, | official_index_end, | ||||
| self._index_end, | self._index_end, | ||||
| official_data_end, | official_data_end, | ||||
| self._data_end, | self._data_end, | ||||
| self.default_compression_header, | |||||
| ) | ) | ||||
| return S_HEADER.pack(*data) | return S_HEADER.pack(*data) | ||||
| def default_docket(revlog, version_header): | def default_docket(revlog, version_header): | ||||
| """given a revlog version a new docket object for the given revlog""" | """given a revlog version a new docket object for the given revlog""" | ||||
| if (version_header & 0xFFFF) != constants.REVLOGV2: | if (version_header & 0xFFFF) != constants.REVLOGV2: | ||||
| return None | return None | ||||
| docket = RevlogDocket(revlog, version_header=version_header) | comp = util.compengines[revlog._compengine].revlogheader() | ||||
| docket = RevlogDocket( | |||||
| revlog, | |||||
| version_header=version_header, | |||||
| default_compression_header=comp, | |||||
| ) | |||||
| docket._dirty = True | docket._dirty = True | ||||
| return docket | return docket | ||||
| def parse_docket(revlog, data, use_pending=False): | def parse_docket(revlog, data, use_pending=False): | ||||
| """given some docket data return a docket object for the given revlog""" | """given some docket data return a docket object for the given revlog""" | ||||
| header = S_HEADER.unpack(data[: S_HEADER.size]) | header = S_HEADER.unpack(data[: S_HEADER.size]) | ||||
| version_header = header[0] | version_header = header[0] | ||||
| index_size = header[1] | index_size = header[1] | ||||
| pending_index_size = header[2] | pending_index_size = header[2] | ||||
| data_size = header[3] | data_size = header[3] | ||||
| pending_data_size = header[4] | pending_data_size = header[4] | ||||
| default_compression_header = header[5] | |||||
| docket = RevlogDocket( | docket = RevlogDocket( | ||||
| revlog, | revlog, | ||||
| use_pending=use_pending, | use_pending=use_pending, | ||||
| version_header=version_header, | version_header=version_header, | ||||
| index_end=index_size, | index_end=index_size, | ||||
| pending_index_end=pending_index_size, | pending_index_end=pending_index_size, | ||||
| data_end=data_size, | data_end=data_size, | ||||
| pending_data_end=pending_data_size, | pending_data_end=pending_data_size, | ||||
| default_compression_header=default_compression_header, | |||||
| ) | ) | ||||
| return docket | return docket | ||||