| indygreg | |
| Alphare |
| hg-reviewers |
All path writing sidedata are now using compression (when appropriate).
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/cext/revlog.c (8 lines) | |||
| M | mercurial/pure/parsers.py (8 lines) | |||
| M | mercurial/revlog.py (21 lines) |
| /* Replace an existing index entry's sidedata offset and length with new ones. | /* Replace an existing index entry's sidedata offset and length with new ones. | ||||
| This cannot be used outside of the context of sidedata rewriting, | This cannot be used outside of the context of sidedata rewriting, | ||||
| inside the transaction that creates the given revision. */ | inside the transaction that creates the given revision. */ | ||||
| static PyObject *index_replace_sidedata_info(indexObject *self, PyObject *args) | static PyObject *index_replace_sidedata_info(indexObject *self, PyObject *args) | ||||
| { | { | ||||
| uint64_t offset_flags, sidedata_offset; | uint64_t offset_flags, sidedata_offset; | ||||
| int rev; | int rev; | ||||
| char comp_mode; | |||||
| Py_ssize_t sidedata_comp_len; | Py_ssize_t sidedata_comp_len; | ||||
| char *data; | char *data; | ||||
| #if LONG_MAX == 0x7fffffffL | #if LONG_MAX == 0x7fffffffL | ||||
| const char *const sidedata_format = PY23("nKiK", "nKiK"); | const char *const sidedata_format = PY23("nKiKB", "nKiKB"); | ||||
| #else | #else | ||||
| const char *const sidedata_format = PY23("nkik", "nkik"); | const char *const sidedata_format = PY23("nkikB", "nkikB"); | ||||
| #endif | #endif | ||||
| if (self->entry_size == v1_entry_size || self->inlined) { | if (self->entry_size == v1_entry_size || self->inlined) { | ||||
| /* | /* | ||||
| There is a bug in the transaction handling when going from an | There is a bug in the transaction handling when going from an | ||||
| inline revlog to a separate index and data file. Turn it off until | inline revlog to a separate index and data file. Turn it off until | ||||
| it's fixed, since v2 revlogs sometimes get rewritten on exchange. | it's fixed, since v2 revlogs sometimes get rewritten on exchange. | ||||
| See issue6485. | See issue6485. | ||||
| */ | */ | ||||
| raise_revlog_error(); | raise_revlog_error(); | ||||
| return NULL; | return NULL; | ||||
| } | } | ||||
| if (!PyArg_ParseTuple(args, sidedata_format, &rev, &sidedata_offset, | if (!PyArg_ParseTuple(args, sidedata_format, &rev, &sidedata_offset, | ||||
| &sidedata_comp_len, &offset_flags)) | &sidedata_comp_len, &offset_flags, &comp_mode)) | ||||
| return NULL; | return NULL; | ||||
| if (rev < 0 || rev >= index_length(self)) { | if (rev < 0 || rev >= index_length(self)) { | ||||
| PyErr_SetString(PyExc_IndexError, "revision outside index"); | PyErr_SetString(PyExc_IndexError, "revision outside index"); | ||||
| return NULL; | return NULL; | ||||
| } | } | ||||
| if (rev < self->length) { | if (rev < self->length) { | ||||
| PyErr_SetString( | PyErr_SetString( | ||||
| PyExc_IndexError, | PyExc_IndexError, | ||||
| "cannot rewrite entries outside of this transaction"); | "cannot rewrite entries outside of this transaction"); | ||||
| return NULL; | return NULL; | ||||
| } | } | ||||
| /* Find the newly added node, offset from the "already on-disk" length | /* Find the newly added node, offset from the "already on-disk" length | ||||
| */ | */ | ||||
| data = self->added + self->entry_size * (rev - self->length); | data = self->added + self->entry_size * (rev - self->length); | ||||
| putbe64(offset_flags, data); | putbe64(offset_flags, data); | ||||
| putbe64(sidedata_offset, data + 64); | putbe64(sidedata_offset, data + 64); | ||||
| putbe32(sidedata_comp_len, data + 72); | putbe32(sidedata_comp_len, data + 72); | ||||
| data[76] = (data[76] & ~(3 << 2)) | ((comp_mode & 3) << 2); | |||||
| Py_RETURN_NONE; | Py_RETURN_NONE; | ||||
| } | } | ||||
| static PyObject *index_stats(indexObject *self) | static PyObject *index_stats(indexObject *self) | ||||
| { | { | ||||
| PyObject *obj = PyDict_New(); | PyObject *obj = PyDict_New(); | ||||
| PyObject *s = NULL; | PyObject *s = NULL; | ||||
| cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject | cls = InlinedIndexObject2 if revlogv2 else InlinedIndexObject | ||||
| return cls(data, inline), (0, data) | return cls(data, inline), (0, data) | ||||
| class Index2Mixin(object): | class Index2Mixin(object): | ||||
| index_format = revlog_constants.INDEX_ENTRY_V2 | index_format = revlog_constants.INDEX_ENTRY_V2 | ||||
| def replace_sidedata_info( | def replace_sidedata_info( | ||||
| self, rev, sidedata_offset, sidedata_length, offset_flags | self, | ||||
| rev, | |||||
| sidedata_offset, | |||||
| sidedata_length, | |||||
| offset_flags, | |||||
| compression_mode, | |||||
| ): | ): | ||||
| """ | """ | ||||
| Replace an existing index entry's sidedata offset and length with new | Replace an existing index entry's sidedata offset and length with new | ||||
| ones. | ones. | ||||
| This cannot be used outside of the context of sidedata rewriting, | This cannot be used outside of the context of sidedata rewriting, | ||||
| inside the transaction that creates the revision `rev`. | inside the transaction that creates the revision `rev`. | ||||
| """ | """ | ||||
| if rev < 0: | if rev < 0: | ||||
| raise KeyError | raise KeyError | ||||
| self._check_index(rev) | self._check_index(rev) | ||||
| if rev < self._lgt: | if rev < self._lgt: | ||||
| msg = b"cannot rewrite entries outside of this transaction" | msg = b"cannot rewrite entries outside of this transaction" | ||||
| raise KeyError(msg) | raise KeyError(msg) | ||||
| else: | else: | ||||
| entry = list(self[rev]) | entry = list(self[rev]) | ||||
| entry[0] = offset_flags | entry[0] = offset_flags | ||||
| entry[8] = sidedata_offset | entry[8] = sidedata_offset | ||||
| entry[9] = sidedata_length | entry[9] = sidedata_length | ||||
| entry[11] = compression_mode | |||||
| entry = tuple(entry) | entry = tuple(entry) | ||||
| new = self._pack_entry(entry) | new = self._pack_entry(entry) | ||||
| self._extra[rev - self._lgt] = new | self._extra[rev - self._lgt] = new | ||||
| def _unpack_entry(self, data): | def _unpack_entry(self, data): | ||||
| data = self.index_format.unpack(data) | data = self.index_format.unpack(data) | ||||
| entry = data[:10] | entry = data[:10] | ||||
| data_comp = data[10] & 3 | data_comp = data[10] & 3 | ||||
| sidedata_helpers=helpers, | sidedata_helpers=helpers, | ||||
| sidedata={}, | sidedata={}, | ||||
| rev=rev, | rev=rev, | ||||
| ) | ) | ||||
| serialized_sidedata = sidedatautil.serialize_sidedata( | serialized_sidedata = sidedatautil.serialize_sidedata( | ||||
| new_sidedata | new_sidedata | ||||
| ) | ) | ||||
| sidedata_compression_mode = COMP_MODE_INLINE | |||||
| if serialized_sidedata and self.hassidedata: | |||||
| sidedata_compression_mode = COMP_MODE_PLAIN | |||||
| h, comp_sidedata = self.compress(serialized_sidedata) | |||||
| if ( | |||||
| h != b'u' | |||||
| and comp_sidedata[0] != b'\0' | |||||
| and len(comp_sidedata) < len(serialized_sidedata) | |||||
| ): | |||||
| assert not h | |||||
| if ( | |||||
| comp_sidedata[0] | |||||
| == self._docket.default_compression_header | |||||
| ): | |||||
| sidedata_compression_mode = COMP_MODE_DEFAULT | |||||
| serialized_sidedata = comp_sidedata | |||||
| else: | |||||
| sidedata_compression_mode = COMP_MODE_INLINE | |||||
| serialized_sidedata = comp_sidedata | |||||
| if entry[8] != 0 or entry[9] != 0: | if entry[8] != 0 or entry[9] != 0: | ||||
| # rewriting entries that already have sidedata is not | # rewriting entries that already have sidedata is not | ||||
| # supported yet, because it introduces garbage data in the | # supported yet, because it introduces garbage data in the | ||||
| # revlog. | # revlog. | ||||
| msg = b"rewriting existing sidedata is not supported yet" | msg = b"rewriting existing sidedata is not supported yet" | ||||
| raise error.Abort(msg) | raise error.Abort(msg) | ||||
| # Apply (potential) flags to add and to remove after running | # Apply (potential) flags to add and to remove after running | ||||
| # the sidedata helpers | # the sidedata helpers | ||||
| new_offset_flags = entry[0] | flags[0] & ~flags[1] | new_offset_flags = entry[0] | flags[0] & ~flags[1] | ||||
| entry_update = ( | entry_update = ( | ||||
| current_offset, | current_offset, | ||||
| len(serialized_sidedata), | len(serialized_sidedata), | ||||
| new_offset_flags, | new_offset_flags, | ||||
| sidedata_compression_mode, | |||||
| ) | ) | ||||
| # the sidedata computation might have move the file cursors around | # the sidedata computation might have move the file cursors around | ||||
| dfh.seek(current_offset, os.SEEK_SET) | dfh.seek(current_offset, os.SEEK_SET) | ||||
| dfh.write(serialized_sidedata) | dfh.write(serialized_sidedata) | ||||
| new_entries.append(entry_update) | new_entries.append(entry_update) | ||||
| current_offset += len(serialized_sidedata) | current_offset += len(serialized_sidedata) | ||||
| if self._docket is not None: | if self._docket is not None: | ||||