| indygreg | |
| Alphare |
| hg-reviewers |
This is useful to make sure we always start writing at the right location,
without effort.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/configitems.py (1 line) | |||
| M | mercurial/revlog.py (38 lines) | |||
| M | mercurial/revlogutils/docket.py (41 lines) |
| # "out of experimental" todo list. | # "out of experimental" todo list. | ||||
| # | # | ||||
| # * include management of a persistent nodemap in the main docket | # * include management of a persistent nodemap in the main docket | ||||
| # * enforce a "no-truncate" policy for mmap safety | # * enforce a "no-truncate" policy for mmap safety | ||||
| # - for censoring operation | # - for censoring operation | ||||
| # - for stripping operation | # - for stripping operation | ||||
| # - for rollback operation | # - for rollback operation | ||||
| # * proper streaming (race free) of the docket file | # * proper streaming (race free) of the docket file | ||||
| # * store the data size in the docket to simplify sidedata rewrite. | |||||
| # * track garbage data to evemtually allow rewriting -existing- sidedata. | # * track garbage data to evemtually allow rewriting -existing- sidedata. | ||||
| # * Exchange-wise, we will also need to do something more efficient than | # * Exchange-wise, we will also need to do something more efficient than | ||||
| # keeping references to the affected revlogs, especially memory-wise when | # keeping references to the affected revlogs, especially memory-wise when | ||||
| # rewriting sidedata. | # rewriting sidedata. | ||||
| # * sidedata compression | # * sidedata compression | ||||
| # * introduce a proper solution to reduce the number of filelog related files. | # * introduce a proper solution to reduce the number of filelog related files. | ||||
| # * Improvement to consider | # * Improvement to consider | ||||
| # - track compression mode in the index entris instead of the chunks | # - track compression mode in the index entris instead of the chunks | ||||
| r = len(self) | r = len(self) | ||||
| dsize = 0 | dsize = 0 | ||||
| if r: | if r: | ||||
| dsize = self.end(r - 1) | dsize = self.end(r - 1) | ||||
| dfh = None | dfh = None | ||||
| if not self._inline: | if not self._inline: | ||||
| try: | try: | ||||
| dfh = self._datafp(b"r+") | dfh = self._datafp(b"r+") | ||||
| if self._docket is None: | |||||
| dfh.seek(0, os.SEEK_END) | dfh.seek(0, os.SEEK_END) | ||||
| else: | |||||
| dfh.seek(self._docket.data_end, os.SEEK_SET) | |||||
| except IOError as inst: | except IOError as inst: | ||||
| if inst.errno != errno.ENOENT: | if inst.errno != errno.ENOENT: | ||||
| raise | raise | ||||
| dfh = self._datafp(b"w+") | dfh = self._datafp(b"w+") | ||||
| transaction.add(self._datafile, dsize) | transaction.add(self._datafile, dsize) | ||||
| try: | try: | ||||
| isize = r * self.index.entry_size | isize = r * self.index.entry_size | ||||
| ifh = self.__index_write_fp() | ifh = self.__index_write_fp() | ||||
| """Returns the current offset in the (in-transaction) data file. | """Returns the current offset in the (in-transaction) data file. | ||||
| Versions < 2 of the revlog can get this 0(1), revlog v2 needs a docket | Versions < 2 of the revlog can get this 0(1), revlog v2 needs a docket | ||||
| file to store that information: since sidedata can be rewritten to the | file to store that information: since sidedata can be rewritten to the | ||||
| end of the data file within a transaction, you can have cases where, for | end of the data file within a transaction, you can have cases where, for | ||||
| example, rev `n` does not have sidedata while rev `n - 1` does, leading | example, rev `n` does not have sidedata while rev `n - 1` does, leading | ||||
| to `n - 1`'s sidedata being written after `n`'s data. | to `n - 1`'s sidedata being written after `n`'s data. | ||||
| TODO cache this in a docket file before getting out of experimental.""" | TODO cache this in a docket file before getting out of experimental.""" | ||||
| if self._format_version != REVLOGV2: | if self._docket is None: | ||||
| return self.end(prev) | return self.end(prev) | ||||
| else: | |||||
| offset = 0 | return self._docket.data_end | ||||
| for rev, entry in enumerate(self.index): | |||||
| sidedata_end = entry[8] + entry[9] | |||||
| # Sidedata for a previous rev has potentially been written after | |||||
| # this rev's end, so take the max. | |||||
| offset = max(self.end(rev), offset, sidedata_end) | |||||
| return offset | |||||
| def _writeentry(self, transaction, entry, data, link, offset, sidedata): | def _writeentry(self, transaction, entry, data, link, offset, sidedata): | ||||
| # Files opened in a+ mode have inconsistent behavior on various | # Files opened in a+ mode have inconsistent behavior on various | ||||
| # platforms. Windows requires that a file positioning call be made | # platforms. Windows requires that a file positioning call be made | ||||
| # when the file handle transitions between reads and writes. See | # when the file handle transitions between reads and writes. See | ||||
| # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other | # 3686fa2b8eee and the mixedfilemodewrapper in windows.py. On other | ||||
| # platforms, Python or the platform itself can be buggy. Some versions | # platforms, Python or the platform itself can be buggy. Some versions | ||||
| # of Solaris have been observed to not append at the end of the file | # of Solaris have been observed to not append at the end of the file | ||||
| # if the file was seeked to before the end. See issue4943 for more. | # if the file was seeked to before the end. See issue4943 for more. | ||||
| # | # | ||||
| # We work around this issue by inserting a seek() before writing. | # We work around this issue by inserting a seek() before writing. | ||||
| # Note: This is likely not necessary on Python 3. However, because | # Note: This is likely not necessary on Python 3. However, because | ||||
| # the file handle is reused for reads and may be seeked there, we need | # the file handle is reused for reads and may be seeked there, we need | ||||
| # to be careful before changing this. | # to be careful before changing this. | ||||
| if self._writinghandles is None: | if self._writinghandles is None: | ||||
| msg = b'adding revision outside `revlog._writing` context' | msg = b'adding revision outside `revlog._writing` context' | ||||
| raise error.ProgrammingError(msg) | raise error.ProgrammingError(msg) | ||||
| ifh, dfh = self._writinghandles | ifh, dfh = self._writinghandles | ||||
| if self._docket is None: | if self._docket is None: | ||||
| ifh.seek(0, os.SEEK_END) | ifh.seek(0, os.SEEK_END) | ||||
| else: | else: | ||||
| ifh.seek(self._docket.index_end, os.SEEK_SET) | ifh.seek(self._docket.index_end, os.SEEK_SET) | ||||
| if dfh: | if dfh: | ||||
| if self._docket is None: | |||||
| dfh.seek(0, os.SEEK_END) | dfh.seek(0, os.SEEK_END) | ||||
| else: | |||||
| dfh.seek(self._docket.data_end, os.SEEK_SET) | |||||
| curr = len(self) - 1 | curr = len(self) - 1 | ||||
| if not self._inline: | if not self._inline: | ||||
| transaction.add(self._datafile, offset) | transaction.add(self._datafile, offset) | ||||
| transaction.add(self._indexfile, curr * len(entry)) | transaction.add(self._indexfile, curr * len(entry)) | ||||
| if data[0]: | if data[0]: | ||||
| dfh.write(data[0]) | dfh.write(data[0]) | ||||
| dfh.write(data[1]) | dfh.write(data[1]) | ||||
| if sidedata: | if sidedata: | ||||
| dfh.write(sidedata) | dfh.write(sidedata) | ||||
| ifh.write(entry) | ifh.write(entry) | ||||
| else: | else: | ||||
| offset += curr * self.index.entry_size | offset += curr * self.index.entry_size | ||||
| transaction.add(self._indexfile, offset) | transaction.add(self._indexfile, offset) | ||||
| ifh.write(entry) | ifh.write(entry) | ||||
| ifh.write(data[0]) | ifh.write(data[0]) | ||||
| ifh.write(data[1]) | ifh.write(data[1]) | ||||
| if sidedata: | if sidedata: | ||||
| ifh.write(sidedata) | ifh.write(sidedata) | ||||
| self._enforceinlinesize(transaction) | self._enforceinlinesize(transaction) | ||||
| if self._docket is not None: | if self._docket is not None: | ||||
| self._docket.index_end = self._writinghandles[0].tell() | self._docket.index_end = self._writinghandles[0].tell() | ||||
| self._docket.data_end = self._writinghandles[1].tell() | |||||
| nodemaputil.setup_persistent_nodemap(transaction, self) | nodemaputil.setup_persistent_nodemap(transaction, self) | ||||
| def addgroup( | def addgroup( | ||||
| self, | self, | ||||
| deltas, | deltas, | ||||
| linkmapper, | linkmapper, | ||||
| transaction, | transaction, | ||||
| if len(self) == 0: | if len(self) == 0: | ||||
| return | return | ||||
| rev, _ = self.getstrippoint(minlink) | rev, _ = self.getstrippoint(minlink) | ||||
| if rev == len(self): | if rev == len(self): | ||||
| return | return | ||||
| # first truncate the files on disk | # first truncate the files on disk | ||||
| end = self.start(rev) | data_end = self.start(rev) | ||||
| if not self._inline: | if not self._inline: | ||||
| transaction.add(self._datafile, end) | transaction.add(self._datafile, data_end) | ||||
| end = rev * self.index.entry_size | end = rev * self.index.entry_size | ||||
| else: | else: | ||||
| end += rev * self.index.entry_size | end = data_end + (rev * self.index.entry_size) | ||||
| transaction.add(self._indexfile, end) | transaction.add(self._indexfile, end) | ||||
| if self._docket is not None: | if self._docket is not None: | ||||
| # XXX we could, leverage the docket while stripping. However it is | # XXX we could, leverage the docket while stripping. However it is | ||||
| # not powerfull enough at the time of this comment | # not powerfull enough at the time of this comment | ||||
| self._docket.index_end = end | self._docket.index_end = end | ||||
| self._docket.data_end = data_end | |||||
| self._docket.write(transaction, stripping=True) | self._docket.write(transaction, stripping=True) | ||||
| # then reset internal state in memory to forget those revisions | # then reset internal state in memory to forget those revisions | ||||
| self._revisioncache = None | self._revisioncache = None | ||||
| self._chaininfocache = util.lrucachedict(500) | self._chaininfocache = util.lrucachedict(500) | ||||
| self._chunkclear() | self._chunkclear() | ||||
| del self.index[rev:-1] | del self.index[rev:-1] | ||||
| if not helpers[1] and not helpers[2]: | if not helpers[1] and not helpers[2]: | ||||
| # Nothing to generate or remove | # Nothing to generate or remove | ||||
| return | return | ||||
| new_entries = [] | new_entries = [] | ||||
| # append the new sidedata | # append the new sidedata | ||||
| with self._writing(transaction): | with self._writing(transaction): | ||||
| ifh, dfh = self._writinghandles | ifh, dfh = self._writinghandles | ||||
| if self._docket is not None: | |||||
| dfh.seek(self._docket.data_end, os.SEEK_SET) | |||||
| else: | |||||
| dfh.seek(0, os.SEEK_END) | dfh.seek(0, os.SEEK_END) | ||||
| current_offset = dfh.tell() | current_offset = dfh.tell() | ||||
| for rev in range(startrev, endrev + 1): | for rev in range(startrev, endrev + 1): | ||||
| entry = self.index[rev] | entry = self.index[rev] | ||||
| new_sidedata, flags = sidedatautil.run_sidedata_helpers( | new_sidedata, flags = sidedatautil.run_sidedata_helpers( | ||||
| store=self, | store=self, | ||||
| sidedata_helpers=helpers, | sidedata_helpers=helpers, | ||||
| sidedata={}, | sidedata={}, | ||||
| rev=rev, | rev=rev, | ||||
| entry = (new_offset_flags,) + entry[1:8] | entry = (new_offset_flags,) + entry[1:8] | ||||
| entry += (current_offset, len(serialized_sidedata)) | entry += (current_offset, len(serialized_sidedata)) | ||||
| # the sidedata computation might have move the file cursors around | # the sidedata computation might have move the file cursors around | ||||
| dfh.seek(current_offset, os.SEEK_SET) | dfh.seek(current_offset, os.SEEK_SET) | ||||
| dfh.write(serialized_sidedata) | dfh.write(serialized_sidedata) | ||||
| new_entries.append(entry) | new_entries.append(entry) | ||||
| current_offset += len(serialized_sidedata) | current_offset += len(serialized_sidedata) | ||||
| if self._docket is not None: | |||||
| self._docket.data_end = dfh.tell() | |||||
| # rewrite the new index entries | # rewrite the new index entries | ||||
| ifh.seek(startrev * self.index.entry_size) | ifh.seek(startrev * self.index.entry_size) | ||||
| for i, e in enumerate(new_entries): | for i, e in enumerate(new_entries): | ||||
| rev = startrev + i | rev = startrev + i | ||||
| self.index.replace_sidedata_info(rev, e[8], e[9], e[0]) | self.index.replace_sidedata_info(rev, e[8], e[9], e[0]) | ||||
| packed = self.index.entry_binary(rev) | packed = self.index.entry_binary(rev) | ||||
| if rev == 0 and self._docket is None: | if rev == 0 and self._docket is None: | ||||
| header = self._format_flags | self._format_version | header = self._format_flags | self._format_version | ||||
| header = self.index.pack_header(header) | header = self.index.pack_header(header) | ||||
| packed = header + packed | packed = header + packed | ||||
| ifh.write(packed) | ifh.write(packed) | ||||
| constants, | constants, | ||||
| ) | ) | ||||
| # Docket format | # Docket format | ||||
| # | # | ||||
| # * 4 bytes: revlog version | # * 4 bytes: revlog version | ||||
| # | This is mandatory as docket must be compatible with the previous | # | This is mandatory as docket must be compatible with the previous | ||||
| # | revlog index header. | # | revlog index header. | ||||
| # * 8 bytes: size of index data | # * 8 bytes: size of index-data | ||||
| # * 8 bytes: pending size of index data | # * 8 bytes: pending size of index-data | ||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LL') | # * 8 bytes: size of data | ||||
| # * 8 bytes: pending size of data | |||||
| S_HEADER = struct.Struct(constants.INDEX_HEADER.format + 'LLLL') | |||||
| class RevlogDocket(object): | class RevlogDocket(object): | ||||
| """metadata associated with revlog""" | """metadata associated with revlog""" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| revlog, | revlog, | ||||
| use_pending=False, | use_pending=False, | ||||
| version_header=None, | version_header=None, | ||||
| index_end=0, | index_end=0, | ||||
| pending_index_end=0, | pending_index_end=0, | ||||
| data_end=0, | |||||
| pending_data_end=0, | |||||
| ): | ): | ||||
| self._version_header = version_header | self._version_header = version_header | ||||
| self._read_only = bool(use_pending) | self._read_only = bool(use_pending) | ||||
| self._dirty = False | self._dirty = False | ||||
| self._radix = revlog.radix | self._radix = revlog.radix | ||||
| self._path = revlog._docket_file | self._path = revlog._docket_file | ||||
| self._opener = revlog.opener | self._opener = revlog.opener | ||||
| # this assert should be True as long as we have a single index filename | # thes asserts should be True as long as we have a single index filename | ||||
| assert index_end <= pending_index_end | assert index_end <= pending_index_end | ||||
| assert data_end <= pending_data_end | |||||
| self._initial_index_end = index_end | self._initial_index_end = index_end | ||||
| self._pending_index_end = pending_index_end | self._pending_index_end = pending_index_end | ||||
| self._initial_data_end = data_end | |||||
| self._pending_data_end = pending_data_end | |||||
| if use_pending: | if use_pending: | ||||
| self._index_end = self._pending_index_end | self._index_end = self._pending_index_end | ||||
| self._data_end = self._pending_data_end | |||||
| else: | else: | ||||
| self._index_end = self._initial_index_end | self._index_end = self._initial_index_end | ||||
| self._data_end = self._initial_data_end | |||||
| def index_filepath(self): | def index_filepath(self): | ||||
| """file path to the current index file associated to this docket""" | """file path to the current index file associated to this docket""" | ||||
| # very simplistic version at first | # very simplistic version at first | ||||
| return b"%s.idx" % self._radix | return b"%s.idx" % self._radix | ||||
| @property | @property | ||||
| def index_end(self): | def index_end(self): | ||||
| return self._index_end | return self._index_end | ||||
| @index_end.setter | @index_end.setter | ||||
| def index_end(self, new_size): | def index_end(self, new_size): | ||||
| if new_size != self._index_end: | if new_size != self._index_end: | ||||
| self._index_end = new_size | self._index_end = new_size | ||||
| self._dirty = True | self._dirty = True | ||||
| @property | |||||
| def data_end(self): | |||||
| return self._data_end | |||||
| @data_end.setter | |||||
| def data_end(self, new_size): | |||||
| if new_size != self._data_end: | |||||
| self._data_end = new_size | |||||
| self._dirty = True | |||||
| def write(self, transaction, pending=False, stripping=False): | def write(self, transaction, pending=False, stripping=False): | ||||
| """write the modification of disk if any | """write the modification of disk if any | ||||
| This make the new content visible to all process""" | This make the new content visible to all process""" | ||||
| if not self._dirty: | if not self._dirty: | ||||
| return False | return False | ||||
| else: | else: | ||||
| if self._read_only: | if self._read_only: | ||||
| msg = b'writing read-only docket: %s' | msg = b'writing read-only docket: %s' | ||||
| msg %= self._path | msg %= self._path | ||||
| raise error.ProgrammingError(msg) | raise error.ProgrammingError(msg) | ||||
| if not stripping: | if not stripping: | ||||
| # XXX we could, leverage the docket while stripping. However it | # XXX we could, leverage the docket while stripping. However it | ||||
| # is not powerfull enough at the time of this comment | # is not powerfull enough at the time of this comment | ||||
| transaction.addbackup(self._path, location=b'store') | transaction.addbackup(self._path, location=b'store') | ||||
| with self._opener(self._path, mode=b'w', atomictemp=True) as f: | with self._opener(self._path, mode=b'w', atomictemp=True) as f: | ||||
| f.write(self._serialize(pending=pending)) | f.write(self._serialize(pending=pending)) | ||||
| # if pending we still need to the write final data eventually | # if pending we still need to the write final data eventually | ||||
| self._dirty = pending | self._dirty = pending | ||||
| return True | return True | ||||
| def _serialize(self, pending=False): | def _serialize(self, pending=False): | ||||
| if pending: | if pending: | ||||
| official_index_end = self._initial_index_end | official_index_end = self._initial_index_end | ||||
| official_data_end = self._initial_data_end | |||||
| else: | else: | ||||
| official_index_end = self._index_end | official_index_end = self._index_end | ||||
| official_data_end = self._data_end | |||||
| # this assert should be True as long as we have a single index filename | # this assert should be True as long as we have a single index filename | ||||
| assert official_index_end <= self._index_end | assert official_data_end <= self._data_end | ||||
| data = ( | data = ( | ||||
| self._version_header, | self._version_header, | ||||
| official_index_end, | official_index_end, | ||||
| self._index_end, | self._index_end, | ||||
| official_data_end, | |||||
| self._data_end, | |||||
| ) | ) | ||||
| return S_HEADER.pack(*data) | return S_HEADER.pack(*data) | ||||
| def default_docket(revlog, version_header): | def default_docket(revlog, version_header): | ||||
| """given a revlog version a new docket object for the given revlog""" | """given a revlog version a new docket object for the given revlog""" | ||||
| if (version_header & 0xFFFF) != constants.REVLOGV2: | if (version_header & 0xFFFF) != constants.REVLOGV2: | ||||
| return None | return None | ||||
| docket = RevlogDocket(revlog, version_header=version_header) | docket = RevlogDocket(revlog, version_header=version_header) | ||||
| docket._dirty = True | docket._dirty = True | ||||
| return docket | return docket | ||||
| def parse_docket(revlog, data, use_pending=False): | def parse_docket(revlog, data, use_pending=False): | ||||
| """given some docket data return a docket object for the given revlog""" | """given some docket data return a docket object for the given revlog""" | ||||
| header = S_HEADER.unpack(data[: S_HEADER.size]) | header = S_HEADER.unpack(data[: S_HEADER.size]) | ||||
| version_header, index_size, pending_index_size = header | version_header = header[0] | ||||
| index_size = header[1] | |||||
| pending_index_size = header[2] | |||||
| data_size = header[3] | |||||
| pending_data_size = header[4] | |||||
| docket = RevlogDocket( | docket = RevlogDocket( | ||||
| revlog, | revlog, | ||||
| use_pending=use_pending, | use_pending=use_pending, | ||||
| version_header=version_header, | version_header=version_header, | ||||
| index_end=index_size, | index_end=index_size, | ||||
| pending_index_end=pending_index_size, | pending_index_end=pending_index_size, | ||||
| data_end=data_size, | |||||
| pending_data_end=pending_data_size, | |||||
| ) | ) | ||||
| return docket | return docket | ||||