| indygreg | |
| Alphare |
| hg-reviewers |
Same reasoning as for indexfile and datafile, lets hide these implementation
details.
| No Linters Available |
| No Unit Test Coverage |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/revlog.py (14 lines) | |||
| M | mercurial/revlogutils/nodemap.py (22 lines) |
| Commit | Parents | Author | Summary | Date |
|---|---|---|---|---|
| 5f34b8d34eca | 2ba232f6e5e5 | Pierre-Yves David | May 3 2021, 6:25 AM |
| indexfile = b'%s.i.a' % self.radix | indexfile = b'%s.i.a' % self.radix | ||||
| datafile = b'%s.d' % self.radix | datafile = b'%s.d' % self.radix | ||||
| else: | else: | ||||
| indexfile = b'%s.i.%s' % (self.radix, postfix) | indexfile = b'%s.i.%s' % (self.radix, postfix) | ||||
| datafile = b'%s.d.%s' % (self.radix, postfix) | datafile = b'%s.d.%s' % (self.radix, postfix) | ||||
| self._indexfile = indexfile | self._indexfile = indexfile | ||||
| self._datafile = datafile | self._datafile = datafile | ||||
| self.nodemap_file = None | self._nodemap_file = None | ||||
| self.postfix = postfix | self.postfix = postfix | ||||
| self.opener = opener | self.opener = opener | ||||
| if persistentnodemap: | if persistentnodemap: | ||||
| self.nodemap_file = nodemaputil.get_nodemap_file(self) | self._nodemap_file = nodemaputil.get_nodemap_file(self) | ||||
| assert target[0] in ALL_KINDS | assert target[0] in ALL_KINDS | ||||
| assert len(target) == 2 | assert len(target) == 2 | ||||
| self.target = target | self.target = target | ||||
| # When True, indexfile is opened with checkambig=True at writing, to | # When True, indexfile is opened with checkambig=True at writing, to | ||||
| # avoid file stat ambiguity. | # avoid file stat ambiguity. | ||||
| self._checkambig = checkambig | self._checkambig = checkambig | ||||
| self._mmaplargeindex = mmaplargeindex | self._mmaplargeindex = mmaplargeindex | ||||
| # sparse-revlog can't be on without general-delta (issue6056) | # sparse-revlog can't be on without general-delta (issue6056) | ||||
| if not self._generaldelta: | if not self._generaldelta: | ||||
| self._sparserevlog = False | self._sparserevlog = False | ||||
| self._storedeltachains = True | self._storedeltachains = True | ||||
| devel_nodemap = ( | devel_nodemap = ( | ||||
| self.nodemap_file | self._nodemap_file | ||||
| and force_nodemap | and force_nodemap | ||||
| and parse_index_v1_nodemap is not None | and parse_index_v1_nodemap is not None | ||||
| ) | ) | ||||
| use_rust_index = False | use_rust_index = False | ||||
| if rustrevlog is not None: | if rustrevlog is not None: | ||||
| if self.nodemap_file is not None: | if self._nodemap_file is not None: | ||||
| use_rust_index = True | use_rust_index = True | ||||
| else: | else: | ||||
| use_rust_index = self.opener.options.get(b'rust.index') | use_rust_index = self.opener.options.get(b'rust.index') | ||||
| self._parse_index = parse_index_v1 | self._parse_index = parse_index_v1 | ||||
| if self._format_version == REVLOGV0: | if self._format_version == REVLOGV0: | ||||
| self._parse_index = revlogv0.parse_index_v0 | self._parse_index = revlogv0.parse_index_v0 | ||||
| elif fmt == REVLOGV2: | elif fmt == REVLOGV2: | ||||
| self._parse_index = parse_index_v2 | self._parse_index = parse_index_v2 | ||||
| elif devel_nodemap: | elif devel_nodemap: | ||||
| self._parse_index = parse_index_v1_nodemap | self._parse_index = parse_index_v1_nodemap | ||||
| elif use_rust_index: | elif use_rust_index: | ||||
| self._parse_index = parse_index_v1_mixed | self._parse_index = parse_index_v1_mixed | ||||
| try: | try: | ||||
| d = self._parse_index(indexdata, self._inline) | d = self._parse_index(indexdata, self._inline) | ||||
| index, _chunkcache = d | index, _chunkcache = d | ||||
| use_nodemap = ( | use_nodemap = ( | ||||
| not self._inline | not self._inline | ||||
| and self.nodemap_file is not None | and self._nodemap_file is not None | ||||
| and util.safehasattr(index, 'update_nodemap_data') | and util.safehasattr(index, 'update_nodemap_data') | ||||
| ) | ) | ||||
| if use_nodemap: | if use_nodemap: | ||||
| nodemap_data = nodemaputil.persisted_data(self) | nodemap_data = nodemaputil.persisted_data(self) | ||||
| if nodemap_data is not None: | if nodemap_data is not None: | ||||
| docket = nodemap_data[0] | docket = nodemap_data[0] | ||||
| if ( | if ( | ||||
| len(d[0]) > docket.tip_rev | len(d[0]) > docket.tip_rev | ||||
| # rawtext contents) and the delta could be incompatible. | # rawtext contents) and the delta could be incompatible. | ||||
| if (self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS) or ( | if (self.flags(baserev) & REVIDX_RAWTEXT_CHANGING_FLAGS) or ( | ||||
| self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS | self.flags(rev) & REVIDX_RAWTEXT_CHANGING_FLAGS | ||||
| ): | ): | ||||
| return False | return False | ||||
| return True | return True | ||||
| def update_caches(self, transaction): | def update_caches(self, transaction): | ||||
| if self.nodemap_file is not None: | if self._nodemap_file is not None: | ||||
| if transaction is None: | if transaction is None: | ||||
| nodemaputil.update_persistent_nodemap(self) | nodemaputil.update_persistent_nodemap(self) | ||||
| else: | else: | ||||
| nodemaputil.setup_persistent_nodemap(transaction, self) | nodemaputil.setup_persistent_nodemap(transaction, self) | ||||
| def clearcaches(self): | def clearcaches(self): | ||||
| self._revisioncache = None | self._revisioncache = None | ||||
| self._chainbasecache.clear() | self._chainbasecache.clear() | ||||
| self._chunkcache = (0, b'') | self._chunkcache = (0, b'') | ||||
| self._pcache = {} | self._pcache = {} | ||||
| self._nodemap_docket = None | self._nodemap_docket = None | ||||
| self.index.clearcaches() | self.index.clearcaches() | ||||
| # The python code is the one responsible for validating the docket, we | # The python code is the one responsible for validating the docket, we | ||||
| # end up having to refresh it here. | # end up having to refresh it here. | ||||
| use_nodemap = ( | use_nodemap = ( | ||||
| not self._inline | not self._inline | ||||
| and self.nodemap_file is not None | and self._nodemap_file is not None | ||||
| and util.safehasattr(self.index, 'update_nodemap_data') | and util.safehasattr(self.index, 'update_nodemap_data') | ||||
| ) | ) | ||||
| if use_nodemap: | if use_nodemap: | ||||
| nodemap_data = nodemaputil.persisted_data(self) | nodemap_data = nodemaputil.persisted_data(self) | ||||
| if nodemap_data is not None: | if nodemap_data is not None: | ||||
| self._nodemap_docket = nodemap_data[0] | self._nodemap_docket = nodemap_data[0] | ||||
| self.index.update_nodemap_data(*nodemap_data) | self.index.update_nodemap_data(*nodemap_data) | ||||
| class NodeMap(dict): | class NodeMap(dict): | ||||
| def __missing__(self, x): | def __missing__(self, x): | ||||
| raise error.RevlogError(b'unknown node: %s' % x) | raise error.RevlogError(b'unknown node: %s' % x) | ||||
| def persisted_data(revlog): | def persisted_data(revlog): | ||||
| """read the nodemap for a revlog from disk""" | """read the nodemap for a revlog from disk""" | ||||
| if revlog.nodemap_file is None: | if revlog._nodemap_file is None: | ||||
| return None | return None | ||||
| pdata = revlog.opener.tryread(revlog.nodemap_file) | pdata = revlog.opener.tryread(revlog._nodemap_file) | ||||
| if not pdata: | if not pdata: | ||||
| return None | return None | ||||
| offset = 0 | offset = 0 | ||||
| (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size]) | (version,) = S_VERSION.unpack(pdata[offset : offset + S_VERSION.size]) | ||||
| if version != ONDISK_VERSION: | if version != ONDISK_VERSION: | ||||
| return None | return None | ||||
| offset += S_VERSION.size | offset += S_VERSION.size | ||||
| headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size]) | headers = S_HEADER.unpack(pdata[offset : offset + S_HEADER.size]) | ||||
| def setup_persistent_nodemap(tr, revlog): | def setup_persistent_nodemap(tr, revlog): | ||||
| """Install whatever is needed transaction side to persist a nodemap on disk | """Install whatever is needed transaction side to persist a nodemap on disk | ||||
| (only actually persist the nodemap if this is relevant for this revlog) | (only actually persist the nodemap if this is relevant for this revlog) | ||||
| """ | """ | ||||
| if revlog._inline: | if revlog._inline: | ||||
| return # inlined revlog are too small for this to be relevant | return # inlined revlog are too small for this to be relevant | ||||
| if revlog.nodemap_file is None: | if revlog._nodemap_file is None: | ||||
| return # we do not use persistent_nodemap on this revlog | return # we do not use persistent_nodemap on this revlog | ||||
| # we need to happen after the changelog finalization, in that use "cl-" | # we need to happen after the changelog finalization, in that use "cl-" | ||||
| callback_id = b"nm-revlog-persistent-nodemap-%s" % revlog.nodemap_file | callback_id = b"nm-revlog-persistent-nodemap-%s" % revlog._nodemap_file | ||||
| if tr.hasfinalize(callback_id): | if tr.hasfinalize(callback_id): | ||||
| return # no need to register again | return # no need to register again | ||||
| tr.addpending( | tr.addpending( | ||||
| callback_id, lambda tr: persist_nodemap(tr, revlog, pending=True) | callback_id, lambda tr: persist_nodemap(tr, revlog, pending=True) | ||||
| ) | ) | ||||
| tr.addfinalize(callback_id, lambda tr: persist_nodemap(tr, revlog)) | tr.addfinalize(callback_id, lambda tr: persist_nodemap(tr, revlog)) | ||||
| def update_persistent_nodemap(revlog): | def update_persistent_nodemap(revlog): | ||||
| """update the persistent nodemap right now | """update the persistent nodemap right now | ||||
| To be used for updating the nodemap on disk outside of a normal transaction | To be used for updating the nodemap on disk outside of a normal transaction | ||||
| setup (eg, `debugupdatecache`). | setup (eg, `debugupdatecache`). | ||||
| """ | """ | ||||
| if revlog._inline: | if revlog._inline: | ||||
| return # inlined revlog are too small for this to be relevant | return # inlined revlog are too small for this to be relevant | ||||
| if revlog.nodemap_file is None: | if revlog._nodemap_file is None: | ||||
| return # we do not use persistent_nodemap on this revlog | return # we do not use persistent_nodemap on this revlog | ||||
| notr = _NoTransaction() | notr = _NoTransaction() | ||||
| persist_nodemap(notr, revlog) | persist_nodemap(notr, revlog) | ||||
| for k in sorted(notr._postclose): | for k in sorted(notr._postclose): | ||||
| notr._postclose[k](None) | notr._postclose[k](None) | ||||
| def delete_nodemap(tr, repo, revlog): | def delete_nodemap(tr, repo, revlog): | ||||
| """ Delete nodemap data on disk for a given revlog""" | """ Delete nodemap data on disk for a given revlog""" | ||||
| if revlog.nodemap_file is None: | if revlog._nodemap_file is None: | ||||
| msg = "calling persist nodemap on a revlog without the feature enabled" | msg = "calling persist nodemap on a revlog without the feature enabled" | ||||
| raise error.ProgrammingError(msg) | raise error.ProgrammingError(msg) | ||||
| repo.svfs.unlink(revlog.nodemap_file) | repo.svfs.unlink(revlog._nodemap_file) | ||||
| def persist_nodemap(tr, revlog, pending=False, force=False): | def persist_nodemap(tr, revlog, pending=False, force=False): | ||||
| """Write nodemap data on disk for a given revlog""" | """Write nodemap data on disk for a given revlog""" | ||||
| if getattr(revlog, 'filteredrevs', ()): | if getattr(revlog, 'filteredrevs', ()): | ||||
| raise error.ProgrammingError( | raise error.ProgrammingError( | ||||
| "cannot persist nodemap of a filtered changelog" | "cannot persist nodemap of a filtered changelog" | ||||
| ) | ) | ||||
| if revlog.nodemap_file is None: | if revlog._nodemap_file is None: | ||||
| if force: | if force: | ||||
| revlog.nodemap_file = get_nodemap_file(revlog) | revlog._nodemap_file = get_nodemap_file(revlog) | ||||
| else: | else: | ||||
| msg = "calling persist nodemap on a revlog without the feature enabled" | msg = "calling persist nodemap on a revlog without the feature enabled" | ||||
| raise error.ProgrammingError(msg) | raise error.ProgrammingError(msg) | ||||
| can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental") | can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental") | ||||
| ondisk_docket = revlog._nodemap_docket | ondisk_docket = revlog._nodemap_docket | ||||
| feed_data = util.safehasattr(revlog.index, "update_nodemap_data") | feed_data = util.safehasattr(revlog.index, "update_nodemap_data") | ||||
| use_mmap = revlog.opener.options.get(b"persistent-nodemap.mmap") | use_mmap = revlog.opener.options.get(b"persistent-nodemap.mmap") | ||||
| else: | else: | ||||
| fd.flush() | fd.flush() | ||||
| new_data = util.buffer(util.mmapread(fd, len(data))) | new_data = util.buffer(util.mmapread(fd, len(data))) | ||||
| target_docket.data_length = len(data) | target_docket.data_length = len(data) | ||||
| target_docket.tip_rev = revlog.tiprev() | target_docket.tip_rev = revlog.tiprev() | ||||
| target_docket.tip_node = revlog.node(target_docket.tip_rev) | target_docket.tip_node = revlog.node(target_docket.tip_rev) | ||||
| # EXP-TODO: if this is a cache, this should use a cache vfs, not a | # EXP-TODO: if this is a cache, this should use a cache vfs, not a | ||||
| # store vfs | # store vfs | ||||
| file_path = revlog.nodemap_file | file_path = revlog._nodemap_file | ||||
| if pending: | if pending: | ||||
| file_path += b'.a' | file_path += b'.a' | ||||
| tr.registertmp(file_path) | tr.registertmp(file_path) | ||||
| else: | else: | ||||
| tr.addbackup(file_path) | tr.addbackup(file_path) | ||||
| with revlog.opener(file_path, b'w', atomictemp=True) as fp: | with revlog.opener(file_path, b'w', atomictemp=True) as fp: | ||||
| fp.write(target_docket.serialize()) | fp.write(target_docket.serialize()) | ||||
| revlog._nodemap_docket = target_docket | revlog._nodemap_docket = target_docket | ||||
| if feed_data: | if feed_data: | ||||
| revlog.index.update_nodemap_data(target_docket, new_data) | revlog.index.update_nodemap_data(target_docket, new_data) | ||||
| # search for old index file in all cases, some older process might have | # search for old index file in all cases, some older process might have | ||||
| # left one behind. | # left one behind. | ||||
| olds = _other_rawdata_filepath(revlog, target_docket) | olds = _other_rawdata_filepath(revlog, target_docket) | ||||
| if olds: | if olds: | ||||
| realvfs = getattr(revlog, '_realopener', revlog.opener) | realvfs = getattr(revlog, '_realopener', revlog.opener) | ||||
| def cleanup(tr): | def cleanup(tr): | ||||
| for oldfile in olds: | for oldfile in olds: | ||||
| realvfs.tryunlink(oldfile) | realvfs.tryunlink(oldfile) | ||||
| callback_id = b"revlog-cleanup-nodemap-%s" % revlog.nodemap_file | callback_id = b"revlog-cleanup-nodemap-%s" % revlog._nodemap_file | ||||
| tr.addpostclose(callback_id, cleanup) | tr.addpostclose(callback_id, cleanup) | ||||
| ### Nodemap docket file | ### Nodemap docket file | ||||
| # | # | ||||
| # The nodemap data are stored on disk using 2 files: | # The nodemap data are stored on disk using 2 files: | ||||
| # | # | ||||
| # * a raw data files containing a persistent nodemap | # * a raw data files containing a persistent nodemap | ||||