| indygreg | |
| Alphare |
| hg-reviewers |
The previous changeset extracted this caching logic from the revlog class into
a new class. Adding a new instance of that class allows using the same logic
for side data.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | mercurial/changelog.py (3 lines) | |||
| M | mercurial/revlog.py (51 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | SimonSapin | ||
| Closed | SimonSapin |
| self._realopener.unlink(self._indexfile + b'.a') | self._realopener.unlink(self._indexfile + b'.a') | ||||
| self.opener = _divertopener(self._realopener, self._indexfile) | self.opener = _divertopener(self._realopener, self._indexfile) | ||||
| else: | else: | ||||
| self._delaybuf = [] | self._delaybuf = [] | ||||
| self.opener = _delayopener( | self.opener = _delayopener( | ||||
| self._realopener, self._indexfile, self._delaybuf | self._realopener, self._indexfile, self._delaybuf | ||||
| ) | ) | ||||
| self._segmentfile.opener = self.opener | self._segmentfile.opener = self.opener | ||||
| self._segmentfile_sidedata.opener = self.opener | |||||
| self._delayed = True | self._delayed = True | ||||
| tr.addpending(b'cl-%i' % id(self), self._writepending) | tr.addpending(b'cl-%i' % id(self), self._writepending) | ||||
| tr.addfinalize(b'cl-%i' % id(self), self._finalize) | tr.addfinalize(b'cl-%i' % id(self), self._finalize) | ||||
| def _finalize(self, tr): | def _finalize(self, tr): | ||||
| """finalize index updates""" | """finalize index updates""" | ||||
| self._delayed = False | self._delayed = False | ||||
| self.opener = self._realopener | self.opener = self._realopener | ||||
| self._segmentfile.opener = self.opener | self._segmentfile.opener = self.opener | ||||
| self._segmentfile_sidedata.opener = self.opener | |||||
| # move redirected index data back into place | # move redirected index data back into place | ||||
| if self._docket is not None: | if self._docket is not None: | ||||
| self._write_docket(tr) | self._write_docket(tr) | ||||
| elif self._divert: | elif self._divert: | ||||
| assert not self._delaybuf | assert not self._delaybuf | ||||
| tmpname = self._indexfile + b".a" | tmpname = self._indexfile + b".a" | ||||
| nfile = self.opener.open(tmpname) | nfile = self.opener.open(tmpname) | ||||
| nfile.close() | nfile.close() | ||||
| # add pending data | # add pending data | ||||
| fp2.write(b"".join(self._delaybuf)) | fp2.write(b"".join(self._delaybuf)) | ||||
| fp2.close() | fp2.close() | ||||
| # switch modes so finalize can simply rename | # switch modes so finalize can simply rename | ||||
| self._delaybuf = None | self._delaybuf = None | ||||
| self._divert = True | self._divert = True | ||||
| self.opener = _divertopener(self._realopener, self._indexfile) | self.opener = _divertopener(self._realopener, self._indexfile) | ||||
| self._segmentfile.opener = self.opener | self._segmentfile.opener = self.opener | ||||
| self._segmentfile_sidedata.opener = self.opener | |||||
| if self._divert: | if self._divert: | ||||
| return True | return True | ||||
| return False | return False | ||||
| def _enforceinlinesize(self, tr): | def _enforceinlinesize(self, tr): | ||||
| if not self._delayed: | if not self._delayed: | ||||
| ) | ) | ||||
| self.index = index | self.index = index | ||||
| self._segmentfile = randomaccessfile.randomaccessfile( | self._segmentfile = randomaccessfile.randomaccessfile( | ||||
| self.opener, | self.opener, | ||||
| (self._indexfile if self._inline else self._datafile), | (self._indexfile if self._inline else self._datafile), | ||||
| self._chunkcachesize, | self._chunkcachesize, | ||||
| chunkcache, | chunkcache, | ||||
| ) | ) | ||||
| self._segmentfile_sidedata = randomaccessfile.randomaccessfile( | |||||
| self.opener, | |||||
| self._sidedatafile, | |||||
| self._chunkcachesize, | |||||
| ) | |||||
| # revnum -> (chain-length, sum-delta-length) | # revnum -> (chain-length, sum-delta-length) | ||||
| self._chaininfocache = util.lrucachedict(500) | self._chaininfocache = util.lrucachedict(500) | ||||
| # revlog header -> revlog compressor | # revlog header -> revlog compressor | ||||
| self._decompressors = {} | self._decompressors = {} | ||||
| @util.propertycache | @util.propertycache | ||||
| def revlog_kind(self): | def revlog_kind(self): | ||||
| return self.target[0] | return self.target[0] | ||||
| nodemaputil.update_persistent_nodemap(self) | nodemaputil.update_persistent_nodemap(self) | ||||
| else: | else: | ||||
| nodemaputil.setup_persistent_nodemap(transaction, self) | nodemaputil.setup_persistent_nodemap(transaction, self) | ||||
| def clearcaches(self): | def clearcaches(self): | ||||
| self._revisioncache = None | self._revisioncache = None | ||||
| self._chainbasecache.clear() | self._chainbasecache.clear() | ||||
| self._segmentfile.clear_cache() | self._segmentfile.clear_cache() | ||||
| self._segmentfile_sidedata.clear_cache() | |||||
| self._pcache = {} | self._pcache = {} | ||||
| self._nodemap_docket = None | self._nodemap_docket = None | ||||
| self.index.clearcaches() | self.index.clearcaches() | ||||
| # The python code is the one responsible for validating the docket, we | # The python code is the one responsible for validating the docket, we | ||||
| # end up having to refresh it here. | # end up having to refresh it here. | ||||
| use_nodemap = ( | use_nodemap = ( | ||||
| not self._inline | not self._inline | ||||
| and self._nodemap_file is not None | and self._nodemap_file is not None | ||||
| sidedata_offset = index_entry[8] | sidedata_offset = index_entry[8] | ||||
| sidedata_size = index_entry[9] | sidedata_size = index_entry[9] | ||||
| if self._inline: | if self._inline: | ||||
| sidedata_offset += self.index.entry_size * (1 + rev) | sidedata_offset += self.index.entry_size * (1 + rev) | ||||
| if sidedata_size == 0: | if sidedata_size == 0: | ||||
| return {} | return {} | ||||
| # XXX this need caching, as we do for data | |||||
| with self._sidedatareadfp() as sdf: | |||||
| if self._docket.sidedata_end < sidedata_offset + sidedata_size: | if self._docket.sidedata_end < sidedata_offset + sidedata_size: | ||||
| filename = self._sidedatafile | filename = self._sidedatafile | ||||
| end = self._docket.sidedata_end | end = self._docket.sidedata_end | ||||
| offset = sidedata_offset | offset = sidedata_offset | ||||
| length = sidedata_size | length = sidedata_size | ||||
| m = FILE_TOO_SHORT_MSG % (filename, length, offset, end) | m = FILE_TOO_SHORT_MSG % (filename, length, offset, end) | ||||
| raise error.RevlogError(m) | raise error.RevlogError(m) | ||||
| sdf.seek(sidedata_offset, os.SEEK_SET) | comp_segment = self._segmentfile_sidedata.read_chunk( | ||||
| comp_segment = sdf.read(sidedata_size) | sidedata_offset, sidedata_size | ||||
| if len(comp_segment) < sidedata_size: | |||||
| filename = self._sidedatafile | |||||
| length = sidedata_size | |||||
| offset = sidedata_offset | |||||
| got = len(comp_segment) | |||||
| m = randomaccessfile.PARTIAL_READ_MSG % ( | |||||
| filename, | |||||
| length, | |||||
| offset, | |||||
| got, | |||||
| ) | ) | ||||
| raise error.RevlogError(m) | |||||
| comp = self.index[rev][11] | comp = self.index[rev][11] | ||||
| if comp == COMP_MODE_PLAIN: | if comp == COMP_MODE_PLAIN: | ||||
| segment = comp_segment | segment = comp_segment | ||||
| elif comp == COMP_MODE_DEFAULT: | elif comp == COMP_MODE_DEFAULT: | ||||
| segment = self._decompressor(comp_segment) | segment = self._decompressor(comp_segment) | ||||
| elif comp == COMP_MODE_INLINE: | elif comp == COMP_MODE_INLINE: | ||||
| segment = self.decompress(comp_segment) | segment = self.decompress(comp_segment) | ||||
| existing_handles = True | existing_handles = True | ||||
| fp = self._writinghandles[0] | fp = self._writinghandles[0] | ||||
| fp.flush() | fp.flush() | ||||
| fp.close() | fp.close() | ||||
| # We can't use the cached file handle after close(). So prevent | # We can't use the cached file handle after close(). So prevent | ||||
| # its usage. | # its usage. | ||||
| self._writinghandles = None | self._writinghandles = None | ||||
| self._segmentfile.writing_handle = None | self._segmentfile.writing_handle = None | ||||
| # No need to deal with sidedata writing handle as it is only | |||||
| # relevant with revlog-v2 which is never inline, not reaching | |||||
| # this code | |||||
| new_dfh = self._datafp(b'w+') | new_dfh = self._datafp(b'w+') | ||||
| new_dfh.truncate(0) # drop any potentially existing data | new_dfh.truncate(0) # drop any potentially existing data | ||||
| try: | try: | ||||
| with self._indexfp() as read_ifh: | with self._indexfp() as read_ifh: | ||||
| for r in self: | for r in self: | ||||
| new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1]) | new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1]) | ||||
| if troffset <= self.start(r) + r * self.index.entry_size: | if troffset <= self.start(r) + r * self.index.entry_size: | ||||
| ) | ) | ||||
| if existing_handles: | if existing_handles: | ||||
| # switched from inline to conventional reopen the index | # switched from inline to conventional reopen the index | ||||
| ifh = self.__index_write_fp() | ifh = self.__index_write_fp() | ||||
| self._writinghandles = (ifh, new_dfh, None) | self._writinghandles = (ifh, new_dfh, None) | ||||
| self._segmentfile.writing_handle = new_dfh | self._segmentfile.writing_handle = new_dfh | ||||
| new_dfh = None | new_dfh = None | ||||
| # No need to deal with sidedata writing handle as it is only | |||||
| # relevant with revlog-v2 which is never inline, not reaching | |||||
| # this code | |||||
| finally: | finally: | ||||
| if new_dfh is not None: | if new_dfh is not None: | ||||
| new_dfh.close() | new_dfh.close() | ||||
| def _nodeduplicatecallback(self, transaction, node): | def _nodeduplicatecallback(self, transaction, node): | ||||
| """called when trying to add a node already stored.""" | """called when trying to add a node already stored.""" | ||||
| @contextlib.contextmanager | @contextlib.contextmanager | ||||
| ifh = self.__index_write_fp() | ifh = self.__index_write_fp() | ||||
| if self._inline: | if self._inline: | ||||
| transaction.add(self._indexfile, dsize + isize) | transaction.add(self._indexfile, dsize + isize) | ||||
| else: | else: | ||||
| transaction.add(self._indexfile, isize) | transaction.add(self._indexfile, isize) | ||||
| # exposing all file handle for writing. | # exposing all file handle for writing. | ||||
| self._writinghandles = (ifh, dfh, sdfh) | self._writinghandles = (ifh, dfh, sdfh) | ||||
| self._segmentfile.writing_handle = ifh if self._inline else dfh | self._segmentfile.writing_handle = ifh if self._inline else dfh | ||||
| self._segmentfile_sidedata.writing_handle = sdfh | |||||
| yield | yield | ||||
| if self._docket is not None: | if self._docket is not None: | ||||
| self._write_docket(transaction) | self._write_docket(transaction) | ||||
| finally: | finally: | ||||
| self._writinghandles = None | self._writinghandles = None | ||||
| self._segmentfile.writing_handle = None | self._segmentfile.writing_handle = None | ||||
| self._segmentfile_sidedata.writing_handle = None | |||||
| if dfh is not None: | if dfh is not None: | ||||
| dfh.close() | dfh.close() | ||||
| if sdfh is not None: | if sdfh is not None: | ||||
| sdfh.close() | sdfh.close() | ||||
| # closing the index file last to avoid exposing referent to | # closing the index file last to avoid exposing referent to | ||||
| # potential unflushed data content. | # potential unflushed data content. | ||||
| if ifh is not None: | if ifh is not None: | ||||
| ifh.close() | ifh.close() | ||||
| self._docket.data_end = data_end | self._docket.data_end = data_end | ||||
| self._docket.sidedata_end = sidedata_end | self._docket.sidedata_end = sidedata_end | ||||
| self._docket.write(transaction, stripping=True) | self._docket.write(transaction, stripping=True) | ||||
| # then reset internal state in memory to forget those revisions | # then reset internal state in memory to forget those revisions | ||||
| self._revisioncache = None | self._revisioncache = None | ||||
| self._chaininfocache = util.lrucachedict(500) | self._chaininfocache = util.lrucachedict(500) | ||||
| self._segmentfile.clear_cache() | self._segmentfile.clear_cache() | ||||
| self._segmentfile_sidedata.clear_cache() | |||||
| del self.index[rev:-1] | del self.index[rev:-1] | ||||
| def checksize(self): | def checksize(self): | ||||
| """Check size of index and data files | """Check size of index and data files | ||||
| return a (dd, di) tuple. | return a (dd, di) tuple. | ||||
| - dd: extra bytes for the "data" file | - dd: extra bytes for the "data" file | ||||