| indygreg | |
| Alphare |
| hg-reviewers |
We want to make the actual location of the datafile and location more of an
implementation details than what is is currently. In that process, we make the
attribute private.
| No Linters Available |
| No Unit Test Coverage |
| Path | Packages | |||
|---|---|---|---|---|
| M | contrib/perf.py (7 lines) | |||
| M | mercurial/revlog.py (28 lines) | |||
| M | mercurial/upgrade_utils/engine.py (6 lines) | |||
| M | tests/test-contrib-perf.t (2 lines) | |||
| M | tests/test-revlog-raw.py (2 lines) |
| Commit | Parents | Author | Summary | Date |
|---|---|---|---|---|
| b1a228516ef0 | 4945557169e2 | Pierre-Yves David | May 3 2021, 6:22 AM |
| if util.safehasattr(orig, k): | if util.safehasattr(orig, k): | ||||
| revlogkwargs[k] = getattr(orig, k) | revlogkwargs[k] = getattr(orig, k) | ||||
| indexfile = getattr(orig, '_indexfile', None) | indexfile = getattr(orig, '_indexfile', None) | ||||
| if indexfile is None: | if indexfile is None: | ||||
| # compatibility with <= hg-5.8 | # compatibility with <= hg-5.8 | ||||
| indexfile = getattr(orig, 'indexfile') | indexfile = getattr(orig, 'indexfile') | ||||
| origindexpath = orig.opener.join(indexfile) | origindexpath = orig.opener.join(indexfile) | ||||
| origdatapath = orig.opener.join(orig.datafile) | |||||
| datafile = getattr(orig, '_datafile', getattr(orig, 'datafile')) | |||||
| origdatapath = orig.opener.join(datafile) | |||||
| indexname = 'revlog.i' | indexname = 'revlog.i' | ||||
| dataname = 'revlog.d' | dataname = 'revlog.d' | ||||
| tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-') | tmpdir = tempfile.mkdtemp(prefix='tmp-hgperf-') | ||||
| try: | try: | ||||
| # copy the data file in a temporary directory | # copy the data file in a temporary directory | ||||
| ui.debug('copying data in %s\n' % tmpdir) | ui.debug('copying data in %s\n' % tmpdir) | ||||
| destindexpath = os.path.join(tmpdir, 'revlog.i') | destindexpath = os.path.join(tmpdir, 'revlog.i') | ||||
| def rlfh(rl): | def rlfh(rl): | ||||
| if rl._inline: | if rl._inline: | ||||
| indexfile = getattr(rl, '_indexfile', None) | indexfile = getattr(rl, '_indexfile', None) | ||||
| if indexfile is None: | if indexfile is None: | ||||
| # compatibility with <= hg-5.8 | # compatibility with <= hg-5.8 | ||||
| indexfile = getattr(rl, 'indexfile') | indexfile = getattr(rl, 'indexfile') | ||||
| return getsvfs(repo)(indexfile) | return getsvfs(repo)(indexfile) | ||||
| else: | else: | ||||
| return getsvfs(repo)(rl.datafile) | datafile = getattr(rl, 'datafile', getattr(rl, 'datafile')) | ||||
| return getsvfs(repo)(datafile) | |||||
| def doread(): | def doread(): | ||||
| rl.clearcaches() | rl.clearcaches() | ||||
| for rev in revs: | for rev in revs: | ||||
| segmentforrevs(rev, rev) | segmentforrevs(rev, rev) | ||||
| def doreadcachedfh(): | def doreadcachedfh(): | ||||
| rl.clearcaches() | rl.clearcaches() | ||||
| ) | ) | ||||
| if datafile is None: | if datafile is None: | ||||
| datafile = indexfile[:-2] + b".d" | datafile = indexfile[:-2] + b".d" | ||||
| if postfix is not None: | if postfix is not None: | ||||
| datafile = b'%s.%s' % (datafile, postfix) | datafile = b'%s.%s' % (datafile, postfix) | ||||
| if postfix is not None: | if postfix is not None: | ||||
| indexfile = b'%s.%s' % (indexfile, postfix) | indexfile = b'%s.%s' % (indexfile, postfix) | ||||
| self._indexfile = indexfile | self._indexfile = indexfile | ||||
| self.datafile = datafile | self._datafile = datafile | ||||
| self.nodemap_file = None | self.nodemap_file = None | ||||
| self.postfix = postfix | self.postfix = postfix | ||||
| if persistentnodemap: | if persistentnodemap: | ||||
| self.nodemap_file = nodemaputil.get_nodemap_file( | self.nodemap_file = nodemaputil.get_nodemap_file( | ||||
| opener, self._indexfile | opener, self._indexfile | ||||
| ) | ) | ||||
| self.opener = opener | self.opener = opener | ||||
| if mode != b'r': | if mode != b'r': | ||||
| args['checkambig'] = self._checkambig | args['checkambig'] = self._checkambig | ||||
| if mode == b'w': | if mode == b'w': | ||||
| args['atomictemp'] = True | args['atomictemp'] = True | ||||
| return self.opener(self._indexfile, **args) | return self.opener(self._indexfile, **args) | ||||
| def _datafp(self, mode=b'r'): | def _datafp(self, mode=b'r'): | ||||
| """file object for the revlog's data file""" | """file object for the revlog's data file""" | ||||
| return self.opener(self.datafile, mode=mode) | return self.opener(self._datafile, mode=mode) | ||||
| @contextlib.contextmanager | @contextlib.contextmanager | ||||
| def _datareadfp(self, existingfp=None): | def _datareadfp(self, existingfp=None): | ||||
| """file object suitable to read data""" | """file object suitable to read data""" | ||||
| # Use explicit file handle, if given. | # Use explicit file handle, if given. | ||||
| if existingfp is not None: | if existingfp is not None: | ||||
| yield existingfp | yield existingfp | ||||
| startoffset = offset - realoffset | startoffset = offset - realoffset | ||||
| if len(d) - startoffset < length: | if len(d) - startoffset < length: | ||||
| raise error.RevlogError( | raise error.RevlogError( | ||||
| _( | _( | ||||
| b'partial read of revlog %s; expected %d bytes from ' | b'partial read of revlog %s; expected %d bytes from ' | ||||
| b'offset %d, got %d' | b'offset %d, got %d' | ||||
| ) | ) | ||||
| % ( | % ( | ||||
| self._indexfile if self._inline else self.datafile, | self._indexfile if self._inline else self._datafile, | ||||
| length, | length, | ||||
| realoffset, | realoffset, | ||||
| len(d) - startoffset, | len(d) - startoffset, | ||||
| ) | ) | ||||
| ) | ) | ||||
| return util.buffer(d, startoffset, length) | return util.buffer(d, startoffset, length) | ||||
| if len(d) < length: | if len(d) < length: | ||||
| raise error.RevlogError( | raise error.RevlogError( | ||||
| _( | _( | ||||
| b'partial read of revlog %s; expected %d bytes from offset ' | b'partial read of revlog %s; expected %d bytes from offset ' | ||||
| b'%d, got %d' | b'%d, got %d' | ||||
| ) | ) | ||||
| % ( | % ( | ||||
| self._indexfile if self._inline else self.datafile, | self._indexfile if self._inline else self._datafile, | ||||
| length, | length, | ||||
| offset, | offset, | ||||
| len(d), | len(d), | ||||
| ) | ) | ||||
| ) | ) | ||||
| return d | return d | ||||
| return | return | ||||
| troffset = tr.findoffset(self._indexfile) | troffset = tr.findoffset(self._indexfile) | ||||
| if troffset is None: | if troffset is None: | ||||
| raise error.RevlogError( | raise error.RevlogError( | ||||
| _(b"%s not found in the transaction") % self._indexfile | _(b"%s not found in the transaction") % self._indexfile | ||||
| ) | ) | ||||
| trindex = 0 | trindex = 0 | ||||
| tr.add(self.datafile, 0) | tr.add(self._datafile, 0) | ||||
| if fp: | if fp: | ||||
| fp.flush() | fp.flush() | ||||
| fp.close() | fp.close() | ||||
| # We can't use the cached file handle after close(). So prevent | # We can't use the cached file handle after close(). So prevent | ||||
| # its usage. | # its usage. | ||||
| self._writinghandles = None | self._writinghandles = None | ||||
| self._concurrencychecker( | self._concurrencychecker( | ||||
| ifh, self._indexfile, offset + curr * self.index.entry_size | ifh, self._indexfile, offset + curr * self.index.entry_size | ||||
| ) | ) | ||||
| else: | else: | ||||
| # Entries in the .i are a consistent size. | # Entries in the .i are a consistent size. | ||||
| self._concurrencychecker( | self._concurrencychecker( | ||||
| ifh, self._indexfile, curr * self.index.entry_size | ifh, self._indexfile, curr * self.index.entry_size | ||||
| ) | ) | ||||
| self._concurrencychecker(dfh, self.datafile, offset) | self._concurrencychecker(dfh, self._datafile, offset) | ||||
| p1r, p2r = self.rev(p1), self.rev(p2) | p1r, p2r = self.rev(p1), self.rev(p2) | ||||
| # full versions are inserted when the needed deltas | # full versions are inserted when the needed deltas | ||||
| # become comparable to the uncompressed text | # become comparable to the uncompressed text | ||||
| if rawtext is None: | if rawtext is None: | ||||
| # need rawtext size, before changed by flag processors, which is | # need rawtext size, before changed by flag processors, which is | ||||
| # the non-raw size. use revlog explicitly to avoid filelog's extra | # the non-raw size. use revlog explicitly to avoid filelog's extra | ||||
| # the file handle is reused for reads and may be seeked there, we need | # the file handle is reused for reads and may be seeked there, we need | ||||
| # to be careful before changing this. | # to be careful before changing this. | ||||
| ifh.seek(0, os.SEEK_END) | ifh.seek(0, os.SEEK_END) | ||||
| if dfh: | if dfh: | ||||
| dfh.seek(0, os.SEEK_END) | dfh.seek(0, os.SEEK_END) | ||||
| curr = len(self) - 1 | curr = len(self) - 1 | ||||
| if not self._inline: | if not self._inline: | ||||
| transaction.add(self.datafile, offset) | transaction.add(self._datafile, offset) | ||||
| transaction.add(self._indexfile, curr * len(entry)) | transaction.add(self._indexfile, curr * len(entry)) | ||||
| if data[0]: | if data[0]: | ||||
| dfh.write(data[0]) | dfh.write(data[0]) | ||||
| dfh.write(data[1]) | dfh.write(data[1]) | ||||
| if sidedata: | if sidedata: | ||||
| dfh.write(sidedata) | dfh.write(sidedata) | ||||
| ifh.write(entry) | ifh.write(entry) | ||||
| else: | else: | ||||
| end = self.end(r - 1) | end = self.end(r - 1) | ||||
| ifh = self._indexfp(b"a+") | ifh = self._indexfp(b"a+") | ||||
| isize = r * self.index.entry_size | isize = r * self.index.entry_size | ||||
| if self._inline: | if self._inline: | ||||
| transaction.add(self._indexfile, end + isize) | transaction.add(self._indexfile, end + isize) | ||||
| dfh = None | dfh = None | ||||
| else: | else: | ||||
| transaction.add(self._indexfile, isize) | transaction.add(self._indexfile, isize) | ||||
| transaction.add(self.datafile, end) | transaction.add(self._datafile, end) | ||||
| dfh = self._datafp(b"a+") | dfh = self._datafp(b"a+") | ||||
| def flush(): | def flush(): | ||||
| if dfh: | if dfh: | ||||
| dfh.flush() | dfh.flush() | ||||
| ifh.flush() | ifh.flush() | ||||
| self._writinghandles = (ifh, dfh) | self._writinghandles = (ifh, dfh) | ||||
| rev, _ = self.getstrippoint(minlink) | rev, _ = self.getstrippoint(minlink) | ||||
| if rev == len(self): | if rev == len(self): | ||||
| return | return | ||||
| # first truncate the files on disk | # first truncate the files on disk | ||||
| end = self.start(rev) | end = self.start(rev) | ||||
| if not self._inline: | if not self._inline: | ||||
| transaction.add(self.datafile, end) | transaction.add(self._datafile, end) | ||||
| end = rev * self.index.entry_size | end = rev * self.index.entry_size | ||||
| else: | else: | ||||
| end += rev * self.index.entry_size | end += rev * self.index.entry_size | ||||
| transaction.add(self._indexfile, end) | transaction.add(self._indexfile, end) | ||||
| # then reset internal state in memory to forget those revisions | # then reset internal state in memory to forget those revisions | ||||
| self._revisioncache = None | self._revisioncache = None | ||||
| raise | raise | ||||
| di = 0 | di = 0 | ||||
| return (dd, di) | return (dd, di) | ||||
| def files(self): | def files(self): | ||||
| res = [self._indexfile] | res = [self._indexfile] | ||||
| if not self._inline: | if not self._inline: | ||||
| res.append(self.datafile) | res.append(self._datafile) | ||||
| return res | return res | ||||
| def emitrevisions( | def emitrevisions( | ||||
| self, | self, | ||||
| nodes, | nodes, | ||||
| nodesorder=None, | nodesorder=None, | ||||
| revisiondata=False, | revisiondata=False, | ||||
| assumehaveparentrevisions=False, | assumehaveparentrevisions=False, | ||||
| ) | ) | ||||
| flags = flags | new_flags[0] & ~new_flags[1] | flags = flags | new_flags[0] & ~new_flags[1] | ||||
| ifh = destrevlog.opener( | ifh = destrevlog.opener( | ||||
| destrevlog._indexfile, b'a+', checkambig=False | destrevlog._indexfile, b'a+', checkambig=False | ||||
| ) | ) | ||||
| dfh = None | dfh = None | ||||
| if not destrevlog._inline: | if not destrevlog._inline: | ||||
| dfh = destrevlog.opener(destrevlog.datafile, b'a+') | dfh = destrevlog.opener(destrevlog._datafile, b'a+') | ||||
| try: | try: | ||||
| destrevlog._addrevision( | destrevlog._addrevision( | ||||
| node, | node, | ||||
| rawtext, | rawtext, | ||||
| tr, | tr, | ||||
| linkrev, | linkrev, | ||||
| p1, | p1, | ||||
| p2, | p2, | ||||
| rawtext = self.rawdata(rev) | rawtext = self.rawdata(rev) | ||||
| newrl.addrawrevision( | newrl.addrawrevision( | ||||
| rawtext, tr, self.linkrev(rev), p1, p2, node, self.flags(rev) | rawtext, tr, self.linkrev(rev), p1, p2, node, self.flags(rev) | ||||
| ) | ) | ||||
| tr.addbackup(self._indexfile, location=b'store') | tr.addbackup(self._indexfile, location=b'store') | ||||
| if not self._inline: | if not self._inline: | ||||
| tr.addbackup(self.datafile, location=b'store') | tr.addbackup(self._datafile, location=b'store') | ||||
| self.opener.rename(newrl._indexfile, self._indexfile) | self.opener.rename(newrl._indexfile, self._indexfile) | ||||
| if not self._inline: | if not self._inline: | ||||
| self.opener.rename(newrl.datafile, self.datafile) | self.opener.rename(newrl._datafile, self._datafile) | ||||
| self.clearcaches() | self.clearcaches() | ||||
| self._loadindex() | self._loadindex() | ||||
| def verifyintegrity(self, state): | def verifyintegrity(self, state): | ||||
| """Verifies the integrity of the revlog. | """Verifies the integrity of the revlog. | ||||
| Yields ``revlogproblem`` instances describing problems that are | Yields ``revlogproblem`` instances describing problems that are | ||||
| trackedsize=False, | trackedsize=False, | ||||
| storedsize=False, | storedsize=False, | ||||
| ): | ): | ||||
| d = {} | d = {} | ||||
| if exclusivefiles: | if exclusivefiles: | ||||
| d[b'exclusivefiles'] = [(self.opener, self._indexfile)] | d[b'exclusivefiles'] = [(self.opener, self._indexfile)] | ||||
| if not self._inline: | if not self._inline: | ||||
| d[b'exclusivefiles'].append((self.opener, self.datafile)) | d[b'exclusivefiles'].append((self.opener, self._datafile)) | ||||
| if sharedfiles: | if sharedfiles: | ||||
| d[b'sharedfiles'] = [] | d[b'sharedfiles'] = [] | ||||
| if revisionscount: | if revisionscount: | ||||
| d[b'revisionscount'] = len(self) | d[b'revisionscount'] = len(self) | ||||
| if trackedsize: | if trackedsize: | ||||
| oldrl = getattr(oldrl, '_revlog', oldrl) | oldrl = getattr(oldrl, '_revlog', oldrl) | ||||
| newrl = _revlogfrompath(destrepo, rl_type, unencodedname) | newrl = _revlogfrompath(destrepo, rl_type, unencodedname) | ||||
| newrl = getattr(newrl, '_revlog', newrl) | newrl = getattr(newrl, '_revlog', newrl) | ||||
| oldvfs = oldrl.opener | oldvfs = oldrl.opener | ||||
| newvfs = newrl.opener | newvfs = newrl.opener | ||||
| oldindex = oldvfs.join(oldrl._indexfile) | oldindex = oldvfs.join(oldrl._indexfile) | ||||
| newindex = newvfs.join(newrl._indexfile) | newindex = newvfs.join(newrl._indexfile) | ||||
| olddata = oldvfs.join(oldrl.datafile) | olddata = oldvfs.join(oldrl._datafile) | ||||
| newdata = newvfs.join(newrl.datafile) | newdata = newvfs.join(newrl._datafile) | ||||
| with newvfs(newrl._indexfile, b'w'): | with newvfs(newrl._indexfile, b'w'): | ||||
| pass # create all the directories | pass # create all the directories | ||||
| util.copyfile(oldindex, newindex) | util.copyfile(oldindex, newindex) | ||||
| copydata = oldrl.opener.exists(oldrl.datafile) | copydata = oldrl.opener.exists(oldrl._datafile) | ||||
| if copydata: | if copydata: | ||||
| util.copyfile(olddata, newdata) | util.copyfile(olddata, newdata) | ||||
| if rl_type & store.FILEFLAGS_FILELOG: | if rl_type & store.FILEFLAGS_FILELOG: | ||||
| destrepo.svfs.fncache.add(unencodedname) | destrepo.svfs.fncache.add(unencodedname) | ||||
| if copydata: | if copydata: | ||||
| destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') | destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') | ||||
| import newer module separately in try clause for early Mercurial | import newer module separately in try clause for early Mercurial | ||||
| contrib/perf.py:\d+: (re) | contrib/perf.py:\d+: (re) | ||||
| > from mercurial import ( | > from mercurial import ( | ||||
| import newer module separately in try clause for early Mercurial | import newer module separately in try clause for early Mercurial | ||||
| contrib/perf.py:\d+: (re) | contrib/perf.py:\d+: (re) | ||||
| > origindexpath = orig.opener.join(indexfile) | > origindexpath = orig.opener.join(indexfile) | ||||
| use getvfs()/getsvfs() for early Mercurial | use getvfs()/getsvfs() for early Mercurial | ||||
| contrib/perf.py:\d+: (re) | contrib/perf.py:\d+: (re) | ||||
| > origdatapath = orig.opener.join(orig.datafile) | > origdatapath = orig.opener.join(datafile) | ||||
| use getvfs()/getsvfs() for early Mercurial | use getvfs()/getsvfs() for early Mercurial | ||||
| contrib/perf.py:\d+: (re) | contrib/perf.py:\d+: (re) | ||||
| > vfs = vfsmod.vfs(tmpdir) | > vfs = vfsmod.vfs(tmpdir) | ||||
| use getvfs()/getsvfs() for early Mercurial | use getvfs()/getsvfs() for early Mercurial | ||||
| contrib/perf.py:\d+: (re) | contrib/perf.py:\d+: (re) | ||||
| > vfs.options = getattr(orig.opener, 'options', None) | > vfs.options = getattr(orig.opener, 'options', None) | ||||
| use getvfs()/getsvfs() for early Mercurial | use getvfs()/getsvfs() for early Mercurial | ||||
| [1] | [1] | ||||
| ) | ) | ||||
| text = None | text = None | ||||
| cachedelta = (deltaparent, rlog.revdiff(deltaparent, r)) | cachedelta = (deltaparent, rlog.revdiff(deltaparent, r)) | ||||
| flags = rlog.flags(r) | flags = rlog.flags(r) | ||||
| ifh = dfh = None | ifh = dfh = None | ||||
| try: | try: | ||||
| ifh = dlog.opener(dlog._indexfile, b'a+') | ifh = dlog.opener(dlog._indexfile, b'a+') | ||||
| if not dlog._inline: | if not dlog._inline: | ||||
| dfh = dlog.opener(dlog.datafile, b'a+') | dfh = dlog.opener(dlog._datafile, b'a+') | ||||
| dlog._addrevision( | dlog._addrevision( | ||||
| rlog.node(r), text, tr, r, p1, p2, flags, cachedelta, ifh, dfh | rlog.node(r), text, tr, r, p1, p2, flags, cachedelta, ifh, dfh | ||||
| ) | ) | ||||
| finally: | finally: | ||||
| if dfh is not None: | if dfh is not None: | ||||
| dfh.close() | dfh.close() | ||||
| if ifh is not None: | if ifh is not None: | ||||
| ifh.close() | ifh.close() | ||||