This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10888

copies: Keep changelog sidedata file open during copy tracing
ClosedPublic

Authored by SimonSapin on Jun 21 2021, 3:46 AM.

Details

Reviewers
indygreg
pulkit
Group Reviewers
hg-reviewers
Commits
rHG5fa083a5ff04: copies: Keep changelog sidedata file open during copy tracing
Summary

Instead of having a callback that opens and closes that file many times,
a add and use a context manager method on the revlog class that keeps
files open for its duration.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

SimonSapin created this revision.Jun 21 2021, 3:46 AM
joerg.sonnenberger added a subscriber: joerg.sonnenberger.Jun 21 2021, 5:49 AM

Can you comment on how many of them we keep open at a time?

SimonSapin added a comment.Jun 22 2021, 10:11 AM

One call to revlog.reading opens (if they aren’t open already) two files for the "main" data and for sidedata for that revlog. In the case of copy tracing, it is called on the changelog.

baymax updated this revision to Diff 28670.Jun 22 2021, 11:18 AM

✅ refresh by Heptapod after a successful CI run (🐙 💚)

pulkit accepted this revision.Jun 24 2021, 3:12 AM
This revision is now accepted and ready to land.Jun 24 2021, 3:12 AM

Revision Contents
Changeset List

Diff 28689

mercurial/copies.py

for r in revs: for r in revs:
for p in cl.parentrevs(r): for p in cl.parentrevs(r):
if p == nullrev: if p == nullrev:
continue continue
children_count[r] = 0 children_count[r] = 0
if p in children_count: if p in children_count:
children_count[p] += 1 children_count[p] += 1
revinfo = _revinfo_getter(repo, match) revinfo = _revinfo_getter(repo, match)
with repo.changelog.reading():
return _combine_changeset_copies( return _combine_changeset_copies(
revs, revs,
children_count, children_count,
b.rev(), b.rev(),
revinfo, revinfo,
match, match,
isancestor, isancestor,
multi_thread, multi_thread,
) )
else: else:
# When not using side-data, we will process the edges "from" the parent. # When not using side-data, we will process the edges "from" the parent.
# so we need a full mapping of the parent -> children relation. # so we need a full mapping of the parent -> children relation.
children = dict((r, []) for r in roots) children = dict((r, []) for r in roots)
for r in revs: for r in revs:
for p in cl.parentrevs(r): for p in cl.parentrevs(r):
if p == nullrev: if p == nullrev:
continue continue

mercurial/revlog.py

finally: finally:
if new_dfh is not None: if new_dfh is not None:
new_dfh.close() new_dfh.close()
def _nodeduplicatecallback(self, transaction, node): def _nodeduplicatecallback(self, transaction, node):
"""called when trying to add a node already stored.""" """called when trying to add a node already stored."""
@contextlib.contextmanager @contextlib.contextmanager
def reading(self):
"""Context manager that keeps data and sidedata files open for reading"""
with self._segmentfile.reading():
with self._segmentfile_sidedata.reading():
yield
@contextlib.contextmanager
def _writing(self, transaction): def _writing(self, transaction):
if self._trypending: if self._trypending:
msg = b'try to write in a `trypending` revlog: %s' msg = b'try to write in a `trypending` revlog: %s'
msg %= self.display_id msg %= self.display_id
raise error.ProgrammingError(msg) raise error.ProgrammingError(msg)
if self._writinghandles is not None: if self._writinghandles is not None:
yield yield
else: else:

mercurial/revlogutils/randomaccessfile.py

): ):
# Required by bitwise manipulation below # Required by bitwise manipulation below
assert _is_power_of_two(default_cached_chunk_size) assert _is_power_of_two(default_cached_chunk_size)
self.opener = opener self.opener = opener
self.filename = filename self.filename = filename
self.default_cached_chunk_size = default_cached_chunk_size self.default_cached_chunk_size = default_cached_chunk_size
self.writing_handle = None # This is set from revlog.py self.writing_handle = None # This is set from revlog.py
self.reading_handle = None
self._cached_chunk = b'' self._cached_chunk = b''
self._cached_chunk_position = 0 # Offset from the start of the file self._cached_chunk_position = 0 # Offset from the start of the file
if initial_cache: if initial_cache:
self._cached_chunk_position, self._cached_chunk = initial_cache self._cached_chunk_position, self._cached_chunk = initial_cache
def clear_cache(self): def clear_cache(self):
self._cached_chunk = b'' self._cached_chunk = b''
self._cached_chunk_position = 0 self._cached_chunk_position = 0
# Use a file handle being actively used for writes, if available. # Use a file handle being actively used for writes, if available.
# There is some danger to doing this because reads will seek the # There is some danger to doing this because reads will seek the
# file. However, revlog._writeentry performs a SEEK_END before all # file. However, revlog._writeentry performs a SEEK_END before all
# writes, so we should be safe. # writes, so we should be safe.
elif self.writing_handle: elif self.writing_handle:
yield self.writing_handle yield self.writing_handle
elif self.reading_handle:
yield self.reading_handle
# Otherwise open a new file handle. # Otherwise open a new file handle.
else: else:
with self._open() as fp: with self._open() as fp:
yield fp yield fp
@contextlib.contextmanager
def reading(self):
"""Context manager that keeps the file open for reading"""
if (
self.reading_handle is None
and self.writing_handle is None
and self.filename is not None
):
with self._open() as fp:
self.reading_handle = fp
try:
yield
finally:
self.reading_handle = None
else:
yield
def read_chunk(self, offset, length, existing_file_obj=None): def read_chunk(self, offset, length, existing_file_obj=None):
"""Read a chunk of bytes from the file. """Read a chunk of bytes from the file.
Accepts an absolute offset, length to read, and an optional existing Accepts an absolute offset, length to read, and an optional existing
file handle to read from. file handle to read from.
If an existing file handle is passed, it will be seeked and the If an existing file handle is passed, it will be seeked and the
original seek position will NOT be restored. original seek position will NOT be restored.