This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8625

git: add debug logging when there's a mismatch in the cached heads list
ClosedPublic

Authored by durin42 on Jun 9 2020, 5:15 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list
Summary

The dag rebuild can be expensive, so let's try and avoid bugs where it
transparently rebuilds all the time for no reason. This would have
prevented the issue fixed in D8622.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Jun 9 2020, 5:15 PM
durin42 updated this revision to Diff 21601.Jun 9 2020, 5:17 PM
durin42 added a commit: rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list.Jun 10 2020, 1:41 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list (authored by durin42). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/git/__init__.py (13 lines)
Mhgext/git/gitlog.py (2 lines)
Mhgext/git/index.py (14 lines)
Mtests/test-git-interop.t (13 lines)

Diff 21602

hgext/git/__init__.py

from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
commands, commands,
error, error,
extensions, extensions,
localrepo, localrepo,
pycompat, pycompat,
registrar,
scmutil, scmutil,
store, store,
util, util,
) )
from . import ( from . import (
dirstate, dirstate,
gitlog, gitlog,
gitutil, gitutil,
index, index,
) )
configtable = {}
configitem = registrar.configitem(configtable)
# git.log-index-cache-miss: internal knob for testing
configitem(
b"git", b"log-index-cache-miss", default=False,
)
# TODO: extract an interface for this in core # TODO: extract an interface for this in core
class gitstore(object): # store.basicstore): class gitstore(object): # store.basicstore):
def __init__(self, path, vfstype): def __init__(self, path, vfstype):
self.vfs = vfstype(path) self.vfs = vfstype(path)
self.path = self.vfs.base self.path = self.vfs.base
self.createmode = store._calcmode(self.vfs) self.createmode = store._calcmode(self.vfs)
# above lines should go away in favor of: # above lines should go away in favor of:
# super(gitstore, self).__init__(path, vfstype) # super(gitstore, self).__init__(path, vfstype)
self.git = gitutil.get_pygit2().Repository( self.git = gitutil.get_pygit2().Repository(
os.path.normpath(os.path.join(path, b'..', b'.git')) os.path.normpath(os.path.join(path, b'..', b'.git'))
) )
self._progress_factory = lambda *args, **kwargs: None self._progress_factory = lambda *args, **kwargs: None
self._logfn = lambda x: None
@util.propertycache @util.propertycache
def _db(self): def _db(self):
# We lazy-create the database because we want to thread a # We lazy-create the database because we want to thread a
# progress callback down to the indexing process if it's # progress callback down to the indexing process if it's
# required, and we don't have a ui handle in makestore(). # required, and we don't have a ui handle in makestore().
return index.get_index(self.git, self._progress_factory) return index.get_index(self.git, self._logfn, self._progress_factory)
def join(self, f): def join(self, f):
"""Fake store.join method for git repositories. """Fake store.join method for git repositories.
For the most part, store.join is used for @storecache For the most part, store.join is used for @storecache
decorators to invalidate caches when various files decorators to invalidate caches when various files
change. We'll map the ones we care about, and ignore the rest. change. We'll map the ones we care about, and ignore the rest.
""" """
return 0 return 0
return orig(ui, dest=dest, **opts) return orig(ui, dest=dest, **opts)
def reposetup(ui, repo): def reposetup(ui, repo):
if repo.local() and isinstance(repo.store, gitstore): if repo.local() and isinstance(repo.store, gitstore):
orig = repo.__class__ orig = repo.__class__
repo.store._progress_factory = repo.ui.makeprogress repo.store._progress_factory = repo.ui.makeprogress
if ui.configbool(b'git', b'log-index-cache-miss'):
repo.store._logfn = repo.ui.warn
class gitlocalrepo(orig): class gitlocalrepo(orig):
def _makedirstate(self): def _makedirstate(self):
# TODO narrow support here # TODO narrow support here
return dirstate.gitdirstate( return dirstate.gitdirstate(
self.ui, self.vfs.base, self.store.git self.ui, self.vfs.base, self.store.git
) )

hgext/git/gitlog.py

# Set up an internal reference to force the commit into the # Set up an internal reference to force the commit into the
# changelog. Hypothetically, we could even use this refs/hg/ # changelog. Hypothetically, we could even use this refs/hg/
# namespace to allow for anonymous heads on git repos, which # namespace to allow for anonymous heads on git repos, which
# would be neat. # would be neat.
self.gitrepo.references.create( self.gitrepo.references.create(
'refs/hg/internal/latest-commit', oid, force=True 'refs/hg/internal/latest-commit', oid, force=True
) )
# Reindex now to pick up changes. We omit the progress # Reindex now to pick up changes. We omit the progress
# callback because this will be very quick. # and log callbacks because this will be very quick.
index._index_repo(self.gitrepo, self._db) index._index_repo(self.gitrepo, self._db)
return oid.raw return oid.raw
class manifestlog(baselog): class manifestlog(baselog):
def __getitem__(self, node): def __getitem__(self, node):
return self.get(b'', node) return self.get(b'', node)

hgext/git/index.py

'UPDATE changedfiles SET ' 'UPDATE changedfiles SET '
'p1node = ?, p1filenode = ?, p2node = ?, p2filenode = ? ' 'p1node = ?, p1filenode = ?, p2node = ?, p2filenode = ? '
'WHERE node = ? AND filename = ? AND filenode = ?', 'WHERE node = ? AND filename = ? AND filenode = ?',
(p1node, p1fnode, p2node, p2fnode, commit.id.hex, path, filenode), (p1node, p1fnode, p2node, p2fnode, commit.id.hex, path, filenode),
) )
db.commit() db.commit()
def _index_repo(gitrepo, db, progress_factory=lambda *args, **kwargs: None): def _index_repo(
gitrepo,
db,
logfn=lambda x: None,
progress_factory=lambda *args, **kwargs: None,
):
# Identify all references so we can tell the walker to visit all of them. # Identify all references so we can tell the walker to visit all of them.
all_refs = gitrepo.listall_references() all_refs = gitrepo.listall_references()
possible_heads = set() possible_heads = set()
prog = progress_factory(b'refs') prog = progress_factory(b'refs')
for pos, ref in enumerate(all_refs): for pos, ref in enumerate(all_refs):
if prog is not None: if prog is not None:
prog.update(pos) prog.update(pos)
if not ( if not (
cache_heads = { cache_heads = {
pycompat.sysstr(x[0]) pycompat.sysstr(x[0])
for x in db.execute('SELECT node FROM possible_heads') for x in db.execute('SELECT node FROM possible_heads')
} }
walker = None walker = None
cur_cache_heads = {h.hex for h in possible_heads} cur_cache_heads = {h.hex for h in possible_heads}
if cur_cache_heads == cache_heads: if cur_cache_heads == cache_heads:
return return
logfn(b'heads mismatch, rebuilding dagcache\n')
for start in possible_heads: for start in possible_heads:
if walker is None: if walker is None:
walker = gitrepo.walk(start, _OUR_ORDER) walker = gitrepo.walk(start, _OUR_ORDER)
else: else:
walker.push(start) walker.push(start)
# Empty out the existing changelog. Even for large-ish histories # Empty out the existing changelog. Even for large-ish histories
# we can do the top-level "walk all the commits" dance very # we can do the top-level "walk all the commits" dance very
if not haschild: if not haschild:
db.execute('INSERT INTO heads (node) VALUES(?)', (h,)) db.execute('INSERT INTO heads (node) VALUES(?)', (h,))
db.commit() db.commit()
if prog is not None: if prog is not None:
prog.complete() prog.complete()
def get_index(gitrepo, progress_factory=lambda *args, **kwargs: None): def get_index(
gitrepo, logfn=lambda x: None, progress_factory=lambda *args, **kwargs: None
):
cachepath = os.path.join( cachepath = os.path.join(
pycompat.fsencode(gitrepo.path), b'..', b'.hg', b'cache' pycompat.fsencode(gitrepo.path), b'..', b'.hg', b'cache'
) )
if not os.path.exists(cachepath): if not os.path.exists(cachepath):
os.makedirs(cachepath) os.makedirs(cachepath)
dbpath = os.path.join(cachepath, b'git-commits.sqlite') dbpath = os.path.join(cachepath, b'git-commits.sqlite')
db = _createdb(dbpath) db = _createdb(dbpath)
# TODO check against gitrepo heads before doing a full index # TODO check against gitrepo heads before doing a full index
# TODO thread a ui.progress call into this layer # TODO thread a ui.progress call into this layer
_index_repo(gitrepo, db, progress_factory) _index_repo(gitrepo, db, logfn, progress_factory)
return db return db

tests/test-git-interop.t

$ hg status $ hg status
? .hg/cache/git-commits.sqlite ? .hg/cache/git-commits.sqlite
? .hg/cache/git-commits.sqlite-shm ? .hg/cache/git-commits.sqlite-shm
? .hg/cache/git-commits.sqlite-wal ? .hg/cache/git-commits.sqlite-wal
? .hg/requires ? .hg/requires
$ cd .. $ cd ..
Now globally enable extension for the rest of the test: Now globally enable extension for the rest of the test:
$ echo "[extensions]" >> $HGRCPATH $ cat <<EOF >> $HGRCPATH
> echo "git=" >> $HGRCPATH > [extensions]
> git=
> [git]
> log-index-cache-miss = yes
> EOF
Make a new repo with git: Make a new repo with git:
$ mkdir foo $ mkdir foo
$ cd foo $ cd foo
$ git init $ git init
Initialized empty Git repository in $TESTTMP/foo/.git/ Initialized empty Git repository in $TESTTMP/foo/.git/
Ignore the .hg directory within git: Ignore the .hg directory within git:
$ echo .hg >> .git/info/exclude $ echo .hg >> .git/info/exclude
Without creating the .hg, hg status fails: Without creating the .hg, hg status fails:
$ hg status $ hg status
abort: no repository found in '$TESTTMP/foo' (.hg not found)! abort: no repository found in '$TESTTMP/foo' (.hg not found)!
[255] [255]
But if you run hg init --git, it works: But if you run hg init --git, it works:
$ hg init --git $ hg init --git
$ hg id --traceback $ hg id --traceback
heads mismatch, rebuilding dagcache
3d9be8deba43 tip master 3d9be8deba43 tip master
$ hg status $ hg status
? gamma ? gamma
Log works too: Log works too:
$ hg log $ hg log
changeset: 1:3d9be8deba43 changeset: 1:3d9be8deba43
bookmark: master bookmark: master
tag: tip tag: tip
$ rm gamma $ rm gamma
hg log FILE hg log FILE
$ echo a >> alpha $ echo a >> alpha
$ hg ci -m 'more alpha' --traceback --date '1583522787 18000' $ hg ci -m 'more alpha' --traceback --date '1583522787 18000'
$ echo b >> beta $ echo b >> beta
$ hg ci -m 'more beta' $ hg ci -m 'more beta'
heads mismatch, rebuilding dagcache
$ echo a >> alpha $ echo a >> alpha
$ hg ci -m 'even more alpha' $ hg ci -m 'even more alpha'
heads mismatch, rebuilding dagcache
$ hg log -G alpha $ hg log -G alpha
heads mismatch, rebuilding dagcache
@ changeset: 4:6626247b7dc8 @ changeset: 4:6626247b7dc8
: bookmark: master : bookmark: master
: tag: tip : tag: tip
: user: test <test> : user: test <test>
: date: Thu Jan 01 00:00:00 1970 +0000 : date: Thu Jan 01 00:00:00 1970 +0000
: summary: even more alpha : summary: even more alpha
: :
o changeset: 2:a1983dd7fb19 o changeset: 2:a1983dd7fb19
beta beta
hg and git status both clean hg and git status both clean
$ git status $ git status
On branch master On branch master
nothing to commit, working tree clean nothing to commit, working tree clean
$ hg status $ hg status
heads mismatch, rebuilding dagcache
node|shortest works correctly node|shortest works correctly
$ hg log -T '{node}\n' | sort $ hg log -T '{node}\n' | sort
3d9be8deba43482be2c81a4cb4be1f10d85fa8bc 3d9be8deba43482be2c81a4cb4be1f10d85fa8bc
6626247b7dc8f231b183b8a4761c89139baca2ad 6626247b7dc8f231b183b8a4761c89139baca2ad
a1983dd7fb19cbd83ad5a1c2fc8bf3d775dea12f a1983dd7fb19cbd83ad5a1c2fc8bf3d775dea12f
ae1ab744f95bfd5b07cf573baef98a778058537b ae1ab744f95bfd5b07cf573baef98a778058537b