This is an archive of the discontinued Mercurial Phabricator instance.

Differential D8625

git: add debug logging when there's a mismatch in the cached heads list
ClosedPublic

Authored by durin42 on Jun 9 2020, 5:15 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list
Summary

The dag rebuild can be expensive, so let's try and avoid bugs where it
transparently rebuilds all the time for no reason. This would have
prevented the issue fixed in D8622.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

durin42 created this revision.Jun 9 2020, 5:15 PM
durin42 updated this revision to Diff 21601.Jun 9 2020, 5:17 PM
durin42 added a commit: rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list.Jun 10 2020, 1:41 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG83e41b73d115: git: add debug logging when there's a mismatch in the cached heads list (authored by durin42). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/git/__init__.py (13 lines)
Mhgext/git/gitlog.py (2 lines)
Mhgext/git/index.py (14 lines)
Mtests/test-git-interop.t (35 lines)

Diff 21599

hgext/git/__init__.py

from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
commands, commands,
error, error,
extensions, extensions,
localrepo, localrepo,
pycompat, pycompat,
registrar,
scmutil, scmutil,
store, store,
util, util,
) )
from . import ( from . import (
dirstate, dirstate,
gitlog, gitlog,
gitutil, gitutil,
index, index,
) )
configtable = {}
configitem = registrar.configitem(configtable)
# git.log-index-cache-miss: internal knob for testing
configitem(
b"git", b"log-index-cache-miss", default=False,
)
# TODO: extract an interface for this in core # TODO: extract an interface for this in core
class gitstore(object): # store.basicstore): class gitstore(object): # store.basicstore):
def __init__(self, path, vfstype): def __init__(self, path, vfstype):
self.vfs = vfstype(path) self.vfs = vfstype(path)
self.path = self.vfs.base self.path = self.vfs.base
self.createmode = store._calcmode(self.vfs) self.createmode = store._calcmode(self.vfs)
# above lines should go away in favor of: # above lines should go away in favor of:
# super(gitstore, self).__init__(path, vfstype) # super(gitstore, self).__init__(path, vfstype)
self.git = gitutil.get_pygit2().Repository( self.git = gitutil.get_pygit2().Repository(
os.path.normpath(os.path.join(path, b'..', b'.git')) os.path.normpath(os.path.join(path, b'..', b'.git'))
) )
self._progress_factory = lambda *args, **kwargs: None self._progress_factory = lambda *args, **kwargs: None
self._logfn = lambda x: None
@util.propertycache @util.propertycache
def _db(self): def _db(self):
# We lazy-create the database because we want to thread a # We lazy-create the database because we want to thread a
# progress callback down to the indexing process if it's # progress callback down to the indexing process if it's
# required, and we don't have a ui handle in makestore(). # required, and we don't have a ui handle in makestore().
return index.get_index(self.git, self._progress_factory) return index.get_index(self.git, self._logfn, self._progress_factory)
def join(self, f): def join(self, f):
"""Fake store.join method for git repositories. """Fake store.join method for git repositories.
For the most part, store.join is used for @storecache For the most part, store.join is used for @storecache
decorators to invalidate caches when various files decorators to invalidate caches when various files
change. We'll map the ones we care about, and ignore the rest. change. We'll map the ones we care about, and ignore the rest.
""" """
return 0 return 0
return orig(ui, dest=dest, **opts) return orig(ui, dest=dest, **opts)
def reposetup(ui, repo): def reposetup(ui, repo):
if repo.local() and isinstance(repo.store, gitstore): if repo.local() and isinstance(repo.store, gitstore):
orig = repo.__class__ orig = repo.__class__
repo.store._progress_factory = repo.ui.makeprogress repo.store._progress_factory = repo.ui.makeprogress
if ui.configbool(b'git', b'log-index-cache-miss'):
repo.store._logfn = repo.ui.warn
class gitlocalrepo(orig): class gitlocalrepo(orig):
def _makedirstate(self): def _makedirstate(self):
# TODO narrow support here # TODO narrow support here
return dirstate.gitdirstate( return dirstate.gitdirstate(
self.ui, self.vfs.base, self.store.git self.ui, self.vfs.base, self.store.git
) )

hgext/git/gitlog.py

# Set up an internal reference to force the commit into the # Set up an internal reference to force the commit into the
# changelog. Hypothetically, we could even use this refs/hg/ # changelog. Hypothetically, we could even use this refs/hg/
# namespace to allow for anonymous heads on git repos, which # namespace to allow for anonymous heads on git repos, which
# would be neat. # would be neat.
self.gitrepo.references.create( self.gitrepo.references.create(
'refs/hg/internal/latest-commit', oid, force=True 'refs/hg/internal/latest-commit', oid, force=True
) )
# Reindex now to pick up changes. We omit the progress # Reindex now to pick up changes. We omit the progress
# callback because this will be very quick. # and log callbacks because this will be very quick.
index._index_repo(self.gitrepo, self._db) index._index_repo(self.gitrepo, self._db)
return oid.raw return oid.raw
class manifestlog(baselog): class manifestlog(baselog):
def __getitem__(self, node): def __getitem__(self, node):
return self.get(b'', node) return self.get(b'', node)

hgext/git/index.py

'UPDATE changedfiles SET ' 'UPDATE changedfiles SET '
'p1node = ?, p1filenode = ?, p2node = ?, p2filenode = ? ' 'p1node = ?, p1filenode = ?, p2node = ?, p2filenode = ? '
'WHERE node = ? AND filename = ? AND filenode = ?', 'WHERE node = ? AND filename = ? AND filenode = ?',
(p1node, p1fnode, p2node, p2fnode, commit.id.hex, path, filenode), (p1node, p1fnode, p2node, p2fnode, commit.id.hex, path, filenode),
) )
db.commit() db.commit()
def _index_repo(gitrepo, db, progress_factory=lambda *args, **kwargs: None): def _index_repo(
gitrepo,
db,
logfn=lambda x: None,
progress_factory=lambda *args, **kwargs: None,
):
# Identify all references so we can tell the walker to visit all of them. # Identify all references so we can tell the walker to visit all of them.
all_refs = gitrepo.listall_references() all_refs = gitrepo.listall_references()
possible_heads = set() possible_heads = set()
prog = progress_factory(b'refs') prog = progress_factory(b'refs')
for pos, ref in enumerate(all_refs): for pos, ref in enumerate(all_refs):
if prog is not None: if prog is not None:
prog.update(pos) prog.update(pos)
if not ( if not (
cache_heads = { cache_heads = {
pycompat.sysstr(x[0]) pycompat.sysstr(x[0])
for x in db.execute('SELECT node FROM possible_heads') for x in db.execute('SELECT node FROM possible_heads')
} }
walker = None walker = None
cur_cache_heads = {h.hex for h in possible_heads} cur_cache_heads = {h.hex for h in possible_heads}
if cur_cache_heads == cache_heads: if cur_cache_heads == cache_heads:
return return
logfn(b'heads mismatch, rebuilding dagcache\n')
for start in possible_heads: for start in possible_heads:
if walker is None: if walker is None:
walker = gitrepo.walk(start, _OUR_ORDER) walker = gitrepo.walk(start, _OUR_ORDER)
else: else:
walker.push(start) walker.push(start)
# Empty out the existing changelog. Even for large-ish histories # Empty out the existing changelog. Even for large-ish histories
# we can do the top-level "walk all the commits" dance very # we can do the top-level "walk all the commits" dance very
if not haschild: if not haschild:
db.execute('INSERT INTO heads (node) VALUES(?)', (h,)) db.execute('INSERT INTO heads (node) VALUES(?)', (h,))
db.commit() db.commit()
if prog is not None: if prog is not None:
prog.complete() prog.complete()
def get_index(gitrepo, progress_factory=lambda *args, **kwargs: None): def get_index(
gitrepo, logfn=lambda x: None, progress_factory=lambda *args, **kwargs: None
):
cachepath = os.path.join( cachepath = os.path.join(
pycompat.fsencode(gitrepo.path), b'..', b'.hg', b'cache' pycompat.fsencode(gitrepo.path), b'..', b'.hg', b'cache'
) )
if not os.path.exists(cachepath): if not os.path.exists(cachepath):
os.makedirs(cachepath) os.makedirs(cachepath)
dbpath = os.path.join(cachepath, b'git-commits.sqlite') dbpath = os.path.join(cachepath, b'git-commits.sqlite')
db = _createdb(dbpath) db = _createdb(dbpath)
# TODO check against gitrepo heads before doing a full index # TODO check against gitrepo heads before doing a full index
# TODO thread a ui.progress call into this layer # TODO thread a ui.progress call into this layer
_index_repo(gitrepo, db, progress_factory) _index_repo(gitrepo, db, logfn, progress_factory)
return db return db

tests/test-git-interop.t

$ hg status $ hg status
? .hg/cache/git-commits.sqlite ? .hg/cache/git-commits.sqlite
? .hg/cache/git-commits.sqlite-shm ? .hg/cache/git-commits.sqlite-shm
? .hg/cache/git-commits.sqlite-wal ? .hg/cache/git-commits.sqlite-wal
? .hg/requires ? .hg/requires
$ cd .. $ cd ..
Now globally enable extension for the rest of the test: Now globally enable extension for the rest of the test:
$ echo "[extensions]" >> $HGRCPATH $ cat <<EOF >> $HGRCPATH
> echo "git=" >> $HGRCPATH > [extensions]
> git=
> [git]
> log-index-cache-miss = yes
> EOF
Make a new repo with git: Make a new repo with git:
$ mkdir foo $ mkdir foo
$ cd foo $ cd foo
$ git init $ git init
Initialized empty Git repository in $TESTTMP/foo/.git/ Initialized empty Git repository in $TESTTMP/foo/.git/
Ignore the .hg directory within git: Ignore the .hg directory within git:
$ echo .hg >> .git/info/exclude $ echo .hg >> .git/info/exclude
Without creating the .hg, hg status fails: Without creating the .hg, hg status fails:
$ hg status $ hg status
abort: no repository found in '$TESTTMP/foo' (.hg not found)! abort: no repository found in '$TESTTMP/foo' (.hg not found)!
[255] [255]
But if you run hg init --git, it works: But if you run hg init --git, it works:
$ hg init --git $ hg init --git
$ hg id --traceback $ hg id --traceback
heads mismatch, rebuilding dagcache
3d9be8deba43 tip master 3d9be8deba43 tip master
$ hg status $ hg status
heads mismatch, rebuilding dagcache
? gamma ? gamma
Log works too: Log works too:
$ hg log $ hg log
heads mismatch, rebuilding dagcache
changeset: 1:3d9be8deba43 changeset: 1:3d9be8deba43
bookmark: master bookmark: master
tag: tip tag: tip
user: test <test@example.org> user: test <test@example.org>
date: Mon Jan 01 00:00:11 2007 +0000 date: Mon Jan 01 00:00:11 2007 +0000
summary: Add beta summary: Add beta
changeset: 0:c5864c9d16fb changeset: 0:c5864c9d16fb
user: test <test@example.org> user: test <test@example.org>
date: Mon Jan 01 00:00:10 2007 +0000 date: Mon Jan 01 00:00:10 2007 +0000
summary: Add alpha summary: Add alpha
and bookmarks: and bookmarks:
$ hg bookmarks $ hg bookmarks
* master 1:3d9be8deba43 * masterheads mismatch, rebuilding dagcache
1:3d9be8deba43
diff even works transparently in both systems: diff even works transparently in both systems:
$ echo blah >> alpha $ echo blah >> alpha
$ git diff $ git diff
diff --git a/alpha b/alpha diff --git a/alpha b/alpha
index 4a58007..faed1b7 100644 index 4a58007..faed1b7 100644
--- a/alpha --- a/alpha
+++ b/alpha +++ b/alpha
@@ -1* +1,2 @@ (glob) @@ -1* +1,2 @@ (glob)
alpha alpha
+blah +blah
$ hg diff --git $ hg diff --git
heads mismatch, rebuilding dagcache
diff --git a/alpha b/alpha diff --git a/alpha b/alpha
--- a/alpha --- a/alpha
+++ b/alpha +++ b/alpha
@@ -1,1 +1,2 @@ @@ -1,1 +1,2 @@
alpha alpha
+blah +blah
Remove a file, it shows as such: Remove a file, it shows as such:
$ rm alpha $ rm alpha
$ hg status $ hg status
heads mismatch, rebuilding dagcache
! alpha ! alpha
? gamma ? gamma
Revert works: Revert works:
$ hg revert alpha --traceback $ hg revert alpha --traceback
heads mismatch, rebuilding dagcache
$ hg status $ hg status
heads mismatch, rebuilding dagcache
? gamma ? gamma
$ git status $ git status
On branch master On branch master
Untracked files: Untracked files:
(use "git add <file>..." to include in what will be committed) (use "git add <file>..." to include in what will be committed)
gamma gamma
nothing added to commit but untracked files present (use "git add" to track) nothing added to commit but untracked files present (use "git add" to track)
Add shows sanely in both: Add shows sanely in both:
$ hg add gamma $ hg add gamma
$ hg status $ hg status
heads mismatch, rebuilding dagcache
A gamma A gamma
$ hg files $ hg files
alpha alpha
beta beta
gamma gamma
$ git ls-files $ git ls-files
alpha alpha
beta beta
gamma gamma
$ git status $ git status
On branch master On branch master
Changes to be committed: Changes to be committed:
(use "git restore --staged <file>..." to unstage) (use "git restore --staged <file>..." to unstage)
new file: gamma new file: gamma
forget does what it should as well: forget does what it should as well:
$ hg forget gamma $ hg forget gamma
heads mismatch, rebuilding dagcache
$ hg status $ hg status
heads mismatch, rebuilding dagcache
? gamma ? gamma
$ git status $ git status
On branch master On branch master
Untracked files: Untracked files:
(use "git add <file>..." to include in what will be committed) (use "git add <file>..." to include in what will be committed)
gamma gamma
nothing added to commit but untracked files present (use "git add" to track) nothing added to commit but untracked files present (use "git add" to track)
clean up untracked file clean up untracked file
$ rm gamma $ rm gamma
hg log FILE hg log FILE
$ echo a >> alpha $ echo a >> alpha
$ hg ci -m 'more alpha' --traceback --date '1583522787 18000' $ hg ci -m 'more alpha' --traceback --date '1583522787 18000'
heads mismatch, rebuilding dagcache
$ echo b >> beta $ echo b >> beta
$ hg ci -m 'more beta' $ hg ci -m 'more beta'
heads mismatch, rebuilding dagcache
$ echo a >> alpha $ echo a >> alpha
$ hg ci -m 'even more alpha' $ hg ci -m 'even more alpha'
heads mismatch, rebuilding dagcache
$ hg log -G alpha $ hg log -G alpha
heads mismatch, rebuilding dagcache
@ changeset: 4:6626247b7dc8 @ changeset: 4:6626247b7dc8
: bookmark: master : bookmark: master
: tag: tip : tag: tip
: user: test <test> : user: test <test>
: date: Thu Jan 01 00:00:00 1970 +0000 : date: Thu Jan 01 00:00:00 1970 +0000
: summary: even more alpha : summary: even more alpha
: :
o changeset: 2:a1983dd7fb19 o changeset: 2:a1983dd7fb19
: user: test <test> : user: test <test>
: date: Fri Mar 06 14:26:27 2020 -0500 : date: Fri Mar 06 14:26:27 2020 -0500
: summary: more alpha : summary: more alpha
: :
o changeset: 0:c5864c9d16fb o changeset: 0:c5864c9d16fb
user: test <test@example.org> user: test <test@example.org>
date: Mon Jan 01 00:00:10 2007 +0000 date: Mon Jan 01 00:00:10 2007 +0000
summary: Add alpha summary: Add alpha
$ hg log -G beta $ hg log -G beta
heads mismatch, rebuilding dagcache
o changeset: 3:d8ee22687733 o changeset: 3:d8ee22687733
: user: test <test> : user: test <test>
: date: Thu Jan 01 00:00:00 1970 +0000 : date: Thu Jan 01 00:00:00 1970 +0000
: summary: more beta : summary: more beta
: :
o changeset: 1:3d9be8deba43 o changeset: 1:3d9be8deba43
| user: test <test@example.org> | user: test <test@example.org>
~ date: Mon Jan 01 00:00:11 2007 +0000 ~ date: Mon Jan 01 00:00:11 2007 +0000
summary: Add beta summary: Add beta
$ hg log -r "children(3d9be8deba43)" -T"{node|short} {children}\n" $ hg log -r "children(3d9be8deba43)" -T"{node|short} {children}\n"
heads mismatch, rebuilding dagcache
a1983dd7fb19 3:d8ee22687733 a1983dd7fb19 3:d8ee22687733
hg annotate hg annotate
$ hg annotate alpha $ hg annotate alpha
heads mismatch, rebuilding dagcache
0: alpha 0: alpha
2: a 2: a
4: a 4: a
$ hg annotate beta $ hg annotate beta
heads mismatch, rebuilding dagcache
1: beta 1: beta
3: b 3: b
Files in subdirectories. TODO: case-folding support, make this `A` Files in subdirectories. TODO: case-folding support, make this `A`
instead of `a`. instead of `a`.
$ mkdir a $ mkdir a
$ echo "This is file mu." > a/mu $ echo "This is file mu." > a/mu
$ hg ci -A -m 'Introduce file a/mu' $ hg ci -A -m 'Introduce file a/mu'
heads mismatch, rebuilding dagcache
adding a/mu adding a/mu
Both hg and git agree a/mu is part of the repo Both hg and git agree a/mu is part of the repo
$ git ls-files $ git ls-files
a/mu a/mu
alpha alpha
beta beta
$ hg files $ hg files
a/mu a/mu
alpha alpha
beta beta
hg and git status both clean hg and git status both clean
$ git status $ git status
On branch master On branch master
nothing to commit, working tree clean nothing to commit, working tree clean
$ hg status $ hg status
heads mismatch, rebuilding dagcache
node|shortest works correctly node|shortest works correctly
$ hg log -T '{node}\n' | sort $ hg log -T '{node}\n' | sort
heads mismatch, rebuilding dagcache
3d9be8deba43482be2c81a4cb4be1f10d85fa8bc 3d9be8deba43482be2c81a4cb4be1f10d85fa8bc
6626247b7dc8f231b183b8a4761c89139baca2ad 6626247b7dc8f231b183b8a4761c89139baca2ad
a1983dd7fb19cbd83ad5a1c2fc8bf3d775dea12f a1983dd7fb19cbd83ad5a1c2fc8bf3d775dea12f
ae1ab744f95bfd5b07cf573baef98a778058537b ae1ab744f95bfd5b07cf573baef98a778058537b
c5864c9d16fb3431fe2c175ff84dc6accdbb2c18 c5864c9d16fb3431fe2c175ff84dc6accdbb2c18
d8ee22687733a1991813560b15128cd9734f4b48 d8ee22687733a1991813560b15128cd9734f4b48
$ hg log -r ae1ab744f95bfd5b07cf573baef98a778058537b --template "{shortest(node,1)}\n" $ hg log -r ae1ab744f95bfd5b07cf573baef98a778058537b --template "{shortest(node,1)}\n"
heads mismatch, rebuilding dagcache
ae ae
This coveres changelog.findmissing() This coveres changelog.findmissing()
$ hg merge --preview 3d9be8deba43 $ hg merge --preview 3d9be8deba43
heads mismatch, rebuilding dagcache
This covers manifest.diff() This covers manifest.diff()
$ hg diff -c 3d9be8deba43 $ hg diff -c 3d9be8deba43
heads mismatch, rebuilding dagcache
diff -r c5864c9d16fb -r 3d9be8deba43 beta diff -r c5864c9d16fb -r 3d9be8deba43 beta
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/beta Mon Jan 01 00:00:11 2007 +0000 +++ b/beta Mon Jan 01 00:00:11 2007 +0000
@@ -0,0 +1,1 @@ @@ -0,0 +1,1 @@
+beta +beta