This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6634

rust-dirstate: call rust dirstatemap from Python
ClosedPublic

Authored by Alphare on Jul 10 2019, 10:34 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHG749ef8c31187: rust-dirstate: call rust dirstatemap from Python
rHG39674659b3f4: rust-dirstate: call rust dirstatemap from Python
Summary

Since Rust-backed Python classes cannot be used as baseclasses (for
rust-cpython anyway), we use composition rather than inheritance.

This also allows us to keep the IO operations in the Python side, removing
(for now) the need to rewrite VFS in Rust, which would be a heavy undertaking.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

Alphare created this revision.Jul 10 2019, 10:34 AM
Alphare updated this revision to Diff 15894.Jul 12 2019, 4:50 AM
Alphare updated this revision to Diff 16053.Jul 24 2019, 11:23 AM
Alphare added a subscriber: reviewers.EditedAug 6 2019, 12:22 PM

@reviewers I am leaving for a vacation on the 9th and I won't be able to answer your questions or work on those patches before the 20th. This series has been idle for a long time, I didn't want it to be forgotten about.

Alphare added a commit: rHG39674659b3f4: rust-dirstate: call rust dirstatemap from Python.Aug 14 2019, 4:41 PM
This revision was not accepted when it landed; it landed in state Needs Review.
Closed by commit rHG39674659b3f4: rust-dirstate: call rust dirstatemap from Python (authored by Alphare). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mhgext/largefiles/overrides.py (2 lines)
Mmercurial/dirstate.py (199 lines)

Diff 16204

hgext/largefiles/overrides.py

if overwrite: if overwrite:
return actions, diverge, renamedelete return actions, diverge, renamedelete
# Convert to dictionary with filename as key and action as value. # Convert to dictionary with filename as key and action as value.
lfiles = set() lfiles = set()
for f in actions: for f in actions:
splitstandin = lfutil.splitstandin(f) splitstandin = lfutil.splitstandin(f)
if splitstandin in p1: if splitstandin is not None and splitstandin in p1:
lfiles.add(splitstandin) lfiles.add(splitstandin)
elif lfutil.standin(f) in p1: elif lfutil.standin(f) in p1:
lfiles.add(f) lfiles.add(f)
for lfile in sorted(lfiles): for lfile in sorted(lfiles):
standin = lfutil.standin(lfile) standin = lfutil.standin(lfile)
(lm, largs, lmsg) = actions.get(lfile, (None, None, None)) (lm, largs, lmsg) = actions.get(lfile, (None, None, None))
(sm, sargs, smsg) = actions.get(standin, (None, None, None)) (sm, sargs, smsg) = actions.get(standin, (None, None, None))

mercurial/dirstate.py

pathutil, pathutil,
policy, policy,
pycompat, pycompat,
scmutil, scmutil,
txnutil, txnutil,
util, util,
) )
orig_parsers = policy.importmod(r'parsers') parsers = policy.importmod(r'parsers')
parsers = policy.importrust(r'parsers', default=orig_parsers) rustmod = policy.importrust(r'dirstate')
propertycache = util.propertycache propertycache = util.propertycache
filecache = scmutil.filecache filecache = scmutil.filecache
_rangemask = 0x7fffffff _rangemask = 0x7fffffff
dirstatetuple = orig_parsers.dirstatetuple dirstatetuple = parsers.dirstatetuple
class repocache(filecache): class repocache(filecache):
"""filecache for files in .hg/""" """filecache for files in .hg/"""
def join(self, obj, fname): def join(self, obj, fname):
return obj._opener.join(fname) return obj._opener.join(fname)
class rootcache(filecache): class rootcache(filecache):
"""filecache for files in the repository root""" """filecache for files in the repository root"""
# filesystem's notion of 'now' # filesystem's notion of 'now'
now = util.fstat(st)[stat.ST_MTIME] & _rangemask now = util.fstat(st)[stat.ST_MTIME] & _rangemask
# enough 'delaywrite' prevents 'pack_dirstate' from dropping # enough 'delaywrite' prevents 'pack_dirstate' from dropping
# timestamp of each entries in dirstate, because of 'now > mtime' # timestamp of each entries in dirstate, because of 'now > mtime'
delaywrite = self._ui.configint('debug', 'dirstate.delaywrite') delaywrite = self._ui.configint('debug', 'dirstate.delaywrite')
if delaywrite > 0: if delaywrite > 0:
# do we have any files to delay for? # do we have any files to delay for?
for f, e in self._map.iteritems(): items = self._map.iteritems()
for f, e in items:
if e[0] == 'n' and e[3] == now: if e[0] == 'n' and e[3] == now:
import time # to avoid useless import import time # to avoid useless import
# rather than sleep n seconds, sleep until the next # rather than sleep n seconds, sleep until the next
# multiple of n seconds # multiple of n seconds
clock = time.time() clock = time.time()
start = int(clock) - (int(clock) % delaywrite) start = int(clock) - (int(clock) % delaywrite)
end = start + delaywrite end = start + delaywrite
time.sleep(end - clock) time.sleep(end - clock)
now = end # trust our estimate that the end is near now now = end # trust our estimate that the end is near now
break break
# since the iterator is potentially not depleted,
# delete the iterator to release the reference for the Rust
# implementation.
# TODO make the Rust implementation behave like Python
# since this would not work with a non ref-counting GC.
del items
self._map.write(st, now) self._map.write(st, now)
self._lastnormaltime = 0 self._lastnormaltime = 0
self._dirty = False self._dirty = False
def _dirignore(self, f): def _dirignore(self, f):
if self._ignore(f): if self._ignore(f):
return True return True
@propertycache @propertycache
def dirfoldmap(self): def dirfoldmap(self):
f = {} f = {}
normcase = util.normcase normcase = util.normcase
for name in self._dirs: for name in self._dirs:
f[normcase(name)] = name f[normcase(name)] = name
return f return f
if rustmod is not None:
class dirstatemap(object):
def __init__(self, ui, opener, root):
self._ui = ui
self._opener = opener
self._root = root
self._filename = 'dirstate'
self._parents = None
self._dirtyparents = False
# for consistent view between _pl() and _read() invocations
self._pendingmode = None
def addfile(self, *args, **kwargs):
return self._rustmap.addfile(*args, **kwargs)
def removefile(self, *args, **kwargs):
return self._rustmap.removefile(*args, **kwargs)
def dropfile(self, *args, **kwargs):
return self._rustmap.dropfile(*args, **kwargs)
def clearambiguoustimes(self, *args, **kwargs):
return self._rustmap.clearambiguoustimes(*args, **kwargs)
def nonnormalentries(self):
return self._rustmap.nonnormalentries()
def get(self, *args, **kwargs):
return self._rustmap.get(*args, **kwargs)
@propertycache
def _rustmap(self):
self._rustmap = rustmod.DirstateMap(self._root)
self.read()
return self._rustmap
@property
def copymap(self):
return self._rustmap.copymap()
def preload(self):
self._rustmap
def clear(self):
self._rustmap.clear()
self.setparents(nullid, nullid)
util.clearcachedproperty(self, "_dirs")
util.clearcachedproperty(self, "_alldirs")
util.clearcachedproperty(self, "dirfoldmap")
def items(self):
return self._rustmap.items()
def keys(self):
return iter(self._rustmap)
def __contains__(self, key):
return key in self._rustmap
def __getitem__(self, item):
return self._rustmap[item]
def __len__(self):
return len(self._rustmap)
def __iter__(self):
return iter(self._rustmap)
# forward for python2,3 compat
iteritems = items
def _opendirstatefile(self):
fp, mode = txnutil.trypending(self._root, self._opener,
self._filename)
if self._pendingmode is not None and self._pendingmode != mode:
fp.close()
raise error.Abort(_('working directory state may be '
'changed parallelly'))
self._pendingmode = mode
return fp
def setparents(self, p1, p2):
self._rustmap.setparents(p1, p2)
self._parents = (p1, p2)
self._dirtyparents = True
def parents(self):
if not self._parents:
try:
fp = self._opendirstatefile()
st = fp.read(40)
fp.close()
except IOError as err:
if err.errno != errno.ENOENT:
raise
# File doesn't exist, so the current state is empty
st = ''
try:
self._parents = self._rustmap.parents(st)
except ValueError:
raise error.Abort(_('working directory state appears '
'damaged!'))
return self._parents
def read(self):
# ignore HG_PENDING because identity is used only for writing
self.identity = util.filestat.frompath(
self._opener.join(self._filename))
try:
fp = self._opendirstatefile()
try:
st = fp.read()
finally:
fp.close()
except IOError as err:
if err.errno != errno.ENOENT:
raise
return
if not st:
return
parse_dirstate = util.nogc(self._rustmap.read)
parents = parse_dirstate(st)
if parents and not self._dirtyparents:
self.setparents(*parents)
def write(self, st, now):
parents = self.parents()
st.write(self._rustmap.write(parents[0], parents[1], now))
st.close()
self._dirtyparents = False
@propertycache
def filefoldmap(self):
"""Returns a dictionary mapping normalized case paths to their
non-normalized versions.
"""
return self._rustmap.filefoldmapasdict()
def hastrackeddir(self, d):
self._dirs # Trigger Python's propertycache
return self._rustmap.hastrackeddir(d)
def hasdir(self, d):
self._dirs # Trigger Python's propertycache
return self._rustmap.hasdir(d)
@propertycache
def _dirs(self):
return self._rustmap.getdirs()
@propertycache
def _alldirs(self):
return self._rustmap.getalldirs()
@propertycache
def identity(self):
self._rustmap
return self.identity
@property
def nonnormalset(self):
nonnorm, otherparents = self._rustmap.nonnormalentries()
return nonnorm
@property
def otherparentset(self):
nonnorm, otherparents = self._rustmap.nonnormalentries()
return otherparents
@propertycache
def dirfoldmap(self):
f = {}
normcase = util.normcase
for name in self._dirs:
f[normcase(name)] = name
return f