( )⚙ D1405 treedirstate: allow absent non-normal sets

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1405

treedirstate: allow absent non-normal sets
ClosedPublic

Authored by mbthomas on Nov 14 2017, 12:39 PM.
Tags
None
Subscribers
durham

Details

Reviewers
durham
Group Reviewers
Restricted Project
Commits
rFBHGX84477debcef7: treedirstate: allow absent non-normal sets
Summary

If the non-normal sets get particularly large, the dirstate root file can get
large again. Avoid this happening by not storing large sets, and instead
recalculating them on-demand as needed. Large sets of non-normal files should
be rare, so this shouldn't be a performance hit in the normal case.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

mbthomas created this revision.Nov 14 2017, 12:39 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptNov 14 2017, 12:39 PM
mbthomas updated this revision to Diff 3602.Nov 17 2017, 9:45 AM
mbthomas updated this revision to Diff 3657.Nov 20 2017, 12:21 PM
durham accepted this revision.Nov 21 2017, 11:13 AM
durham added a subscriber: durham.
durham added inline comments.
treedirstate/__init__.py
346

Can we make 0xfffff... a constant so it's clear it's a marker value?

This revision is now accepted and ready to land.Nov 21 2017, 11:13 AM
mbthomas updated this revision to Diff 3732.Nov 21 2017, 1:35 PM
mbthomas updated this revision to Diff 3773.Nov 22 2017, 1:14 PM
mbthomas updated this revision to Diff 3826.Nov 24 2017, 11:53 AM
mbthomas updated this revision to Diff 3867.Nov 27 2017, 8:00 AM
Closed by commit rFBHGX84477debcef7: treedirstate: allow absent non-normal sets (authored by mbthomas). · Explain WhyNov 28 2017, 6:01 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 3773

rust/treedirstate/src/python.rs

map_pyerr!(py, dirstate.drop_file(filename.data(py)), exc::IOError) map_pyerr!(py, dirstate.drop_file(filename.data(py)), exc::IOError)
} }
// Returns the ID of the root node. // Returns the ID of the root node.
def rootid(&self) -> PyResult<Option<u64>> { def rootid(&self) -> PyResult<Option<u64>> {
Ok(self.dirstate(py).borrow().root_id().map(|id| id.0)) Ok(self.dirstate(py).borrow().root_id().map(|id| id.0))
} }
def computenonnormals(&self, nonnormal: PyObject, otherparent: PyObject) -> PyResult<PyObject> {
let mut dirstate = self.dirstate(py).borrow_mut();
let mut tracked_visitor = |filepath: &Vec<KeyRef>, state: &mut FileState| {
if state.state != 'n' || state.mtime == -1 {
let filename = PyBytes::new(py, &filepath.concat()).into_object();
nonnormal.call(py, (filename,), None).map_err(|e| callback_error(py, e))?;
}
if state.state == 'n' && state.mtime == -2 {
let filename = PyBytes::new(py, &filepath.concat()).into_object();
otherparent.call(py, (filename,), None).map_err(|e| callback_error(py, e))?;
}
Ok(())
};
map_pyerr!(py, dirstate.visit_tracked(&mut tracked_visitor), exc::IOError)?;
let mut removed_visitor = |filepath: &Vec<KeyRef>, _state: &mut FileState| {
let filename = PyBytes::new(py, &filepath.concat()).into_object();
nonnormal.call(py, (filename,), None).map_err(|e| callback_error(py, e))?;
Ok(())
};
map_pyerr!(py, dirstate.visit_removed(&mut removed_visitor), exc::IOError)?;
Ok(py.None())
}
}); });

treedirstate/__init__.py

import struct import struct
from .rusttreedirstate import RustDirstateMap from .rusttreedirstate import RustDirstateMap
dirstateheader = b'########################treedirstate####' dirstateheader = b'########################treedirstate####'
treedirstateversion = 1 treedirstateversion = 1
useinnewrepos = True useinnewrepos = True
# Sentinel length value for when a nonnormalset or otherparentset is absent.
setabsent = 0xffffffff
class _reader(object): class _reader(object):
def __init__(self, data, offset): def __init__(self, data, offset):
self.data = data self.data = data
self.offset = offset self.offset = offset
def readuint(self): def readuint(self):
v = struct.unpack(">L", self.data[self.offset:self.offset + 4]) v = struct.unpack(">L", self.data[self.offset:self.offset + 4])
self.offset += 4 self.offset += 4
""" """
Returns True if the directory exists in the dirstate for either Returns True if the directory exists in the dirstate for either
tracked or removed files. tracked or removed files.
""" """
return self.hastrackeddir(dirname) or self.hasremoveddir(dirname) return self.hastrackeddir(dirname) or self.hasremoveddir(dirname)
def addfile(self, f, oldstate, state, mode, size, mtime): def addfile(self, f, oldstate, state, mode, size, mtime):
self._rmap.addfile(f, oldstate, state, mode, size, mtime) self._rmap.addfile(f, oldstate, state, mode, size, mtime)
if self._nonnormalset is not None:
if state != 'n' or mtime == -1: if state != 'n' or mtime == -1:
self._nonnormalset.add(f) self._nonnormalset.add(f)
else: else:
self._nonnormalset.discard(f) self._nonnormalset.discard(f)
if self._otherparentset is not None:
if size == -2: if size == -2:
self._otherparentset.add(f) self._otherparentset.add(f)
else: else:
self._otherparentset.discard(f) self._otherparentset.discard(f)
def removefile(self, f, oldstate, size): def removefile(self, f, oldstate, size):
self._rmap.removefile(f, oldstate, size) self._rmap.removefile(f, oldstate, size)
if self._nonnormalset is not None:
self._nonnormalset.add(f) self._nonnormalset.add(f)
if size == -2: if size == -2 and self._otherparentset is not None:
self._otherparentset.add(f) self._otherparentset.add(f)
def dropfile(self, f, oldstate): def dropfile(self, f, oldstate):
""" """
Drops a file from the dirstate. Returns True if it was previously Drops a file from the dirstate. Returns True if it was previously
recorded. recorded.
""" """
if self._nonnormalset is not None:
self._nonnormalset.discard(f) self._nonnormalset.discard(f)
if self._otherparentset is not None:
self._otherparentset.discard(f) self._otherparentset.discard(f)
return self._rmap.dropfile(f) return self._rmap.dropfile(f)
def clearambiguoustimes(self, files, now): def clearambiguoustimes(self, files, now):
"""Mark files with an mtime of `now` as being out of date. """Mark files with an mtime of `now` as being out of date.
See mercurial/pure/parsers.py:pack_dirstate in core Mercurial for why See mercurial/pure/parsers.py:pack_dirstate in core Mercurial for why
this is done. this is done.
""" """
def setparents(self, p1, p2): def setparents(self, p1, p2):
""" """
Sets the dirstate parents. Sets the dirstate parents.
""" """
self._parents = (p1, p2) self._parents = (p1, p2)
self._dirtyparents = True self._dirtyparents = True
def _computenonnormals(self):
self._nonnormalset = set()
self._otherparentset = set()
self._rmap.computenonnormals(self._nonnormalset.add, self._otherparentset.add)
@property @property
def nonnormalset(self): def nonnormalset(self):
if self._nonnormalset is None:
self._computenonnormals()
return self._nonnormalset return self._nonnormalset
@property @property
def otherparentset(self): def otherparentset(self):
if self._otherparentset is None:
self._computenonnormals()
return self._otherparentset return self._otherparentset
def getfilefoldmap(self): def getfilefoldmap(self):
"""Returns a dictionary mapping normalized case paths to their """Returns a dictionary mapping normalized case paths to their
non-normalized versions. non-normalized versions.
""" """
raise NotImplementedError() raise NotImplementedError()
copymap = {} copymap = {}
for _i in range(clen): for _i in range(clen):
k = r.readstr() k = r.readstr()
v = r.readstr() v = r.readstr()
copymap[k] = v copymap[k] = v
def readset(): def readset():
slen = r.readuint() slen = r.readuint()
if slen == setabsent:
durhamUnsubmitted
Not Done

Can we make 0xfffff... a constant so it's clear it's a marker value?

durham: Can we make 0xfffff... a constant so it's clear it's a marker value?
return None
s = set() s = set()
for _i in range(slen): for _i in range(slen):
s.add(r.readstr()) s.add(r.readstr())
return s return s
nonnormalset = readset() nonnormalset = readset()
otherparentset = readset() otherparentset = readset()
self.copymap = copymap self.copymap = copymap
self._nonnormalset = nonnormalset self._nonnormalset = nonnormalset
self._otherparentset = otherparentset self._otherparentset = otherparentset
def startwrite(self, tr): def startwrite(self, tr):
# TODO: register map store offset with 'tr' # TODO: register map store offset with 'tr'
pass pass
def write(self, st, now): def write(self, st, now):
"""Write the dirstate to the filehandle st.""" """Write the dirstate to the filehandle st."""
if self._nonnormalset is not None:
nonnormadd = self._nonnormalset.add
else:
def nonnormadd(f):
pass
if self._treeid is None: if self._treeid is None:
self._treeid = '000' self._treeid = '000'
self._rmap.write('dirstate.tree.000', now, self._nonnormalset.add) self._rmap.write('dirstate.tree.000', now, nonnormadd)
else: else:
self._rmap.writedelta(now, self._nonnormalset.add) self._rmap.writedelta(now, nonnormadd)
st.write(self._genrootdata()) st.write(self._genrootdata())
st.close() st.close()
self._dirtyparents = False self._dirtyparents = False
def writeflat(self): def writeflat(self):
st = self._opener("dirstate", "w", atomictemp=True, checkambig=True) st = self._opener("dirstate", "w", atomictemp=True, checkambig=True)
newdmap = {} newdmap = {}
for k, v in self.iteritems(): for k, v in self.iteritems():
w.buffer.write(dirstateheader) w.buffer.write(dirstateheader)
w.writeuint(treedirstateversion) w.writeuint(treedirstateversion)
w.writestr(self._treeid) w.writestr(self._treeid)
w.writeuint(self._rmap.rootid()) w.writeuint(self._rmap.rootid())
w.writeuint(len(self.copymap)) w.writeuint(len(self.copymap))
for k, v in self.copymap.iteritems(): for k, v in self.copymap.iteritems():
w.writestr(k) w.writestr(k)
w.writestr(v) w.writestr(v)
w.writeuint(len(self._nonnormalset))
for v in self._nonnormalset: setthreshold = max(1000, self._rmap.filecount() / 3)
w.writestr(v)
w.writeuint(len(self._otherparentset)) def writeset(s):
for v in self._otherparentset: if s is None or len(s) > setthreshold:
# The set is absent or too large. Mark it as absent.
w.writeuint(setabsent)
else:
w.writeuint(len(s))
for v in s:
w.writestr(v) w.writestr(v)
writeset(self._nonnormalset)
writeset(self._otherparentset)
return w.buffer.getvalue() return w.buffer.getvalue()
def istreedirstate(repo): def istreedirstate(repo):
return (util.safehasattr(repo, 'requirements') and return (util.safehasattr(repo, 'requirements') and
'treedirstate' in repo.requirements) 'treedirstate' in repo.requirements)
def upgrade(ui, repo): def upgrade(ui, repo):
if istreedirstate(repo): if istreedirstate(repo):