( )⚙ D10533 manifests: push down expected node length into the parser

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10533

manifests: push down expected node length into the parser
ClosedPublic

Authored by joerg.sonnenberger on Apr 29 2021, 8:13 PM.

Details

Reviewers
Alphare
Group Reviewers
hg-reviewers
Commits
rHG12450fbea288: manifests: push down expected node length into the parser
Summary

This strictly enforces the node length in the manifest lines according
to what the repository expects. One test case moves large hash testing
into the non-treemanifest part as treemanifests don't provide an
interface for overriding just the node length for now.

Diff Detail

Repository
rHG Mercurial
Branch
default
Lint
No Linters Available
Unit
No Unit Test Coverage

Event Timeline

Alphare accepted this revision.May 3 2021, 7:55 AM
This revision is now accepted and ready to land.May 3 2021, 7:55 AM
marmoute added a subscriber: marmoute.May 3 2021, 8:11 AM

This looks overall sane, but the size of the patch make it hard to be 100% confided nothing fishy sneaked in. I guess this is what testing is for…

Revision Contents
Changeset List

Diff 27219

mercurial/cext/manifest.c

char hash_suffix; char hash_suffix;
bool from_malloc; bool from_malloc;
bool deleted; bool deleted;
} line; } line;
typedef struct { typedef struct {
PyObject_HEAD PyObject_HEAD
PyObject *pydata; PyObject *pydata;
Py_ssize_t nodelen;
line *lines; line *lines;
int numlines; /* number of line entries */ int numlines; /* number of line entries */
int livelines; /* number of non-deleted lines */ int livelines; /* number of non-deleted lines */
int maxlines; /* allocated number of lines */ int maxlines; /* allocated number of lines */
bool dirty; bool dirty;
} lazymanifest; } lazymanifest;
#define MANIFEST_OOM -1 #define MANIFEST_OOM -1
#define MANIFEST_NOT_SORTED -2 #define MANIFEST_NOT_SORTED -2
#define MANIFEST_MALFORMED -3 #define MANIFEST_MALFORMED -3
#define MANIFEST_BOGUS_FILENAME -4 #define MANIFEST_BOGUS_FILENAME -4
#define MANIFEST_TOO_SHORT_LINE -5 #define MANIFEST_TOO_SHORT_LINE -5
/* get the length of the path for a line */ /* get the length of the path for a line */
static Py_ssize_t pathlen(line *l) static Py_ssize_t pathlen(line *l)
{ {
const char *end = memchr(l->start, '\0', l->len); const char *end = memchr(l->start, '\0', l->len);
return (end) ? (Py_ssize_t)(end - l->start) : l->len; return (end) ? (Py_ssize_t)(end - l->start) : l->len;
} }
/* get the node value of a single line */ /* get the node value of a single line */
static PyObject *nodeof(line *l, char *flag) static PyObject *nodeof(Py_ssize_t nodelen, line *l, char *flag)
{ {
char *s = l->start; char *s = l->start;
Py_ssize_t llen = pathlen(l); Py_ssize_t llen = pathlen(l);
Py_ssize_t hlen = l->len - llen - 2; Py_ssize_t hlen = l->len - llen - 2;
Py_ssize_t hlen_raw;
PyObject *hash; PyObject *hash;
if (llen + 1 + 40 + 1 > l->len) { /* path '\0' hash '\n' */ if (llen + 1 + 40 + 1 > l->len) { /* path '\0' hash '\n' */
PyErr_SetString(PyExc_ValueError, "manifest line too short"); PyErr_SetString(PyExc_ValueError, "manifest line too short");
return NULL; return NULL;
} }
/* Detect flags after the hash first. */ /* Detect flags after the hash first. */
switch (s[llen + hlen]) { switch (s[llen + hlen]) {
case 'l': case 'l':
case 't': case 't':
case 'x': case 'x':
*flag = s[llen + hlen]; *flag = s[llen + hlen];
--hlen; --hlen;
break; break;
default: default:
*flag = '\0'; *flag = '\0';
break; break;
} }
switch (hlen) { if (hlen != 2 * nodelen) {
case 40: /* sha1 */
hlen_raw = 20;
break;
case 64: /* new hash */
hlen_raw = 32;
break;
default:
PyErr_SetString(PyExc_ValueError, "invalid node length in manifest"); PyErr_SetString(PyExc_ValueError, "invalid node length in manifest");
return NULL; return NULL;
} }
hash = unhexlify(s + llen + 1, hlen_raw * 2); hash = unhexlify(s + llen + 1, nodelen * 2);
if (!hash) { if (!hash) {
return NULL; return NULL;
} }
if (l->hash_suffix != '\0') { if (l->hash_suffix != '\0') {
char newhash[33]; char newhash[33];
memcpy(newhash, PyBytes_AsString(hash), hlen_raw); memcpy(newhash, PyBytes_AsString(hash), nodelen);
Py_DECREF(hash); Py_DECREF(hash);
newhash[hlen_raw] = l->hash_suffix; newhash[nodelen] = l->hash_suffix;
hash = PyBytes_FromStringAndSize(newhash, hlen_raw+1); hash = PyBytes_FromStringAndSize(newhash, nodelen + 1);
} }
return hash; return hash;
} }
/* get the node hash and flags of a line as a tuple */ /* get the node hash and flags of a line as a tuple */
static PyObject *hashflags(line *l) static PyObject *hashflags(Py_ssize_t nodelen, line *l)
{ {
char flag; char flag;
PyObject *hash = nodeof(l, &flag); PyObject *hash = nodeof(nodelen, l, &flag);
PyObject *flags; PyObject *flags;
PyObject *tup; PyObject *tup;
if (!hash) if (!hash)
return NULL; return NULL;
flags = PyBytes_FromStringAndSize(&flag, flag ? 1 : 0); flags = PyBytes_FromStringAndSize(&flag, flag ? 1 : 0);
if (!flags) { if (!flags) {
Py_DECREF(hash); Py_DECREF(hash);
self->lines = NULL; self->lines = NULL;
self->numlines = 0; self->numlines = 0;
self->maxlines = 0; self->maxlines = 0;
} }
static int lazymanifest_init(lazymanifest *self, PyObject *args) static int lazymanifest_init(lazymanifest *self, PyObject *args)
{ {
char *data; char *data;
Py_ssize_t len; Py_ssize_t nodelen, len;
int err, ret; int err, ret;
PyObject *pydata; PyObject *pydata;
lazymanifest_init_early(self); lazymanifest_init_early(self);
if (!PyArg_ParseTuple(args, "S", &pydata)) { if (!PyArg_ParseTuple(args, "nS", &nodelen, &pydata)) {
return -1; return -1;
} }
err = PyBytes_AsStringAndSize(pydata, &data, &len); if (nodelen != 20 && nodelen != 32) {
/* See fixed buffer in nodeof */
PyErr_Format(PyExc_ValueError, "Unsupported node length");
return -1;
}
self->nodelen = nodelen;
self->dirty = false; self->dirty = false;
err = PyBytes_AsStringAndSize(pydata, &data, &len);
if (err == -1) if (err == -1)
return -1; return -1;
self->pydata = pydata; self->pydata = pydata;
Py_INCREF(self->pydata); Py_INCREF(self->pydata);
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
self->lines = malloc(DEFAULT_LINES * sizeof(line)); self->lines = malloc(DEFAULT_LINES * sizeof(line));
self->maxlines = DEFAULT_LINES; self->maxlines = DEFAULT_LINES;
self->numlines = 0; self->numlines = 0;
} }
/* skip over deleted manifest entries */ /* skip over deleted manifest entries */
} while (self->m->lines[self->pos].deleted); } while (self->m->lines[self->pos].deleted);
return self->m->lines + self->pos; return self->m->lines + self->pos;
} }
static PyObject *lmiter_iterentriesnext(PyObject *o) static PyObject *lmiter_iterentriesnext(PyObject *o)
{ {
lmIter *self = (lmIter *)o;
Py_ssize_t pl; Py_ssize_t pl;
line *l; line *l;
char flag; char flag;
PyObject *ret = NULL, *path = NULL, *hash = NULL, *flags = NULL; PyObject *ret = NULL, *path = NULL, *hash = NULL, *flags = NULL;
l = lmiter_nextline((lmIter *)o); l = lmiter_nextline(self);
if (!l) { if (!l) {
goto done; goto done;
} }
pl = pathlen(l); pl = pathlen(l);
path = PyBytes_FromStringAndSize(l->start, pl); path = PyBytes_FromStringAndSize(l->start, pl);
hash = nodeof(l, &flag); hash = nodeof(self->m->nodelen, l, &flag);
if (!path || !hash) { if (!path || !hash) {
goto done; goto done;
} }
flags = PyBytes_FromStringAndSize(&flag, flag ? 1 : 0); flags = PyBytes_FromStringAndSize(&flag, flag ? 1 : 0);
if (!flags) { if (!flags) {
goto done; goto done;
} }
ret = PyTuple_Pack(3, path, hash, flags); ret = PyTuple_Pack(3, path, hash, flags);
} }
needle.start = PyBytes_AsString(key); needle.start = PyBytes_AsString(key);
hit = bsearch(&needle, self->lines, self->numlines, sizeof(line), hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
&linecmp); &linecmp);
if (!hit || hit->deleted) { if (!hit || hit->deleted) {
PyErr_Format(PyExc_KeyError, "No such manifest entry."); PyErr_Format(PyExc_KeyError, "No such manifest entry.");
return NULL; return NULL;
} }
return hashflags(hit); return hashflags(self->nodelen, hit);
} }
static int lazymanifest_delitem(lazymanifest *self, PyObject *key) static int lazymanifest_delitem(lazymanifest *self, PyObject *key)
{ {
line needle; line needle;
line *hit; line *hit;
if (!PyBytes_Check(key)) { if (!PyBytes_Check(key)) {
PyErr_Format(PyExc_TypeError, PyErr_Format(PyExc_TypeError,
} }
if (PyBytes_AsStringAndSize(key, &path, &plen) == -1) { if (PyBytes_AsStringAndSize(key, &path, &plen) == -1) {
return -1; return -1;
} }
pyhash = PyTuple_GetItem(value, 0); pyhash = PyTuple_GetItem(value, 0);
if (!PyBytes_Check(pyhash)) { if (!PyBytes_Check(pyhash)) {
PyErr_Format(PyExc_TypeError, PyErr_Format(PyExc_TypeError,
"node must be a 20 or 32 bytes string"); "node must be a %zi bytes string", self->nodelen);
return -1; return -1;
} }
hlen = PyBytes_Size(pyhash); hlen = PyBytes_Size(pyhash);
if (hlen != 20 && hlen != 32) { if (hlen != self->nodelen) {
PyErr_Format(PyExc_TypeError, PyErr_Format(PyExc_TypeError,
"node must be a 20 or 32 bytes string"); "node must be a %zi bytes string", self->nodelen);
return -1; return -1;
} }
hash = PyBytes_AsString(pyhash); hash = PyBytes_AsString(pyhash);
pyflags = PyTuple_GetItem(value, 1); pyflags = PyTuple_GetItem(value, 1);
if (!PyBytes_Check(pyflags) || PyBytes_Size(pyflags) > 1) { if (!PyBytes_Check(pyflags) || PyBytes_Size(pyflags) > 1) {
PyErr_Format(PyExc_TypeError, PyErr_Format(PyExc_TypeError,
"flags must a 0 or 1 bytes string"); "flags must a 0 or 1 bytes string");
if (compact(self) != 0) { if (compact(self) != 0) {
goto nomem; goto nomem;
} }
copy = PyObject_New(lazymanifest, &lazymanifestType); copy = PyObject_New(lazymanifest, &lazymanifestType);
if (!copy) { if (!copy) {
goto nomem; goto nomem;
} }
lazymanifest_init_early(copy); lazymanifest_init_early(copy);
copy->nodelen = self->nodelen;
copy->numlines = self->numlines; copy->numlines = self->numlines;
copy->livelines = self->livelines; copy->livelines = self->livelines;
copy->dirty = false; copy->dirty = false;
copy->lines = malloc(self->maxlines *sizeof(line)); copy->lines = malloc(self->maxlines *sizeof(line));
if (!copy->lines) { if (!copy->lines) {
goto nomem; goto nomem;
} }
memcpy(copy->lines, self->lines, self->numlines * sizeof(line)); memcpy(copy->lines, self->lines, self->numlines * sizeof(line));
if (compact(self) != 0) { if (compact(self) != 0) {
goto nomem; goto nomem;
} }
copy = PyObject_New(lazymanifest, &lazymanifestType); copy = PyObject_New(lazymanifest, &lazymanifestType);
if (!copy) { if (!copy) {
goto nomem; goto nomem;
} }
lazymanifest_init_early(copy); lazymanifest_init_early(copy);
copy->nodelen = self->nodelen;
copy->dirty = true; copy->dirty = true;
copy->lines = malloc(self->maxlines * sizeof(line)); copy->lines = malloc(self->maxlines * sizeof(line));
if (!copy->lines) { if (!copy->lines) {
goto nomem; goto nomem;
} }
copy->maxlines = self->maxlines; copy->maxlines = self->maxlines;
copy->numlines = 0; copy->numlines = 0;
copy->pydata = self->pydata; copy->pydata = self->pydata;
result = linecmp(left, right); result = linecmp(left, right);
} }
key = result <= 0 ? key = result <= 0 ?
PyBytes_FromString(left->start) : PyBytes_FromString(left->start) :
PyBytes_FromString(right->start); PyBytes_FromString(right->start);
if (!key) if (!key)
goto nomem; goto nomem;
if (result < 0) { if (result < 0) {
PyObject *l = hashflags(left); PyObject *l = hashflags(self->nodelen, left);
if (!l) { if (!l) {
goto nomem; goto nomem;
} }
outer = PyTuple_Pack(2, l, emptyTup); outer = PyTuple_Pack(2, l, emptyTup);
Py_DECREF(l); Py_DECREF(l);
if (!outer) { if (!outer) {
goto nomem; goto nomem;
} }
PyDict_SetItem(ret, key, outer); PyDict_SetItem(ret, key, outer);
Py_DECREF(outer); Py_DECREF(outer);
sneedle++; sneedle++;
} else if (result > 0) { } else if (result > 0) {
PyObject *r = hashflags(right); PyObject *r = hashflags(self->nodelen, right);
if (!r) { if (!r) {
goto nomem; goto nomem;
} }
outer = PyTuple_Pack(2, emptyTup, r); outer = PyTuple_Pack(2, emptyTup, r);
Py_DECREF(r); Py_DECREF(r);
if (!outer) { if (!outer) {
goto nomem; goto nomem;
} }
PyDict_SetItem(ret, key, outer); PyDict_SetItem(ret, key, outer);
Py_DECREF(outer); Py_DECREF(outer);
oneedle++; oneedle++;
} else { } else {
/* file exists in both manifests */ /* file exists in both manifests */
if (left->len != right->len if (left->len != right->len
|| memcmp(left->start, right->start, left->len) || memcmp(left->start, right->start, left->len)
|| left->hash_suffix != right->hash_suffix) { || left->hash_suffix != right->hash_suffix) {
PyObject *l = hashflags(left); PyObject *l = hashflags(self->nodelen, left);
PyObject *r; PyObject *r;
if (!l) { if (!l) {
goto nomem; goto nomem;
} }
r = hashflags(right); r = hashflags(self->nodelen, right);
if (!r) { if (!r) {
Py_DECREF(l); Py_DECREF(l);
goto nomem; goto nomem;
} }
outer = PyTuple_Pack(2, l, r); outer = PyTuple_Pack(2, l, r);
Py_DECREF(l); Py_DECREF(l);
Py_DECREF(r); Py_DECREF(r);
if (!outer) { if (!outer) {

mercurial/cext/parsers.c

{"fm1readmarkers", fm1readmarkers, METH_VARARGS, {"fm1readmarkers", fm1readmarkers, METH_VARARGS,
"parse v1 obsolete markers\n"}, "parse v1 obsolete markers\n"},
{NULL, NULL}}; {NULL, NULL}};
void dirs_module_init(PyObject *mod); void dirs_module_init(PyObject *mod);
void manifest_module_init(PyObject *mod); void manifest_module_init(PyObject *mod);
void revlog_module_init(PyObject *mod); void revlog_module_init(PyObject *mod);
static const int version = 17; static const int version = 18;
static void module_init(PyObject *mod) static void module_init(PyObject *mod)
{ {
PyObject *capsule = NULL; PyObject *capsule = NULL;
PyModule_AddIntConstant(mod, "version", version); PyModule_AddIntConstant(mod, "version", version);
/* This module constant has two purposes. First, it lets us unit test /* This module constant has two purposes. First, it lets us unit test
* the ImportError raised without hard-coding any error text. This * the ImportError raised without hard-coding any error text. This

mercurial/cext/parsers.pyi

__doc__: str __doc__: str
def __init__(self, source, skipchar: bytes): ... def __init__(self, source, skipchar: bytes): ...
def __iter__(self) -> Iterator[bytes]: ... def __iter__(self) -> Iterator[bytes]: ...
def addpath(self, path: bytes) -> None: ... def addpath(self, path: bytes) -> None: ...
def delpath(self, path: bytes) -> None: ... def delpath(self, path: bytes) -> None: ...
# From manifest.c # From manifest.c
class lazymanifest: class lazymanifest:
def __init__(self, data: bytes): ... def __init__(self, nodelen: int, data: bytes): ...
def __iter__(self) -> Iterator[bytes]: ... def __iter__(self) -> Iterator[bytes]: ...
def __len__(self) -> int: ... def __len__(self) -> int: ...
def __getitem__(self, item: bytes) -> Optional[Tuple[bytes, bytes]]: ... def __getitem__(self, item: bytes) -> Optional[Tuple[bytes, bytes]]: ...
def __setitem__(self, key: bytes, value: Tuple[bytes, bytes]) -> None: ... def __setitem__(self, key: bytes, value: Tuple[bytes, bytes]) -> None: ...
def __delitem__(self, key: bytes) -> None: ... def __delitem__(self, key: bytes) -> None: ...
def iterkeys(self) -> Iterator[bytes]: ... def iterkeys(self) -> Iterator[bytes]: ...

mercurial/manifest.py

parsers = policy.importmod('parsers') parsers = policy.importmod('parsers')
propertycache = util.propertycache propertycache = util.propertycache
# Allow tests to more easily test the alternate path in manifestdict.fastdelta() # Allow tests to more easily test the alternate path in manifestdict.fastdelta()
FASTDELTA_TEXTDIFF_THRESHOLD = 1000 FASTDELTA_TEXTDIFF_THRESHOLD = 1000
def _parse(data): def _parse(nodelen, data):
# This method does a little bit of excessive-looking # This method does a little bit of excessive-looking
# precondition checking. This is so that the behavior of this # precondition checking. This is so that the behavior of this
# class exactly matches its C counterpart to try and help # class exactly matches its C counterpart to try and help
# prevent surprise breakage for anyone that develops against # prevent surprise breakage for anyone that develops against
# the pure version. # the pure version.
if data and data[-1:] != b'\n': if data and data[-1:] != b'\n':
raise ValueError(b'Manifest did not end in a newline.') raise ValueError(b'Manifest did not end in a newline.')
prev = None prev = None
for l in data.splitlines(): for l in data.splitlines():
if prev is not None and prev > l: if prev is not None and prev > l:
raise ValueError(b'Manifest lines not in sorted order.') raise ValueError(b'Manifest lines not in sorted order.')
prev = l prev = l
f, n = l.split(b'\0') f, n = l.split(b'\0')
nl = len(n) nl = len(n)
flags = n[-1:] flags = n[-1:]
if flags in _manifestflags: if flags in _manifestflags:
n = n[:-1] n = n[:-1]
nl -= 1 nl -= 1
else: else:
flags = b'' flags = b''
if nl not in (40, 64): if nl != 2 * nodelen:
raise ValueError(b'Invalid manifest line') raise ValueError(b'Invalid manifest line')
yield f, bin(n), flags yield f, bin(n), flags
def _text(it): def _text(it):
files = [] files = []
lines = [] lines = []
if zeropos == -1 or nlpos == -1 or nlpos < zeropos: if zeropos == -1 or nlpos == -1 or nlpos < zeropos:
raise error.StorageError(b'Invalid manifest line') raise error.StorageError(b'Invalid manifest line')
flags = data[nlpos - 1 : nlpos] flags = data[nlpos - 1 : nlpos]
if flags in _manifestflags: if flags in _manifestflags:
hlen = nlpos - zeropos - 2 hlen = nlpos - zeropos - 2
else: else:
hlen = nlpos - zeropos - 1 hlen = nlpos - zeropos - 1
flags = b'' flags = b''
if hlen not in (40, 64): if hlen != 2 * self.lm._nodelen:
raise error.StorageError(b'Invalid manifest line') raise error.StorageError(b'Invalid manifest line')
hashval = unhexlify( hashval = unhexlify(
data, self.lm.extrainfo[self.pos], zeropos + 1, hlen data, self.lm.extrainfo[self.pos], zeropos + 1, hlen
) )
self.pos += 1 self.pos += 1
return (data[pos:zeropos], hashval, flags) return (data[pos:zeropos], hashval, flags)
__next__ = next __next__ = next
sorted by filename. sorted by filename.
``extradata`` is a list of (key, hash, flags) for entries that were added or ``extradata`` is a list of (key, hash, flags) for entries that were added or
modified since the manifest was created or compacted. modified since the manifest was created or compacted.
""" """
def __init__( def __init__(
self, self,
nodelen,
data, data,
positions=None, positions=None,
extrainfo=None, extrainfo=None,
extradata=None, extradata=None,
hasremovals=False, hasremovals=False,
): ):
self._nodelen = nodelen
if positions is None: if positions is None:
self.positions = self.findlines(data) self.positions = self.findlines(data)
self.extrainfo = [0] * len(self.positions) self.extrainfo = [0] * len(self.positions)
self.data = data self.data = data
self.extradata = [] self.extradata = []
self.hasremovals = False self.hasremovals = False
else: else:
self.positions = positions[:] self.positions = positions[:]
if zeropos == -1 or nlpos == -1 or nlpos < zeropos: if zeropos == -1 or nlpos == -1 or nlpos < zeropos:
raise error.StorageError(b'Invalid manifest line') raise error.StorageError(b'Invalid manifest line')
hlen = nlpos - zeropos - 1 hlen = nlpos - zeropos - 1
flags = data[nlpos - 1 : nlpos] flags = data[nlpos - 1 : nlpos]
if flags in _manifestflags: if flags in _manifestflags:
hlen -= 1 hlen -= 1
else: else:
flags = b'' flags = b''
if hlen not in (40, 64): if hlen != 2 * self._nodelen:
raise error.StorageError(b'Invalid manifest line') raise error.StorageError(b'Invalid manifest line')
hashval = unhexlify(data, self.extrainfo[needle], zeropos + 1, hlen) hashval = unhexlify(data, self.extrainfo[needle], zeropos + 1, hlen)
return (hashval, flags) return (hashval, flags)
def __delitem__(self, key): def __delitem__(self, key):
needle, found = self.bsearch2(key) needle, found = self.bsearch2(key)
if not found: if not found:
raise KeyError raise KeyError
) )
self.extrainfo = ( self.extrainfo = (
self.extrainfo[:needle] + [0] + self.extrainfo[needle:] self.extrainfo[:needle] + [0] + self.extrainfo[needle:]
) )
def copy(self): def copy(self):
# XXX call _compact like in C? # XXX call _compact like in C?
return _lazymanifest( return _lazymanifest(
self._nodelen,
self.data, self.data,
self.positions, self.positions,
self.extrainfo, self.extrainfo,
self.extradata, self.extradata,
self.hasremovals, self.hasremovals,
) )
def _compact(self): def _compact(self):
def __iter__(self): def __iter__(self):
return lazymanifestiter(self) return lazymanifestiter(self)
def __len__(self): def __len__(self):
return len(self.positions) return len(self.positions)
def filtercopy(self, filterfn): def filtercopy(self, filterfn):
# XXX should be optimized # XXX should be optimized
c = _lazymanifest(b'') c = _lazymanifest(self._nodelen, b'')
for f, n, fl in self.iterentries(): for f, n, fl in self.iterentries():
if filterfn(f): if filterfn(f):
c[f] = n, fl c[f] = n, fl
return c return c
try: try:
_lazymanifest = parsers.lazymanifest _lazymanifest = parsers.lazymanifest
except AttributeError: except AttributeError:
pass pass
@interfaceutil.implementer(repository.imanifestdict) @interfaceutil.implementer(repository.imanifestdict)
class manifestdict(object): class manifestdict(object):
def __init__(self, data=b''): def __init__(self, nodelen, data=b''):
self._lm = _lazymanifest(data) self._nodelen = nodelen
self._lm = _lazymanifest(nodelen, data)
def __getitem__(self, key): def __getitem__(self, key):
return self._lm[key][0] return self._lm[key][0]
def find(self, key): def find(self, key):
return self._lm[key] return self._lm[key]
def __len__(self): def __len__(self):
match.bad(fn, None) match.bad(fn, None)
def _matches(self, match): def _matches(self, match):
'''generate a new manifest filtered by the match argument''' '''generate a new manifest filtered by the match argument'''
if match.always(): if match.always():
return self.copy() return self.copy()
if self._filesfastpath(match): if self._filesfastpath(match):
m = manifestdict() m = manifestdict(self._nodelen)
lm = self._lm lm = self._lm
for fn in match.files(): for fn in match.files():
if fn in lm: if fn in lm:
m._lm[fn] = lm[fn] m._lm[fn] = lm[fn]
return m return m
m = manifestdict() m = manifestdict(self._nodelen)
m._lm = self._lm.filtercopy(match) m._lm = self._lm.filtercopy(match)
return m return m
def diff(self, m2, match=None, clean=False): def diff(self, m2, match=None, clean=False):
"""Finds changes between the current manifest and m2. """Finds changes between the current manifest and m2.
Args: Args:
m2: the manifest to which this manifest should be compared. m2: the manifest to which this manifest should be compared.
def flags(self, key): def flags(self, key):
try: try:
return self._lm[key][1] return self._lm[key][1]
except KeyError: except KeyError:
return b'' return b''
def copy(self): def copy(self):
c = manifestdict() c = manifestdict(self._nodelen)
c._lm = self._lm.copy() c._lm = self._lm.copy()
return c return c
def items(self): def items(self):
return (x[:2] for x in self._lm.iterentries()) return (x[:2] for x in self._lm.iterentries())
def iteritems(self): def iteritems(self):
return (x[:2] for x in self._lm.iterentries()) return (x[:2] for x in self._lm.iterentries())
@interfaceutil.implementer(repository.imanifestdict) @interfaceutil.implementer(repository.imanifestdict)
class treemanifest(object): class treemanifest(object):
def __init__(self, nodeconstants, dir=b'', text=b''): def __init__(self, nodeconstants, dir=b'', text=b''):
self._dir = dir self._dir = dir
self.nodeconstants = nodeconstants self.nodeconstants = nodeconstants
self._node = self.nodeconstants.nullid self._node = self.nodeconstants.nullid
self._nodelen = self.nodeconstants.nodelen
self._loadfunc = _noop self._loadfunc = _noop
self._copyfunc = _noop self._copyfunc = _noop
self._dirty = False self._dirty = False
self._dirs = {} self._dirs = {}
self._lazydirs = {} self._lazydirs = {}
# Using _lazymanifest here is a little slower than plain old dicts # Using _lazymanifest here is a little slower than plain old dicts
self._files = {} self._files = {}
self._flags = {} self._flags = {}
_iterativediff(t1, t2, stackls) _iterativediff(t1, t2, stackls)
return result return result
def unmodifiedsince(self, m2): def unmodifiedsince(self, m2):
return not self._dirty and not m2._dirty and self._node == m2._node return not self._dirty and not m2._dirty and self._node == m2._node
def parse(self, text, readsubtree): def parse(self, text, readsubtree):
selflazy = self._lazydirs selflazy = self._lazydirs
for f, n, fl in _parse(text): for f, n, fl in _parse(self._nodelen, text):
if fl == b't': if fl == b't':
f = f + b'/' f = f + b'/'
# False below means "doesn't need to be copied" and can use the # False below means "doesn't need to be copied" and can use the
# cached value from readsubtree directly. # cached value from readsubtree directly.
selflazy[f] = (n, readsubtree, False) selflazy[f] = (n, readsubtree, False)
elif b'/' in f: elif b'/' in f:
# This is a flat manifest, so use __setitem__ and setflag rather # This is a flat manifest, so use __setitem__ and setflag rather
# than assigning directly to _files and _flags, so we can # than assigning directly to _files and _flags, so we can
def update_caches(self, transaction): def update_caches(self, transaction):
return self._rootstore._revlog.update_caches(transaction=transaction) return self._rootstore._revlog.update_caches(transaction=transaction)
@interfaceutil.implementer(repository.imanifestrevisionwritable) @interfaceutil.implementer(repository.imanifestrevisionwritable)
class memmanifestctx(object): class memmanifestctx(object):
def __init__(self, manifestlog): def __init__(self, manifestlog):
self._manifestlog = manifestlog self._manifestlog = manifestlog
self._manifestdict = manifestdict() self._manifestdict = manifestdict(manifestlog.nodeconstants.nodelen)
def _storage(self): def _storage(self):
return self._manifestlog.getstorage(b'') return self._manifestlog.getstorage(b'')
def copy(self): def copy(self):
memmf = memmanifestctx(self._manifestlog) memmf = memmanifestctx(self._manifestlog)
memmf._manifestdict = self.read().copy() memmf._manifestdict = self.read().copy()
return memmf return memmf
return memmf return memmf
@propertycache @propertycache
def parents(self): def parents(self):
return self._storage().parents(self._node) return self._storage().parents(self._node)
def read(self): def read(self):
if self._data is None: if self._data is None:
if self._node == self._manifestlog.nodeconstants.nullid: nc = self._manifestlog.nodeconstants
self._data = manifestdict() if self._node == nc.nullid:
self._data = manifestdict(nc.nodelen)
else: else:
store = self._storage() store = self._storage()
if self._node in store.fulltextcache: if self._node in store.fulltextcache:
text = pycompat.bytestr(store.fulltextcache[self._node]) text = pycompat.bytestr(store.fulltextcache[self._node])
else: else:
text = store.revision(self._node) text = store.revision(self._node)
arraytext = bytearray(text) arraytext = bytearray(text)
store.fulltextcache[self._node] = arraytext store.fulltextcache[self._node] = arraytext
self._data = manifestdict(text) self._data = manifestdict(nc.nodelen, text)
return self._data return self._data
def readfast(self, shallow=False): def readfast(self, shallow=False):
"""Calls either readdelta or read, based on which would be less work. """Calls either readdelta or read, based on which would be less work.
readdelta is called if the delta is against the p1, and therefore can be readdelta is called if the delta is against the p1, and therefore can be
read quickly. read quickly.
If `shallow` is True, nothing changes since this is a flat manifest. If `shallow` is True, nothing changes since this is a flat manifest.
in this manifest, but not in its p1 manifest. This is efficient to read in this manifest, but not in its p1 manifest. This is efficient to read
if the revlog delta is already p1. if the revlog delta is already p1.
Changing the value of `shallow` has no effect on flat manifests. Changing the value of `shallow` has no effect on flat manifests.
""" """
store = self._storage() store = self._storage()
r = store.rev(self._node) r = store.rev(self._node)
d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r)) d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r))
return manifestdict(d) return manifestdict(store.nodeconstants.nodelen, d)
def find(self, key): def find(self, key):
return self.read().find(key) return self.read().find(key)
@interfaceutil.implementer(repository.imanifestrevisionwritable) @interfaceutil.implementer(repository.imanifestrevisionwritable)
class memtreemanifestctx(object): class memtreemanifestctx(object):
def __init__(self, manifestlog, dir=b''): def __init__(self, manifestlog, dir=b''):
subdirectory entry will be reported as it appears in the manifest, i.e. subdirectory entry will be reported as it appears in the manifest, i.e.
the subdirectory will be reported among files and distinguished only by the subdirectory will be reported among files and distinguished only by
its 't' flag. its 't' flag.
""" """
store = self._storage() store = self._storage()
if shallow: if shallow:
r = store.rev(self._node) r = store.rev(self._node)
d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r)) d = mdiff.patchtext(store.revdiff(store.deltaparent(r), r))
return manifestdict(d) return manifestdict(store.nodeconstants.nodelen, d)
else: else:
# Need to perform a slow delta # Need to perform a slow delta
r0 = store.deltaparent(store.rev(self._node)) r0 = store.deltaparent(store.rev(self._node))
m0 = self._manifestlog.get(self._dir, store.node(r0)).read() m0 = self._manifestlog.get(self._dir, store.node(r0)).read()
m1 = self.read() m1 = self.read()
md = treemanifest(self._manifestlog.nodeconstants, dir=self._dir) md = treemanifest(self._manifestlog.nodeconstants, dir=self._dir)
for f, ((n0, fl0), (n1, fl1)) in pycompat.iteritems(m0.diff(m1)): for f, ((n0, fl0), (n1, fl1)) in pycompat.iteritems(m0.diff(m1)):
if n1: if n1:
""" """
store = self._storage() store = self._storage()
r = store.rev(self._node) r = store.rev(self._node)
deltaparent = store.deltaparent(r) deltaparent = store.deltaparent(r)
if deltaparent != nullrev and deltaparent in store.parentrevs(r): if deltaparent != nullrev and deltaparent in store.parentrevs(r):
return self.readdelta(shallow=shallow) return self.readdelta(shallow=shallow)
if shallow: if shallow:
return manifestdict(store.revision(self._node)) return manifestdict(
store.nodeconstants.nodelen, store.revision(self._node)
)
else: else:
return self.read() return self.read()
def find(self, key): def find(self, key):
return self.read().find(key) return self.read().find(key)
class excludeddir(treemanifest): class excludeddir(treemanifest):

mercurial/policy.py

# keep in sync with "version" in C modules # keep in sync with "version" in C modules
_cextversions = { _cextversions = {
('cext', 'base85'): 1, ('cext', 'base85'): 1,
('cext', 'bdiff'): 3, ('cext', 'bdiff'): 3,
('cext', 'mpatch'): 1, ('cext', 'mpatch'): 1,
('cext', 'osutil'): 4, ('cext', 'osutil'): 4,
('cext', 'parsers'): 17, ('cext', 'parsers'): 18,
} }
# map import request to other package or module # map import request to other package or module
_modredirects = { _modredirects = {
('cext', 'charencode'): ('cext', 'parsers'), ('cext', 'charencode'): ('cext', 'parsers'),
('cffi', 'base85'): ('pure', 'base85'), ('cffi', 'base85'): ('pure', 'base85'),
('cffi', 'charencode'): ('pure', 'charencode'), ('cffi', 'charencode'): ('pure', 'charencode'),
('cffi', 'parsers'): ('pure', 'parsers'), ('cffi', 'parsers'): ('pure', 'parsers'),

tests/test-manifest.py

) )
class basemanifesttests(object): class basemanifesttests(object):
def parsemanifest(self, text): def parsemanifest(self, text):
raise NotImplementedError('parsemanifest not implemented by test case') raise NotImplementedError('parsemanifest not implemented by test case')
def testEmptyManifest(self): def testEmptyManifest(self):
m = self.parsemanifest(EMTPY_MANIFEST) m = self.parsemanifest(20, EMTPY_MANIFEST)
self.assertEqual(0, len(m)) self.assertEqual(0, len(m))
self.assertEqual([], list(m)) self.assertEqual([], list(m))
def testManifest(self): def testManifest(self):
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
self.assertEqual([b'bar/baz/qux.py', b'foo'], list(m)) self.assertEqual([b'bar/baz/qux.py', b'foo'], list(m))
self.assertEqual(BIN_HASH_2, m[b'bar/baz/qux.py']) self.assertEqual(BIN_HASH_2, m[b'bar/baz/qux.py'])
self.assertEqual(b'l', m.flags(b'bar/baz/qux.py')) self.assertEqual(b'l', m.flags(b'bar/baz/qux.py'))
self.assertEqual(BIN_HASH_1, m[b'foo']) self.assertEqual(BIN_HASH_1, m[b'foo'])
self.assertEqual(b'', m.flags(b'foo')) self.assertEqual(b'', m.flags(b'foo'))
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
m[b'wat'] m[b'wat']
def testManifestLongHashes(self):
m = self.parsemanifest(b'a\0' + b'f' * 64 + b'\n')
self.assertEqual(binascii.unhexlify(b'f' * 64), m[b'a'])
def testSetItem(self): def testSetItem(self):
want = BIN_HASH_1 want = BIN_HASH_1
m = self.parsemanifest(EMTPY_MANIFEST) m = self.parsemanifest(20, EMTPY_MANIFEST)
m[b'a'] = want m[b'a'] = want
self.assertIn(b'a', m) self.assertIn(b'a', m)
self.assertEqual(want, m[b'a']) self.assertEqual(want, m[b'a'])
self.assertEqual(b'a\0' + HASH_1 + b'\n', m.text()) self.assertEqual(b'a\0' + HASH_1 + b'\n', m.text())
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
m[b'a'] = want m[b'a'] = want
self.assertEqual(want, m[b'a']) self.assertEqual(want, m[b'a'])
self.assertEqual(b'a\0' + HASH_1 + b'\n' + A_SHORT_MANIFEST, m.text()) self.assertEqual(b'a\0' + HASH_1 + b'\n' + A_SHORT_MANIFEST, m.text())
def testSetFlag(self): def testSetFlag(self):
want = b'x' want = b'x'
m = self.parsemanifest(EMTPY_MANIFEST) m = self.parsemanifest(20, EMTPY_MANIFEST)
# first add a file; a file-less flag makes no sense # first add a file; a file-less flag makes no sense
m[b'a'] = BIN_HASH_1 m[b'a'] = BIN_HASH_1
m.setflag(b'a', want) m.setflag(b'a', want)
self.assertEqual(want, m.flags(b'a')) self.assertEqual(want, m.flags(b'a'))
self.assertEqual(b'a\0' + HASH_1 + want + b'\n', m.text()) self.assertEqual(b'a\0' + HASH_1 + want + b'\n', m.text())
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
# first add a file; a file-less flag makes no sense # first add a file; a file-less flag makes no sense
m[b'a'] = BIN_HASH_1 m[b'a'] = BIN_HASH_1
m.setflag(b'a', want) m.setflag(b'a', want)
self.assertEqual(want, m.flags(b'a')) self.assertEqual(want, m.flags(b'a'))
self.assertEqual( self.assertEqual(
b'a\0' + HASH_1 + want + b'\n' + A_SHORT_MANIFEST, m.text() b'a\0' + HASH_1 + want + b'\n' + A_SHORT_MANIFEST, m.text()
) )
def testCopy(self): def testCopy(self):
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
m[b'a'] = BIN_HASH_1 m[b'a'] = BIN_HASH_1
m2 = m.copy() m2 = m.copy()
del m del m
del m2 # make sure we don't double free() anything del m2 # make sure we don't double free() anything
def testCompaction(self): def testCompaction(self):
unhex = binascii.unhexlify unhex = binascii.unhexlify
h1, h2 = unhex(HASH_1), unhex(HASH_2) h1, h2 = unhex(HASH_1), unhex(HASH_2)
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
m[b'alpha'] = h1 m[b'alpha'] = h1
m[b'beta'] = h2 m[b'beta'] = h2
del m[b'foo'] del m[b'foo']
want = b'alpha\0%s\nbar/baz/qux.py\0%sl\nbeta\0%s\n' % ( want = b'alpha\0%s\nbar/baz/qux.py\0%sl\nbeta\0%s\n' % (
HASH_1, HASH_1,
HASH_2, HASH_2,
HASH_2, HASH_2,
) )
self.assertEqual(want, m.text()) self.assertEqual(want, m.text())
self.assertEqual(3, len(m)) self.assertEqual(3, len(m))
self.assertEqual([b'alpha', b'bar/baz/qux.py', b'beta'], list(m)) self.assertEqual([b'alpha', b'bar/baz/qux.py', b'beta'], list(m))
self.assertEqual(h1, m[b'alpha']) self.assertEqual(h1, m[b'alpha'])
self.assertEqual(h2, m[b'bar/baz/qux.py']) self.assertEqual(h2, m[b'bar/baz/qux.py'])
self.assertEqual(h2, m[b'beta']) self.assertEqual(h2, m[b'beta'])
self.assertEqual(b'', m.flags(b'alpha')) self.assertEqual(b'', m.flags(b'alpha'))
self.assertEqual(b'l', m.flags(b'bar/baz/qux.py')) self.assertEqual(b'l', m.flags(b'bar/baz/qux.py'))
self.assertEqual(b'', m.flags(b'beta')) self.assertEqual(b'', m.flags(b'beta'))
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
m[b'foo'] m[b'foo']
def testMatchException(self): def testMatchException(self):
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
match = matchmod.match(util.localpath(b'/repo'), b'', [b're:.*']) match = matchmod.match(util.localpath(b'/repo'), b'', [b're:.*'])
def filt(path): def filt(path):
if path == b'foo': if path == b'foo':
assert False assert False
return True return True
match.matchfn = filt match.matchfn = filt
with self.assertRaises(AssertionError): with self.assertRaises(AssertionError):
m._matches(match) m._matches(match)
def testRemoveItem(self): def testRemoveItem(self):
m = self.parsemanifest(A_SHORT_MANIFEST) m = self.parsemanifest(20, A_SHORT_MANIFEST)
del m[b'foo'] del m[b'foo']
with self.assertRaises(KeyError): with self.assertRaises(KeyError):
m[b'foo'] m[b'foo']
self.assertEqual(1, len(m)) self.assertEqual(1, len(m))
self.assertEqual(1, len(list(m))) self.assertEqual(1, len(list(m)))
# now restore and make sure everything works right # now restore and make sure everything works right
m[b'foo'] = b'a' * 20 m[b'foo'] = b'a' * 20
self.assertEqual(2, len(m)) self.assertEqual(2, len(m))
self.assertEqual(2, len(list(m))) self.assertEqual(2, len(list(m)))
def testManifestDiff(self): def testManifestDiff(self):
MISSING = (None, b'') MISSING = (None, b'')
addl = b'z-only-in-left\0' + HASH_1 + b'\n' addl = b'z-only-in-left\0' + HASH_1 + b'\n'
addr = b'z-only-in-right\0' + HASH_2 + b'x\n' addr = b'z-only-in-right\0' + HASH_2 + b'x\n'
left = self.parsemanifest( left = self.parsemanifest(
A_SHORT_MANIFEST.replace(HASH_1, HASH_3 + b'x') + addl 20, A_SHORT_MANIFEST.replace(HASH_1, HASH_3 + b'x') + addl
) )
right = self.parsemanifest(A_SHORT_MANIFEST + addr) right = self.parsemanifest(20, A_SHORT_MANIFEST + addr)
want = { want = {
b'foo': ((BIN_HASH_3, b'x'), (BIN_HASH_1, b'')), b'foo': ((BIN_HASH_3, b'x'), (BIN_HASH_1, b'')),
b'z-only-in-left': ((BIN_HASH_1, b''), MISSING), b'z-only-in-left': ((BIN_HASH_1, b''), MISSING),
b'z-only-in-right': (MISSING, (BIN_HASH_2, b'x')), b'z-only-in-right': (MISSING, (BIN_HASH_2, b'x')),
} }
self.assertEqual(want, left.diff(right)) self.assertEqual(want, left.diff(right))
want = { want = {
b'bar/baz/qux.py': (MISSING, (BIN_HASH_2, b'l')), b'bar/baz/qux.py': (MISSING, (BIN_HASH_2, b'l')),
b'foo': (MISSING, (BIN_HASH_3, b'x')), b'foo': (MISSING, (BIN_HASH_3, b'x')),
b'z-only-in-left': (MISSING, (BIN_HASH_1, b'')), b'z-only-in-left': (MISSING, (BIN_HASH_1, b'')),
} }
self.assertEqual(want, self.parsemanifest(EMTPY_MANIFEST).diff(left)) self.assertEqual(
want, self.parsemanifest(20, EMTPY_MANIFEST).diff(left)
)
want = { want = {
b'bar/baz/qux.py': ((BIN_HASH_2, b'l'), MISSING), b'bar/baz/qux.py': ((BIN_HASH_2, b'l'), MISSING),
b'foo': ((BIN_HASH_3, b'x'), MISSING), b'foo': ((BIN_HASH_3, b'x'), MISSING),
b'z-only-in-left': ((BIN_HASH_1, b''), MISSING), b'z-only-in-left': ((BIN_HASH_1, b''), MISSING),
} }
self.assertEqual(want, left.diff(self.parsemanifest(EMTPY_MANIFEST))) self.assertEqual(
want, left.diff(self.parsemanifest(20, EMTPY_MANIFEST))
)
copy = right.copy() copy = right.copy()
del copy[b'z-only-in-right'] del copy[b'z-only-in-right']
del right[b'foo'] del right[b'foo']
want = { want = {
b'foo': (MISSING, (BIN_HASH_1, b'')), b'foo': (MISSING, (BIN_HASH_1, b'')),
b'z-only-in-right': ((BIN_HASH_2, b'x'), MISSING), b'z-only-in-right': ((BIN_HASH_2, b'x'), MISSING),
} }
self.assertEqual(want, right.diff(copy)) self.assertEqual(want, right.diff(copy))
short = self.parsemanifest(A_SHORT_MANIFEST) short = self.parsemanifest(20, A_SHORT_MANIFEST)
pruned = short.copy() pruned = short.copy()
del pruned[b'foo'] del pruned[b'foo']
want = { want = {
b'foo': ((BIN_HASH_1, b''), MISSING), b'foo': ((BIN_HASH_1, b''), MISSING),
} }
self.assertEqual(want, short.diff(pruned)) self.assertEqual(want, short.diff(pruned))
want = { want = {
b'foo': (MISSING, (BIN_HASH_1, b'')), b'foo': (MISSING, (BIN_HASH_1, b'')),
} }
self.assertEqual(want, pruned.diff(short)) self.assertEqual(want, pruned.diff(short))
want = { want = {
b'bar/baz/qux.py': None, b'bar/baz/qux.py': None,
b'foo': (MISSING, (BIN_HASH_1, b'')), b'foo': (MISSING, (BIN_HASH_1, b'')),
} }
self.assertEqual(want, pruned.diff(short, clean=True)) self.assertEqual(want, pruned.diff(short, clean=True))
def testReversedLines(self): def testReversedLines(self):
backwards = b''.join( backwards = b''.join(
l + b'\n' for l in reversed(A_SHORT_MANIFEST.split(b'\n')) if l l + b'\n' for l in reversed(A_SHORT_MANIFEST.split(b'\n')) if l
) )
try: try:
self.parsemanifest(backwards) self.parsemanifest(20, backwards)
self.fail('Should have raised ValueError') self.fail('Should have raised ValueError')
except ValueError as v: except ValueError as v:
self.assertIn('Manifest lines not in sorted order.', str(v)) self.assertIn('Manifest lines not in sorted order.', str(v))
def testNoTerminalNewline(self): def testNoTerminalNewline(self):
try: try:
self.parsemanifest(A_SHORT_MANIFEST + b'wat') self.parsemanifest(20, A_SHORT_MANIFEST + b'wat')
self.fail('Should have raised ValueError') self.fail('Should have raised ValueError')
except ValueError as v: except ValueError as v:
self.assertIn('Manifest did not end in a newline.', str(v)) self.assertIn('Manifest did not end in a newline.', str(v))
def testNoNewLineAtAll(self): def testNoNewLineAtAll(self):
try: try:
self.parsemanifest(b'wat') self.parsemanifest(20, b'wat')
self.fail('Should have raised ValueError') self.fail('Should have raised ValueError')
except ValueError as v: except ValueError as v:
self.assertIn('Manifest did not end in a newline.', str(v)) self.assertIn('Manifest did not end in a newline.', str(v))
def testHugeManifest(self): def testHugeManifest(self):
m = self.parsemanifest(A_HUGE_MANIFEST) m = self.parsemanifest(20, A_HUGE_MANIFEST)
self.assertEqual(HUGE_MANIFEST_ENTRIES, len(m)) self.assertEqual(HUGE_MANIFEST_ENTRIES, len(m))
self.assertEqual(len(m), len(list(m))) self.assertEqual(len(m), len(list(m)))
def testMatchesMetadata(self): def testMatchesMetadata(self):
"""Tests matches() for a few specific files to make sure that both """Tests matches() for a few specific files to make sure that both
the set of files as well as their flags and nodeids are correct in the set of files as well as their flags and nodeids are correct in
the resulting manifest.""" the resulting manifest."""
m = self.parsemanifest(A_HUGE_MANIFEST) m = self.parsemanifest(20, A_HUGE_MANIFEST)
match = matchmod.exact([b'file1', b'file200', b'file300']) match = matchmod.exact([b'file1', b'file200', b'file300'])
m2 = m._matches(match) m2 = m._matches(match)
w = (b'file1\0%sx\n' b'file200\0%sl\n' b'file300\0%s\n') % ( w = (b'file1\0%sx\n' b'file200\0%sl\n' b'file300\0%s\n') % (
HASH_2, HASH_2,
HASH_1, HASH_1,
HASH_1, HASH_1,
) )
self.assertEqual(w, m2.text()) self.assertEqual(w, m2.text())
def testMatchesNonexistentFile(self): def testMatchesNonexistentFile(self):
"""Tests matches() for a small set of specific files, including one """Tests matches() for a small set of specific files, including one
nonexistent file to make sure in only matches against existing files. nonexistent file to make sure in only matches against existing files.
""" """
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.exact( match = matchmod.exact(
[b'a/b/c/bar.txt', b'a/b/d/qux.py', b'readme.txt', b'nonexistent'] [b'a/b/c/bar.txt', b'a/b/d/qux.py', b'readme.txt', b'nonexistent']
) )
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual( self.assertEqual(
[b'a/b/c/bar.txt', b'a/b/d/qux.py', b'readme.txt'], m2.keys() [b'a/b/c/bar.txt', b'a/b/d/qux.py', b'readme.txt'], m2.keys()
) )
def testMatchesNonexistentDirectory(self): def testMatchesNonexistentDirectory(self):
"""Tests matches() for a relpath match on a directory that doesn't """Tests matches() for a relpath match on a directory that doesn't
actually exist.""" actually exist."""
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.match( match = matchmod.match(
util.localpath(b'/repo'), b'', [b'a/f'], default=b'relpath' util.localpath(b'/repo'), b'', [b'a/f'], default=b'relpath'
) )
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual([], m2.keys()) self.assertEqual([], m2.keys())
def testMatchesExactLarge(self): def testMatchesExactLarge(self):
"""Tests matches() for files matching a large list of exact files.""" """Tests matches() for files matching a large list of exact files."""
m = self.parsemanifest(A_HUGE_MANIFEST) m = self.parsemanifest(20, A_HUGE_MANIFEST)
flist = m.keys()[80:300] flist = m.keys()[80:300]
match = matchmod.exact(flist) match = matchmod.exact(flist)
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual(flist, m2.keys()) self.assertEqual(flist, m2.keys())
def testMatchesFull(self): def testMatchesFull(self):
'''Tests matches() for what should be a full match.''' '''Tests matches() for what should be a full match.'''
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.match(util.localpath(b'/repo'), b'', [b'']) match = matchmod.match(util.localpath(b'/repo'), b'', [b''])
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual(m.keys(), m2.keys()) self.assertEqual(m.keys(), m2.keys())
def testMatchesDirectory(self): def testMatchesDirectory(self):
"""Tests matches() on a relpath match on a directory, which should """Tests matches() on a relpath match on a directory, which should
match against all files within said directory.""" match against all files within said directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.match( match = matchmod.match(
util.localpath(b'/repo'), b'', [b'a/b'], default=b'relpath' util.localpath(b'/repo'), b'', [b'a/b'], default=b'relpath'
) )
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual( self.assertEqual(
[ [
], ],
m2.keys(), m2.keys(),
) )
def testMatchesExactPath(self): def testMatchesExactPath(self):
"""Tests matches() on an exact match on a directory, which should """Tests matches() on an exact match on a directory, which should
result in an empty manifest because you can't perform an exact match result in an empty manifest because you can't perform an exact match
against a directory.""" against a directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.exact([b'a/b']) match = matchmod.exact([b'a/b'])
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual([], m2.keys()) self.assertEqual([], m2.keys())
def testMatchesCwd(self): def testMatchesCwd(self):
"""Tests matches() on a relpath match with the current directory ('.') """Tests matches() on a relpath match with the current directory ('.')
when not in the root directory.""" when not in the root directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.match( match = matchmod.match(
util.localpath(b'/repo'), b'a/b', [b'.'], default=b'relpath' util.localpath(b'/repo'), b'a/b', [b'.'], default=b'relpath'
) )
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual( self.assertEqual(
[ [
b'a/b/c/bar.py', b'a/b/c/bar.py',
b'a/b/c/bar.txt', b'a/b/c/bar.txt',
b'a/b/c/foo.py', b'a/b/c/foo.py',
b'a/b/c/foo.txt', b'a/b/c/foo.txt',
b'a/b/d/baz.py', b'a/b/d/baz.py',
b'a/b/d/qux.py', b'a/b/d/qux.py',
b'a/b/d/ten.txt', b'a/b/d/ten.txt',
b'a/b/dog.py', b'a/b/dog.py',
b'a/b/fish.py', b'a/b/fish.py',
], ],
m2.keys(), m2.keys(),
) )
def testMatchesWithPattern(self): def testMatchesWithPattern(self):
"""Tests matches() for files matching a pattern that reside """Tests matches() for files matching a pattern that reside
deeper than the specified directory.""" deeper than the specified directory."""
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
match = matchmod.match(util.localpath(b'/repo'), b'', [b'a/b/*/*.txt']) match = matchmod.match(util.localpath(b'/repo'), b'', [b'a/b/*/*.txt'])
m2 = m._matches(match) m2 = m._matches(match)
self.assertEqual( self.assertEqual(
[b'a/b/c/bar.txt', b'a/b/c/foo.txt', b'a/b/d/ten.txt'], m2.keys() [b'a/b/c/bar.txt', b'a/b/c/foo.txt', b'a/b/d/ten.txt'], m2.keys()
) )
class testmanifestdict(unittest.TestCase, basemanifesttests): class testmanifestdict(unittest.TestCase, basemanifesttests):
def parsemanifest(self, text): def parsemanifest(self, nodelen, text):
return manifestmod.manifestdict(text) return manifestmod.manifestdict(nodelen, text)
def testManifestLongHashes(self):
m = self.parsemanifest(32, b'a\0' + b'f' * 64 + b'\n')
self.assertEqual(binascii.unhexlify(b'f' * 64), m[b'a'])
def testObviouslyBogusManifest(self): def testObviouslyBogusManifest(self):
# This is a 163k manifest that came from oss-fuzz. It was a # This is a 163k manifest that came from oss-fuzz. It was a
# timeout there, but when run normally it doesn't seem to # timeout there, but when run normally it doesn't seem to
# present any particular slowness. # present any particular slowness.
data = zlib.decompress( data = zlib.decompress(
b'x\x9c\xed\xce;\n\x83\x00\x10\x04\xd0\x8deNa\x93~\xf1\x03\xc9q\xf4' b'x\x9c\xed\xce;\n\x83\x00\x10\x04\xd0\x8deNa\x93~\xf1\x03\xc9q\xf4'
b'\x14\xeaU\xbdB\xda\xd4\xe6Cj\xc1FA\xde+\x86\xe9f\xa2\xfci\xbb\xfb' b'\x14\xeaU\xbdB\xda\xd4\xe6Cj\xc1FA\xde+\x86\xe9f\xa2\xfci\xbb\xfb'
b'\xa3\xef\xea\xba\xca\x7fk\x86q\x9a\xc6\xc8\xcc&\xb3\xcf\xf8\xb8|#' b'\xa3\xef\xea\xba\xca\x7fk\x86q\x9a\xc6\xc8\xcc&\xb3\xcf\xf8\xb8|#'
b'\x8a9\x00\xd8\xe6v\xf4\x01N\xe1\n\x00\x00\x00\x00\x00\x00\x00\x00' b'\x8a9\x00\xd8\xe6v\xf4\x01N\xe1\n\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
b'\x00\x00\xc0\x8aey\x1d}\x01\xd8\xe0\xb9\xf3\xde\x1b\xcf\x17' b'\x00\x00\xc0\x8aey\x1d}\x01\xd8\xe0\xb9\xf3\xde\x1b\xcf\x17'
b'\xac\xbe' b'\xac\xbe'
) )
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self.parsemanifest(data) self.parsemanifest(20, data)
class testtreemanifest(unittest.TestCase, basemanifesttests): class testtreemanifest(unittest.TestCase, basemanifesttests):
def parsemanifest(self, text): def parsemanifest(self, nodelen, text):
return manifestmod.treemanifest(sha1nodeconstants, b'', text) return manifestmod.treemanifest(sha1nodeconstants, b'', text)
def testWalkSubtrees(self): def testWalkSubtrees(self):
m = self.parsemanifest(A_DEEPER_MANIFEST) m = self.parsemanifest(20, A_DEEPER_MANIFEST)
dirs = [s._dir for s in m.walksubtrees()] dirs = [s._dir for s in m.walksubtrees()]
self.assertEqual( self.assertEqual(
sorted( sorted(
[b'', b'a/', b'a/c/', b'a/d/', b'a/b/', b'a/b/c/', b'a/b/d/'] [b'', b'a/', b'a/c/', b'a/d/', b'a/b/', b'a/b/c/', b'a/b/d/']
), ),
sorted(dirs), sorted(dirs),
) )
match = matchmod.match(util.localpath(b'/repo'), b'', [b'path:a/b/']) match = matchmod.match(util.localpath(b'/repo'), b'', [b'path:a/b/'])
dirs = [s._dir for s in m.walksubtrees(matcher=match)] dirs = [s._dir for s in m.walksubtrees(matcher=match)]
self.assertEqual(sorted([b'a/b/', b'a/b/c/', b'a/b/d/']), sorted(dirs)) self.assertEqual(sorted([b'a/b/', b'a/b/c/', b'a/b/d/']), sorted(dirs))
if __name__ == '__main__': if __name__ == '__main__':
silenttestrunner.main(__name__) silenttestrunner.main(__name__)