Various walk functions return (revlog_type, decoded, encoded) where
decoded could be None. But no-one cares about encoded and expects
unencoded to be present, except verify (because this can only happen
with old repo formats).
Simplify all this by either failing outright if a decoding a filename
fails (instead of almost certainly failing with a type error due to
treating None as a bytes), or skipping the filename but providing in
an out argument for hg verify.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
This looks good aside from the gettext comment, but will (functionally) need to be rebased on top of 5.9 since the fix for issue6528 uses datafiles in mercurial/revlogutils/rewrite.py.
| mercurial/store.py | ||
|---|---|---|
| 589 | This error message should use gettext (i.e. _()) | |
| Path | Packages | |||
|---|---|---|---|---|
| M | hgext/narrow/narrowcommands.py (2 lines) | |||
| M | hgext/remotefilelog/contentstore.py (2 lines) | |||
| M | hgext/remotefilelog/remotefilelogserver.py (10 lines) | |||
| M | mercurial/repair.py (2 lines) | |||
| M | mercurial/revlogutils/rewrite.py (2 lines) | |||
| M | mercurial/store.py (47 lines) | |||
| M | mercurial/streamclone.py (4 lines) | |||
| M | mercurial/upgrade_utils/engine.py (2 lines) | |||
| M | mercurial/verify.py (18 lines) | |||
| M | mercurial/wireprotov2server.py (2 lines) | |||
| M | tests/simplestorerepo.py (14 lines) |
| Status | Author | Revision | |
|---|---|---|---|
| Closed | valentin.gatienbaron | ||
| Closed | valentin.gatienbaron | ||
| Closed | valentin.gatienbaron |
| if backup: | if backup: | ||||
| ui.status(_(b'moving unwanted changesets to backup\n')) | ui.status(_(b'moving unwanted changesets to backup\n')) | ||||
| else: | else: | ||||
| ui.status(_(b'deleting unwanted changesets\n')) | ui.status(_(b'deleting unwanted changesets\n')) | ||||
| with ui.configoverride(overrides, b'narrow'): | with ui.configoverride(overrides, b'narrow'): | ||||
| repair.strip(ui, unfi, tostrip, topic=b'narrow', backup=backup) | repair.strip(ui, unfi, tostrip, topic=b'narrow', backup=backup) | ||||
| todelete = [] | todelete = [] | ||||
| for t, f, f2, size in repo.store.datafiles(): | for t, f, size in repo.store.datafiles(): | ||||
| if f.startswith(b'data/'): | if f.startswith(b'data/'): | ||||
| file = f[5:-2] | file = f[5:-2] | ||||
| if not newmatch(file): | if not newmatch(file): | ||||
| todelete.append(f) | todelete.append(f) | ||||
| elif f.startswith(b'meta/'): | elif f.startswith(b'meta/'): | ||||
| dir = f[5:-13] | dir = f[5:-13] | ||||
| dirs = sorted(pathutil.dirs({dir})) + [dir] | dirs = sorted(pathutil.dirs({dir})) + [dir] | ||||
| include = True | include = True | ||||
| if linkrev < startlinkrev: | if linkrev < startlinkrev: | ||||
| break | break | ||||
| if linkrev > endlinkrev: | if linkrev > endlinkrev: | ||||
| continue | continue | ||||
| node = rl.node(rev) | node = rl.node(rev) | ||||
| ledger.markdataentry(self, treename, node) | ledger.markdataentry(self, treename, node) | ||||
| ledger.markhistoryentry(self, treename, node) | ledger.markhistoryentry(self, treename, node) | ||||
| for t, path, encoded, size in self._store.datafiles(): | for t, path, size in self._store.datafiles(): | ||||
| if path[:5] != b'meta/' or path[-2:] != b'.i': | if path[:5] != b'meta/' or path[-2:] != b'.i': | ||||
| continue | continue | ||||
| treename = path[5 : -len(b'/00manifest')] | treename = path[5 : -len(b'/00manifest')] | ||||
| rl = revlog.revlog(self._svfs, indexfile=path[:-2]) | rl = revlog.revlog(self._svfs, indexfile=path[:-2]) | ||||
| for rev in pycompat.xrange(len(rl) - 1, -1, -1): | for rev in pycompat.xrange(len(rl) - 1, -1, -1): | ||||
| linkrev = rl.linkrev(rev) | linkrev = rl.linkrev(rev) | ||||
| fp = p + b'/' + f | fp = p + b'/' + f | ||||
| if kind == stat.S_IFREG: | if kind == stat.S_IFREG: | ||||
| if not fp.endswith(b'.i') and not fp.endswith( | if not fp.endswith(b'.i') and not fp.endswith( | ||||
| b'.d' | b'.d' | ||||
| ): | ): | ||||
| n = util.pconvert(fp[striplen:]) | n = util.pconvert(fp[striplen:]) | ||||
| d = store.decodedir(n) | d = store.decodedir(n) | ||||
| t = store.FILETYPE_OTHER | t = store.FILETYPE_OTHER | ||||
| yield (t, d, n, st.st_size) | yield (t, d, st.st_size) | ||||
| if kind == stat.S_IFDIR: | if kind == stat.S_IFDIR: | ||||
| visit.append(fp) | visit.append(fp) | ||||
| if scmutil.istreemanifest(repo): | if scmutil.istreemanifest(repo): | ||||
| for (t, u, e, s) in repo.store.datafiles(): | for (t, u, s) in repo.store.datafiles(): | ||||
| if u.startswith(b'meta/') and ( | if u.startswith(b'meta/') and ( | ||||
| u.endswith(b'.i') or u.endswith(b'.d') | u.endswith(b'.i') or u.endswith(b'.d') | ||||
| ): | ): | ||||
| yield (t, u, e, s) | yield (t, u, s) | ||||
| # Return .d and .i files that do not match the shallow pattern | # Return .d and .i files that do not match the shallow pattern | ||||
| match = state.match | match = state.match | ||||
| if match and not match.always(): | if match and not match.always(): | ||||
| for (t, u, e, s) in repo.store.datafiles(): | for (t, u, s) in repo.store.datafiles(): | ||||
| f = u[5:-2] # trim data/... and .i/.d | f = u[5:-2] # trim data/... and .i/.d | ||||
| if not state.match(f): | if not state.match(f): | ||||
| yield (t, u, e, s) | yield (t, u, s) | ||||
| for x in repo.store.topfiles(): | for x in repo.store.topfiles(): | ||||
| if state.noflatmf and x[1][:11] == b'00manifest.': | if state.noflatmf and x[1][:11] == b'00manifest.': | ||||
| continue | continue | ||||
| yield x | yield x | ||||
| elif shallowutil.isenabled(repo): | elif shallowutil.isenabled(repo): | ||||
| # don't allow cloning from a shallow repo to a full repo | # don't allow cloning from a shallow repo to a full repo | ||||
| revlog.strip(striprev, tr) | revlog.strip(striprev, tr) | ||||
| def manifestrevlogs(repo): | def manifestrevlogs(repo): | ||||
| yield repo.manifestlog.getstorage(b'') | yield repo.manifestlog.getstorage(b'') | ||||
| if scmutil.istreemanifest(repo): | if scmutil.istreemanifest(repo): | ||||
| # This logic is safe if treemanifest isn't enabled, but also | # This logic is safe if treemanifest isn't enabled, but also | ||||
| # pointless, so we skip it if treemanifest isn't enabled. | # pointless, so we skip it if treemanifest isn't enabled. | ||||
| for t, unencoded, encoded, size in repo.store.datafiles(): | for t, unencoded, size in repo.store.datafiles(): | ||||
| if unencoded.startswith(b'meta/') and unencoded.endswith( | if unencoded.startswith(b'meta/') and unencoded.endswith( | ||||
| b'00manifest.i' | b'00manifest.i' | ||||
| ): | ): | ||||
| dir = unencoded[5:-12] | dir = unencoded[5:-12] | ||||
| yield repo.manifestlog.getstorage(dir) | yield repo.manifestlog.getstorage(dir) | ||||
| def rebuildfncache(ui, repo, only_data=False): | def rebuildfncache(ui, repo, only_data=False): | ||||
| if from_report: | if from_report: | ||||
| return _from_report(ui, repo, context, from_report, dry_run) | return _from_report(ui, repo, context, from_report, dry_run) | ||||
| report_entries = [] | report_entries = [] | ||||
| with context(): | with context(): | ||||
| files = list( | files = list( | ||||
| (file_type, path) | (file_type, path) | ||||
| for (file_type, path, _e, _s) in repo.store.datafiles() | for (file_type, path, _s) in repo.store.datafiles() | ||||
| if path.endswith(b'.i') and file_type & store.FILEFLAGS_FILELOG | if path.endswith(b'.i') and file_type & store.FILEFLAGS_FILELOG | ||||
| ) | ) | ||||
| progress = ui.makeprogress( | progress = ui.makeprogress( | ||||
| _(b"looking for affected revisions"), | _(b"looking for affected revisions"), | ||||
| unit=_(b"filelogs"), | unit=_(b"filelogs"), | ||||
| total=len(files), | total=len(files), | ||||
| ) | ) | ||||
| This function will obtain a repository lock to ensure a consistent view of | This function will obtain a repository lock to ensure a consistent view of | ||||
| the store is captured. It therefore may raise LockError. | the store is captured. It therefore may raise LockError. | ||||
| """ | """ | ||||
| entries = [] | entries = [] | ||||
| total_bytes = 0 | total_bytes = 0 | ||||
| # Get consistent snapshot of repo, lock during scan. | # Get consistent snapshot of repo, lock during scan. | ||||
| with repo.lock(): | with repo.lock(): | ||||
| repo.ui.debug(b'scanning\n') | repo.ui.debug(b'scanning\n') | ||||
| for file_type, name, ename, size in _walkstreamfiles(repo): | for file_type, name, size in _walkstreamfiles(repo): | ||||
| if size: | if size: | ||||
| entries.append((name, size)) | entries.append((name, size)) | ||||
| total_bytes += size | total_bytes += size | ||||
| _test_sync_point_walk_1(repo) | _test_sync_point_walk_1(repo) | ||||
| _test_sync_point_walk_2(repo) | _test_sync_point_walk_2(repo) | ||||
| repo.ui.debug( | repo.ui.debug( | ||||
| b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes) | b'%d files, %d bytes to transfer\n' % (len(entries), total_bytes) | ||||
| assert repo._currentlock(repo._lockref) is not None | assert repo._currentlock(repo._lockref) is not None | ||||
| entries = [] | entries = [] | ||||
| totalfilesize = 0 | totalfilesize = 0 | ||||
| matcher = None | matcher = None | ||||
| if includes or excludes: | if includes or excludes: | ||||
| matcher = narrowspec.match(repo.root, includes, excludes) | matcher = narrowspec.match(repo.root, includes, excludes) | ||||
| for rl_type, name, ename, size in _walkstreamfiles(repo, matcher): | for rl_type, name, size in _walkstreamfiles(repo, matcher): | ||||
| if size: | if size: | ||||
| ft = _fileappend | ft = _fileappend | ||||
| if rl_type & store.FILEFLAGS_VOLATILE: | if rl_type & store.FILEFLAGS_VOLATILE: | ||||
| ft = _filefull | ft = _filefull | ||||
| entries.append((_srcstore, name, ft, size)) | entries.append((_srcstore, name, ft, size)) | ||||
| totalfilesize += size | totalfilesize += size | ||||
| for name in _walkstreamfullstorefiles(repo): | for name in _walkstreamfullstorefiles(repo): | ||||
| if repo.svfs.exists(name): | if repo.svfs.exists(name): | ||||
| # key is unencoded filename | # key is unencoded filename | ||||
| # value is revlog_object_from_srcrepo | # value is revlog_object_from_srcrepo | ||||
| manifests = {} | manifests = {} | ||||
| changelogs = {} | changelogs = {} | ||||
| filelogs = {} | filelogs = {} | ||||
| # Perform a pass to collect metadata. This validates we can open all | # Perform a pass to collect metadata. This validates we can open all | ||||
| # source files and allows a unified progress bar to be displayed. | # source files and allows a unified progress bar to be displayed. | ||||
| for rl_type, unencoded, encoded, size in alldatafiles: | for rl_type, unencoded, size in alldatafiles: | ||||
| if not rl_type & store.FILEFLAGS_REVLOG_MAIN: | if not rl_type & store.FILEFLAGS_REVLOG_MAIN: | ||||
| continue | continue | ||||
| # the store.walk function will wrongly pickup transaction backup and | # the store.walk function will wrongly pickup transaction backup and | ||||
| # get confused. As a quick fix for 5.9 release, we ignore those. | # get confused. As a quick fix for 5.9 release, we ignore those. | ||||
| # (this is not a module constants because it seems better to keep the | # (this is not a module constants because it seems better to keep the | ||||
| # hack together) | # hack together) | ||||
| skip_undo = ( | skip_undo = ( | ||||
| msg %= short(m) | msg %= short(m) | ||||
| self._err(c, msg, label) | self._err(c, msg, label) | ||||
| if not dir and subdirnodes: | if not dir and subdirnodes: | ||||
| self.ui.status(_(b"checking directory manifests\n")) | self.ui.status(_(b"checking directory manifests\n")) | ||||
| storefiles = set() | storefiles = set() | ||||
| subdirs = set() | subdirs = set() | ||||
| revlogv1 = self.revlogv1 | revlogv1 = self.revlogv1 | ||||
| for t, f, f2, size in repo.store.datafiles(): | undecodable = [] | ||||
| if not f: | for t, f, size in repo.store.datafiles(undecodable=undecodable): | ||||
| self._err(None, _(b"cannot decode filename '%s'") % f2) | if (size > 0 or not revlogv1) and f.startswith(b'meta/'): | ||||
| elif (size > 0 or not revlogv1) and f.startswith(b'meta/'): | |||||
| storefiles.add(_normpath(f)) | storefiles.add(_normpath(f)) | ||||
| subdirs.add(os.path.dirname(f)) | subdirs.add(os.path.dirname(f)) | ||||
| for f in undecodable: | |||||
| self._err(None, _(b"cannot decode filename '%s'") % f) | |||||
| subdirprogress = ui.makeprogress( | subdirprogress = ui.makeprogress( | ||||
| _(b'checking'), unit=_(b'manifests'), total=len(subdirs) | _(b'checking'), unit=_(b'manifests'), total=len(subdirs) | ||||
| ) | ) | ||||
| for subdir, linkrevs in pycompat.iteritems(subdirnodes): | for subdir, linkrevs in pycompat.iteritems(subdirnodes): | ||||
| subdirfilenodes = self._verifymanifest( | subdirfilenodes = self._verifymanifest( | ||||
| linkrevs, subdir, storefiles, subdirprogress | linkrevs, subdir, storefiles, subdirprogress | ||||
| ) | ) | ||||
| repo = self.repo | repo = self.repo | ||||
| ui = self.ui | ui = self.ui | ||||
| lrugetctx = self.lrugetctx | lrugetctx = self.lrugetctx | ||||
| revlogv1 = self.revlogv1 | revlogv1 = self.revlogv1 | ||||
| havemf = self.havemf | havemf = self.havemf | ||||
| ui.status(_(b"checking files\n")) | ui.status(_(b"checking files\n")) | ||||
| storefiles = set() | storefiles = set() | ||||
| for rl_type, f, f2, size in repo.store.datafiles(): | undecodable = [] | ||||
| if not f: | for t, f, size in repo.store.datafiles(undecodable=undecodable): | ||||
| self._err(None, _(b"cannot decode filename '%s'") % f2) | if (size > 0 or not revlogv1) and f.startswith(b'data/'): | ||||
| elif (size > 0 or not revlogv1) and f.startswith(b'data/'): | |||||
| storefiles.add(_normpath(f)) | storefiles.add(_normpath(f)) | ||||
| for f in undecodable: | |||||
| self._err(None, _(b"cannot decode filename '%s'") % f) | |||||
| state = { | state = { | ||||
| # TODO this assumes revlog storage for changelog. | # TODO this assumes revlog storage for changelog. | ||||
| b'expectedversion': self.repo.changelog._format_version, | b'expectedversion': self.repo.changelog._format_version, | ||||
| b'skipflags': self.skipflags, | b'skipflags': self.skipflags, | ||||
| # experimental config: censor.policy | # experimental config: censor.policy | ||||
| b'erroroncensored': ui.config(b'censor', b'policy') == b'abort', | b'erroroncensored': ui.config(b'censor', b'policy') == b'abort', | ||||
| } | } | ||||
| with repo.lock(): | with repo.lock(): | ||||
| topfiles = list(repo.store.topfiles()) | topfiles = list(repo.store.topfiles()) | ||||
| sendfiles = [] | sendfiles = [] | ||||
| totalsize = 0 | totalsize = 0 | ||||
| # TODO this is a bunch of storage layer interface abstractions because | # TODO this is a bunch of storage layer interface abstractions because | ||||
| # it assumes revlogs. | # it assumes revlogs. | ||||
| for rl_type, name, encodedname, size in topfiles: | for rl_type, name, size in topfiles: | ||||
| # XXX use the `rl_type` for that | # XXX use the `rl_type` for that | ||||
| if b'changelog' in files and name.startswith(b'00changelog'): | if b'changelog' in files and name.startswith(b'00changelog'): | ||||
| pass | pass | ||||
| elif b'manifestlog' in files and name.startswith(b'00manifest'): | elif b'manifestlog' in files and name.startswith(b'00manifest'): | ||||
| pass | pass | ||||
| else: | else: | ||||
| continue | continue | ||||
This error message should use gettext (i.e. _())