This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7851

hgext: replace references to hashlib.sha1 with hashutil.sha1
ClosedPublic

Authored by durin42 on Jan 13 2020, 5:48 PM.

Details

Reviewers
indygreg
Group Reviewers
hg-reviewers
Commits
rHG2d49482d0dd4: hgext: replace references to hashlib.sha1 with hashutil.sha1
rHG6f40a1e56b61: hgext: replace references to hashlib.sha1 with hashutil.sha1
rHG006e7821704a: hgext: replace references to hashlib.sha1 with hashutil.sha1
Summary

When in a non-pure build of Mercurial, this will provide protections
against SHA1 collision attacks.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Jan 13 2020, 5:48 PM
indygreg accepted this revision.Jan 13 2020, 11:26 PM
indygreg added a subscriber: indygreg.

Would you mind a follow-up to teach check-code to ban hashlib.sha so usage doesn't creep back in?

This revision is now accepted and ready to land.Jan 13 2020, 11:26 PM

Revision Contents
Changeset List

Diff 19202

hgext/fastannotate/context.py

# Copyright 2016-present Facebook. All Rights Reserved. # Copyright 2016-present Facebook. All Rights Reserved.
# #
# context: context needed to annotate a file # context: context needed to annotate a file
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import contextlib import contextlib
import hashlib
import os import os
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import ( from mercurial.pycompat import (
getattr, getattr,
open, open,
setattr, setattr,
) )
from mercurial import ( from mercurial import (
error, error,
linelog as linelogmod, linelog as linelogmod,
lock as lockmod, lock as lockmod,
mdiff, mdiff,
node, node,
pycompat, pycompat,
scmutil, scmutil,
util, util,
) )
from mercurial.utils import stringutil from mercurial.utils import (
hashutil,
stringutil,
)
from . import ( from . import (
error as faerror, error as faerror,
revmap as revmapmod, revmap as revmapmod,
) )
# given path, get filelog, cached # given path, get filelog, cached
@util.lrucachefunc @util.lrucachefunc
.replace(b'.lock/', b'.lock.hg/') .replace(b'.lock/', b'.lock.hg/')
) )
def hashdiffopts(diffopts): def hashdiffopts(diffopts):
diffoptstr = stringutil.pprint( diffoptstr = stringutil.pprint(
sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults) sorted((k, getattr(diffopts, k)) for k in mdiff.diffopts.defaults)
) )
return node.hex(hashlib.sha1(diffoptstr).digest())[:6] return node.hex(hashutil.sha1(diffoptstr).digest())[:6]
_defaultdiffopthash = hashdiffopts(mdiff.defaultopts) _defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
class annotateopts(object): class annotateopts(object):
"""like mercurial.mdiff.diffopts, but is for annotate """like mercurial.mdiff.diffopts, but is for annotate

hgext/fsmonitor/__init__.py

# subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`. # subrepos exist. *Workaround*: add nested repo paths to your `.hgignore`.
# #
# The issues related to nested repos and subrepos are probably not fundamental # The issues related to nested repos and subrepos are probably not fundamental
# ones. Patches to fix them are welcome. # ones. Patches to fix them are welcome.
from __future__ import absolute_import from __future__ import absolute_import
import codecs import codecs
import hashlib
import os import os
import stat import stat
import sys import sys
import tempfile import tempfile
import weakref import weakref
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import hex from mercurial.node import hex
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
context, context,
encoding, encoding,
error, error,
extensions, extensions,
localrepo, localrepo,
merge, merge,
pathutil, pathutil,
pycompat, pycompat,
registrar, registrar,
scmutil, scmutil,
util, util,
) )
from mercurial import match as matchmod from mercurial import match as matchmod
from mercurial.utils import stringutil from mercurial.utils import (
hashutil,
stringutil,
)
from . import ( from . import (
pywatchman, pywatchman,
state, state,
watchmanclient, watchmanclient,
) )
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
def _hashignore(ignore): def _hashignore(ignore):
"""Calculate hash for ignore patterns and filenames """Calculate hash for ignore patterns and filenames
If this information changes between Mercurial invocations, we can't If this information changes between Mercurial invocations, we can't
rely on Watchman information anymore and have to re-scan the working rely on Watchman information anymore and have to re-scan the working
copy. copy.
""" """
sha1 = hashlib.sha1() sha1 = hashutil.sha1()
sha1.update(pycompat.byterepr(ignore)) sha1.update(pycompat.byterepr(ignore))
return pycompat.sysbytes(sha1.hexdigest()) return pycompat.sysbytes(sha1.hexdigest())
_watchmanencoding = pywatchman.encoding.get_local_encoding() _watchmanencoding = pywatchman.encoding.get_local_encoding()
_fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding() _fsencoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
_fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding) _fixencoding = codecs.lookup(_watchmanencoding) != codecs.lookup(_fsencoding)

hgext/infinitepush/store.py

# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
# based on bundleheads extension by Gregory Szorc <gps@mozilla.com> # based on bundleheads extension by Gregory Szorc <gps@mozilla.com>
from __future__ import absolute_import from __future__ import absolute_import
import abc import abc
import hashlib
import os import os
import subprocess import subprocess
import tempfile import tempfile
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
node, node,
pycompat, pycompat,
) )
from mercurial.utils import procutil from mercurial.utils import (
hashutil,
procutil,
)
NamedTemporaryFile = tempfile.NamedTemporaryFile NamedTemporaryFile = tempfile.NamedTemporaryFile
class BundleWriteException(Exception): class BundleWriteException(Exception):
pass pass
level directory, next two bytes are the name of the level directory, next two bytes are the name of the
next level directory""" next level directory"""
return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4]) return os.path.join(self.storepath, hashvalue[0:2], hashvalue[2:4])
def _filepath(self, filename): def _filepath(self, filename):
return os.path.join(self._dirpath(filename), filename) return os.path.join(self._dirpath(filename), filename)
def write(self, data): def write(self, data):
filename = node.hex(hashlib.sha1(data).digest()) filename = node.hex(hashutil.sha1(data).digest())
dirpath = self._dirpath(filename) dirpath = self._dirpath(filename)
if not os.path.exists(dirpath): if not os.path.exists(dirpath):
os.makedirs(dirpath) os.makedirs(dirpath)
with open(self._filepath(filename), b'wb') as f: with open(self._filepath(filename), b'wb') as f:
f.write(data) f.write(data)

hgext/largefiles/lfcommands.py

# Copyright 2009-2010 Gregory P. Ward # Copyright 2009-2010 Gregory P. Ward
# Copyright 2009-2010 Intelerad Medical Systems Incorporated # Copyright 2009-2010 Intelerad Medical Systems Incorporated
# Copyright 2010-2011 Fog Creek Software # Copyright 2010-2011 Fog Creek Software
# Copyright 2010-2011 Unity Technologies # Copyright 2010-2011 Unity Technologies
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
'''High-level command function for lfconvert, plus the cmdtable.''' '''High-level command function for lfconvert, plus the cmdtable.'''
from __future__ import absolute_import from __future__ import absolute_import
import errno import errno
import hashlib
import os import os
import shutil import shutil
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
cmdutil, cmdutil,
context, context,
error, error,
exthelper, exthelper,
hg, hg,
lock, lock,
match as matchmod, match as matchmod,
node, node,
pycompat, pycompat,
scmutil, scmutil,
util, util,
) )
from mercurial.utils import hashutil
from ..convert import ( from ..convert import (
convcmd, convcmd,
filemap, filemap,
) )
from . import lfutil, storefactory from . import lfutil, storefactory
if b'l' in fctx.flags(): if b'l' in fctx.flags():
renamed = fctx.copysource() renamed = fctx.copysource()
if renamed and renamed in lfiles: if renamed and renamed in lfiles:
raise error.Abort( raise error.Abort(
_(b'largefile %s becomes symlink') % f _(b'largefile %s becomes symlink') % f
) )
# largefile was modified, update standins # largefile was modified, update standins
m = hashlib.sha1(b'') m = hashutil.sha1(b'')
m.update(ctx[f].data()) m.update(ctx[f].data())
hash = node.hex(m.digest()) hash = node.hex(m.digest())
if f not in lfiletohash or lfiletohash[f] != hash: if f not in lfiletohash or lfiletohash[f] != hash:
rdst.wwrite(f, ctx[f].data(), ctx[f].flags()) rdst.wwrite(f, ctx[f].data(), ctx[f].flags())
executable = b'x' in ctx[f].flags() executable = b'x' in ctx[f].flags()
lfutil.writestandin(rdst, fstandin, hash, executable) lfutil.writestandin(rdst, fstandin, hash, executable)
lfiletohash[f] = hash lfiletohash[f] = hash
else: else:

hgext/largefiles/lfutil.py

# Copyright 2009-2010 Gregory P. Ward # Copyright 2009-2010 Gregory P. Ward
# Copyright 2009-2010 Intelerad Medical Systems Incorporated # Copyright 2009-2010 Intelerad Medical Systems Incorporated
# Copyright 2010-2011 Fog Creek Software # Copyright 2010-2011 Fog Creek Software
# Copyright 2010-2011 Unity Technologies # Copyright 2010-2011 Unity Technologies
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
'''largefiles utility code: must not import other modules in this package.''' '''largefiles utility code: must not import other modules in this package.'''
from __future__ import absolute_import from __future__ import absolute_import
import contextlib import contextlib
import copy import copy
import hashlib
import os import os
import stat import stat
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import hex from mercurial.node import hex
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
dirstate, dirstate,
encoding, encoding,
error, error,
httpconnection, httpconnection,
match as matchmod, match as matchmod,
node, node,
pycompat, pycompat,
scmutil, scmutil,
sparse, sparse,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from mercurial.utils import hashutil
shortname = b'.hglf' shortname = b'.hglf'
shortnameslash = shortname + b'/' shortnameslash = shortname + b'/'
longname = b'largefiles' longname = b'largefiles'
# -- Private worker functions ------------------------------------------ # -- Private worker functions ------------------------------------------
def writestandin(repo, standin, hash, executable): def writestandin(repo, standin, hash, executable):
'''write hash to <repo.root>/<standin>''' '''write hash to <repo.root>/<standin>'''
repo.wwrite(standin, hash + b'\n', executable and b'x' or b'') repo.wwrite(standin, hash + b'\n', executable and b'x' or b'')
def copyandhash(instream, outfile): def copyandhash(instream, outfile):
'''Read bytes from instream (iterable) and write them to outfile, '''Read bytes from instream (iterable) and write them to outfile,
computing the SHA-1 hash of the data along the way. Return the hash.''' computing the SHA-1 hash of the data along the way. Return the hash.'''
hasher = hashlib.sha1(b'') hasher = hashutil.sha1(b'')
for data in instream: for data in instream:
hasher.update(data) hasher.update(data)
outfile.write(data) outfile.write(data)
return hex(hasher.digest()) return hex(hasher.digest())
def hashfile(file): def hashfile(file):
if not os.path.exists(file): if not os.path.exists(file):
for a in arg: for a in arg:
url = join(url, a) url = join(url, a)
return url return url
def hexsha1(fileobj): def hexsha1(fileobj):
"""hexsha1 returns the hex-encoded sha1 sum of the data in the file-like """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
object data""" object data"""
h = hashlib.sha1() h = hashutil.sha1()
for chunk in util.filechunkiter(fileobj): for chunk in util.filechunkiter(fileobj):
h.update(chunk) h.update(chunk)
return hex(h.digest()) return hex(h.digest())
def httpsendfile(ui, filename): def httpsendfile(ui, filename):
return httpconnection.httpsendfile(ui, filename, b'rb') return httpconnection.httpsendfile(ui, filename, b'rb')

hgext/remotefilelog/basepack.py

from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import errno import errno
import hashlib
import mmap import mmap
import os import os
import struct import struct
import time import time
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import ( from mercurial.pycompat import (
getattr, getattr,
open, open,
) )
from mercurial import ( from mercurial import (
node as nodemod, node as nodemod,
policy, policy,
pycompat, pycompat,
util, util,
vfs as vfsmod, vfs as vfsmod,
) )
from mercurial.utils import hashutil
from . import shallowutil from . import shallowutil
osutil = policy.importmod('osutil') osutil = policy.importmod('osutil')
# The pack version supported by this implementation. This will need to be # The pack version supported by this implementation. This will need to be
# rev'd whenever the byte format changes. Ex: changing the fanout prefix, # rev'd whenever the byte format changes. Ex: changing the fanout prefix,
# changing any of the int sizes, changing the delta algorithm, etc. # changing any of the int sizes, changing the delta algorithm, etc.
PACKVERSIONSIZE = 1 PACKVERSIONSIZE = 1
self.packfp, self.packpath = opener.mkstemp( self.packfp, self.packpath = opener.mkstemp(
suffix=self.PACKSUFFIX + b'-tmp' suffix=self.PACKSUFFIX + b'-tmp'
) )
self.idxfp, self.idxpath = opener.mkstemp( self.idxfp, self.idxpath = opener.mkstemp(
suffix=self.INDEXSUFFIX + b'-tmp' suffix=self.INDEXSUFFIX + b'-tmp'
) )
self.packfp = os.fdopen(self.packfp, 'wb+') self.packfp = os.fdopen(self.packfp, 'wb+')
self.idxfp = os.fdopen(self.idxfp, 'wb+') self.idxfp = os.fdopen(self.idxfp, 'wb+')
self.sha = hashlib.sha1() self.sha = hashutil.sha1()
self._closed = False self._closed = False
# The opener provides no way of doing permission fixup on files created # The opener provides no way of doing permission fixup on files created
# via mkstemp, so we must fix it ourselves. We can probably fix this # via mkstemp, so we must fix it ourselves. We can probably fix this
# upstream in vfs.mkstemp so we don't need to use the private method. # upstream in vfs.mkstemp so we don't need to use the private method.
opener._fixfilemode(opener.join(self.packpath)) opener._fixfilemode(opener.join(self.packpath))
opener._fixfilemode(opener.join(self.idxpath)) opener._fixfilemode(opener.join(self.idxpath))

hgext/remotefilelog/basestore.py

from __future__ import absolute_import from __future__ import absolute_import
import errno import errno
import hashlib
import os import os
import shutil import shutil
import stat import stat
import time import time
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import bin, hex from mercurial.node import bin, hex
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
error, error,
pycompat, pycompat,
util, util,
) )
from mercurial.utils import hashutil
from . import ( from . import (
constants, constants,
shallowutil, shallowutil,
) )
class basestore(object): class basestore(object):
def __init__(self, repo, path, reponame, shared=False): def __init__(self, repo, path, reponame, shared=False):
if not hashes: if not hashes:
return {} return {}
filenames = {} filenames = {}
missingfilename = set(hashes) missingfilename = set(hashes)
# Start with a full manifest, since it'll cover the majority of files # Start with a full manifest, since it'll cover the majority of files
for filename in self.repo[b'tip'].manifest(): for filename in self.repo[b'tip'].manifest():
sha = hashlib.sha1(filename).digest() sha = hashutil.sha1(filename).digest()
if sha in missingfilename: if sha in missingfilename:
filenames[filename] = sha filenames[filename] = sha
missingfilename.discard(sha) missingfilename.discard(sha)
# Scan the changelog until we've found every file name # Scan the changelog until we've found every file name
cl = self.repo.unfiltered().changelog cl = self.repo.unfiltered().changelog
for rev in pycompat.xrange(len(cl) - 1, -1, -1): for rev in pycompat.xrange(len(cl) - 1, -1, -1):
if not missingfilename: if not missingfilename:
break break
files = cl.readfiles(cl.node(rev)) files = cl.readfiles(cl.node(rev))
for filename in files: for filename in files:
sha = hashlib.sha1(filename).digest() sha = hashutil.sha1(filename).digest()
if sha in missingfilename: if sha in missingfilename:
filenames[filename] = sha filenames[filename] = sha
missingfilename.discard(sha) missingfilename.discard(sha)
return filenames return filenames
def _getrepocachepath(self): def _getrepocachepath(self):
return ( return (

hgext/remotefilelog/debugcommands.py

# debugcommands.py - debug logic for remotefilelog # debugcommands.py - debug logic for remotefilelog
# #
# Copyright 2013 Facebook, Inc. # Copyright 2013 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import hashlib
import os import os
import zlib import zlib
from mercurial.node import bin, hex, nullid, short from mercurial.node import bin, hex, nullid, short
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
error, error,
filelog, filelog,
lock as lockmod, lock as lockmod,
node as nodemod, node as nodemod,
pycompat, pycompat,
revlog, revlog,
) )
from mercurial.utils import hashutil
from . import ( from . import (
constants, constants,
datapack, datapack,
fileserverclient, fileserverclient,
historypack, historypack,
repack, repack,
shallowutil, shallowutil,
) )
if p1 != nullid: if p1 != nullid:
queue.append(p1) queue.append(p1)
if p2 != nullid: if p2 != nullid:
queue.append(p2) queue.append(p2)
def buildtemprevlog(repo, file): def buildtemprevlog(repo, file):
# get filename key # get filename key
filekey = nodemod.hex(hashlib.sha1(file).digest()) filekey = nodemod.hex(hashutil.sha1(file).digest())
filedir = os.path.join(repo.path, b'store/data', filekey) filedir = os.path.join(repo.path, b'store/data', filekey)
# sort all entries based on linkrev # sort all entries based on linkrev
fctxs = [] fctxs = []
for filenode in os.listdir(filedir): for filenode in os.listdir(filedir):
if b'_old' not in filenode: if b'_old' not in filenode:
fctxs.append(repo.filectx(file, fileid=bin(filenode))) fctxs.append(repo.filectx(file, fileid=bin(filenode)))
) )
) )
ui.write( ui.write(
b"%s %s %s %d\n" b"%s %s %s %d\n"
% ( % (
hashformatter(node), hashformatter(node),
hashformatter(deltabasenode), hashformatter(deltabasenode),
nodemod.hex(hashlib.sha1(delta).digest()), nodemod.hex(hashutil.sha1(delta).digest()),
len(delta), len(delta),
) )
) )
def debughistorypack(ui, path): def debughistorypack(ui, path):
if b'.hist' in path: if b'.hist' in path:
path = path[: path.index(b'.hist')] path = path[: path.index(b'.hist')]

hgext/remotefilelog/fileserverclient.py

# fileserverclient.py - client for communicating with the cache process # fileserverclient.py - client for communicating with the cache process
# #
# Copyright 2013 Facebook, Inc. # Copyright 2013 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import hashlib
import io import io
import os import os
import threading import threading
import time import time
import zlib import zlib
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import bin, hex, nullid from mercurial.node import bin, hex, nullid
from mercurial import ( from mercurial import (
error, error,
node, node,
pycompat, pycompat,
revlog, revlog,
sshpeer, sshpeer,
util, util,
wireprotov1peer, wireprotov1peer,
) )
from mercurial.utils import procutil from mercurial.utils import (
hashutil,
procutil,
)
from . import ( from . import (
constants, constants,
contentstore, contentstore,
metadatastore, metadatastore,
) )
_sshv1peer = sshpeer.sshv1peer _sshv1peer = sshpeer.sshv1peer
# Statistics for debugging # Statistics for debugging
fetchcost = 0 fetchcost = 0
fetches = 0 fetches = 0
fetched = 0 fetched = 0
fetchmisses = 0 fetchmisses = 0
_lfsmod = None _lfsmod = None
def getcachekey(reponame, file, id): def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id) return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id) return os.path.join(pathhash, id)
def peersetup(ui, peer): def peersetup(ui, peer):
class remotefilepeer(peer.__class__): class remotefilepeer(peer.__class__):
@wireprotov1peer.batchable @wireprotov1peer.batchable
def x_rfl_getfile(self, file, node): def x_rfl_getfile(self, file, node):
if not self.capable(b'x_rfl_getfile'): if not self.capable(b'x_rfl_getfile'):

hgext/remotefilelog/historypack.py

from __future__ import absolute_import from __future__ import absolute_import
import hashlib
import struct import struct
from mercurial.node import hex, nullid from mercurial.node import hex, nullid
from mercurial import ( from mercurial import (
pycompat, pycompat,
util, util,
) )
from mercurial.utils import hashutil
from . import ( from . import (
basepack, basepack,
constants, constants,
shallowutil, shallowutil,
) )
# (filename hash, offset, size) # (filename hash, offset, size)
INDEXFORMAT2 = b'!20sQQII' INDEXFORMAT2 = b'!20sQQII'
# Drop the copyfromlen from the end of entry, and replace it # Drop the copyfromlen from the end of entry, and replace it
# with the copyfrom string. # with the copyfrom string.
return entry[:4] + (copyfrom,) return entry[:4] + (copyfrom,)
raise KeyError(b"unable to find history for %s:%s" % (name, hex(node))) raise KeyError(b"unable to find history for %s:%s" % (name, hex(node)))
def _findsection(self, name): def _findsection(self, name):
params = self.params params = self.params
namehash = hashlib.sha1(name).digest() namehash = hashutil.sha1(name).digest()
fanoutkey = struct.unpack( fanoutkey = struct.unpack(
params.fanoutstruct, namehash[: params.fanoutprefix] params.fanoutstruct, namehash[: params.fanoutprefix]
)[0] )[0]
fanout = self._fanouttable fanout = self._fanouttable
start = fanout[fanoutkey] + params.indexstart start = fanout[fanoutkey] + params.indexstart
indexend = self._indexend indexend = self._indexend
rawdata = b''.join(rawstrings) rawdata = b''.join(rawstrings)
sectionlen += len(rawdata) sectionlen += len(rawdata)
self.writeraw(rawdata) self.writeraw(rawdata)
# Record metadata for the index # Record metadata for the index
self.files[filename] = (sectionstart, sectionlen) self.files[filename] = (sectionstart, sectionlen)
node = hashlib.sha1(filename).digest() node = hashutil.sha1(filename).digest()
self.entries[node] = node self.entries[node] = node
def close(self, ledger=None): def close(self, ledger=None):
if self._closed: if self._closed:
return return
self._write() self._write()
return super(mutablehistorypack, self).close(ledger=ledger) return super(mutablehistorypack, self).close(ledger=ledger)
def createindex(self, nodelocations, indexoffset): def createindex(self, nodelocations, indexoffset):
fileindexformat = self.INDEXFORMAT fileindexformat = self.INDEXFORMAT
fileindexlength = self.INDEXENTRYLENGTH fileindexlength = self.INDEXENTRYLENGTH
nodeindexformat = self.NODEINDEXFORMAT nodeindexformat = self.NODEINDEXFORMAT
nodeindexlength = self.NODEINDEXENTRYLENGTH nodeindexlength = self.NODEINDEXENTRYLENGTH
files = ( files = (
(hashlib.sha1(filename).digest(), filename, offset, size) (hashutil.sha1(filename).digest(), filename, offset, size)
for filename, (offset, size) in pycompat.iteritems(self.files) for filename, (offset, size) in pycompat.iteritems(self.files)
) )
files = sorted(files) files = sorted(files)
# node index is after file index size, file index, and node index size # node index is after file index size, file index, and node index size
indexlensize = struct.calcsize(b'!Q') indexlensize = struct.calcsize(b'!Q')
nodeindexoffset = ( nodeindexoffset = (
indexoffset indexoffset

hgext/remotefilelog/shallowutil.py

# shallowutil.py -- remotefilelog utilities # shallowutil.py -- remotefilelog utilities
# #
# Copyright 2014 Facebook, Inc. # Copyright 2014 Facebook, Inc.
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import collections import collections
import errno import errno
import hashlib
import os import os
import stat import stat
import struct import struct
import tempfile import tempfile
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.pycompat import open from mercurial.pycompat import open
from mercurial import ( from mercurial import (
error, error,
node, node,
pycompat, pycompat,
revlog, revlog,
util, util,
) )
from mercurial.utils import ( from mercurial.utils import (
hashutil,
storageutil, storageutil,
stringutil, stringutil,
) )
from . import constants from . import constants
if not pycompat.iswindows: if not pycompat.iswindows:
import grp import grp
def isenabled(repo): def isenabled(repo):
"""returns whether the repository is remotefilelog enabled or not""" """returns whether the repository is remotefilelog enabled or not"""
return constants.SHALLOWREPO_REQUIREMENT in repo.requirements return constants.SHALLOWREPO_REQUIREMENT in repo.requirements
def getcachekey(reponame, file, id): def getcachekey(reponame, file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(reponame, pathhash[:2], pathhash[2:], id) return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
def getlocalkey(file, id): def getlocalkey(file, id):
pathhash = node.hex(hashlib.sha1(file).digest()) pathhash = node.hex(hashutil.sha1(file).digest())
return os.path.join(pathhash, id) return os.path.join(pathhash, id)
def getcachepath(ui, allowempty=False): def getcachepath(ui, allowempty=False):
cachepath = ui.config(b"remotefilelog", b"cachepath") cachepath = ui.config(b"remotefilelog", b"cachepath")
if not cachepath: if not cachepath:
if allowempty: if allowempty:
return None return None

hgext/sqlitestore.py

# following: # following:
# #
# HGREPOFEATURES="sqlitestore" run-tests.py \ # HGREPOFEATURES="sqlitestore" run-tests.py \
# --extra-config-opt extensions.sqlitestore= \ # --extra-config-opt extensions.sqlitestore= \
# --extra-config-opt storage.new-repo-backend=sqlite # --extra-config-opt storage.new-repo-backend=sqlite
from __future__ import absolute_import from __future__ import absolute_import
import hashlib
import sqlite3 import sqlite3
import struct import struct
import threading import threading
import zlib import zlib
from mercurial.i18n import _ from mercurial.i18n import _
from mercurial.node import ( from mercurial.node import (
nullid, nullid,
registrar, registrar,
util, util,
verify, verify,
) )
from mercurial.interfaces import ( from mercurial.interfaces import (
repository, repository,
util as interfaceutil, util as interfaceutil,
) )
from mercurial.utils import storageutil from mercurial.utils import (
hashutil,
storageutil,
)
try: try:
from mercurial import zstd from mercurial import zstd
zstd.__version__ zstd.__version__
except ImportError: except ImportError:
zstd = None zstd = None
for row in rows: for row in rows:
rid, pathid, node = row rid, pathid, node = row
fulltext = resolvedeltachain( fulltext = resolvedeltachain(
self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx self._db, pathid, node, {}, {-1: None}, zstddctx=self._dctx
) )
deltahash = hashlib.sha1(fulltext).digest() deltahash = hashutil.sha1(fulltext).digest()
if self._compengine == b'zstd': if self._compengine == b'zstd':
deltablob = self._cctx.compress(fulltext) deltablob = self._cctx.compress(fulltext)
compression = COMPRESSION_ZSTD compression = COMPRESSION_ZSTD
elif self._compengine == b'zlib': elif self._compengine == b'zlib':
deltablob = zlib.compress(fulltext) deltablob = zlib.compress(fulltext)
compression = COMPRESSION_ZLIB compression = COMPRESSION_ZLIB
elif self._compengine == b'none': elif self._compengine == b'none':
self._db.execute( self._db.execute(
'UPDATE fileindex SET deltaid=?, deltabaseid=NULL ' 'UPDATE fileindex SET deltaid=?, deltabaseid=NULL '
'WHERE id=?', 'WHERE id=?',
(deltaid, rid), (deltaid, rid),
) )
# Now create the tombstone delta and replace the delta on the censored # Now create the tombstone delta and replace the delta on the censored
# node. # node.
deltahash = hashlib.sha1(tombstone).digest() deltahash = hashutil.sha1(tombstone).digest()
tombstonedeltaid = insertdelta( tombstonedeltaid = insertdelta(
self._db, COMPRESSION_NONE, deltahash, tombstone self._db, COMPRESSION_NONE, deltahash, tombstone
) )
flags = self._revisions[censornode].flags flags = self._revisions[censornode].flags
flags |= FLAG_CENSORED flags |= FLAG_CENSORED
self._db.execute( self._db.execute(
baseid = None baseid = None
else: else:
baseid = self._revisions[deltabase].rid baseid = self._revisions[deltabase].rid
# Deltas are stored with a hash of their content. This allows # Deltas are stored with a hash of their content. This allows
# us to de-duplicate. The table is configured to ignore conflicts # us to de-duplicate. The table is configured to ignore conflicts
# and it is faster to just insert and silently noop than to look # and it is faster to just insert and silently noop than to look
# first. # first.
deltahash = hashlib.sha1(delta).digest() deltahash = hashutil.sha1(delta).digest()
if self._compengine == b'zstd': if self._compengine == b'zstd':
deltablob = self._cctx.compress(delta) deltablob = self._cctx.compress(delta)
compression = COMPRESSION_ZSTD compression = COMPRESSION_ZSTD
elif self._compengine == b'zlib': elif self._compengine == b'zlib':
deltablob = zlib.compress(delta) deltablob = zlib.compress(delta)
compression = COMPRESSION_ZLIB compression = COMPRESSION_ZLIB
elif self._compengine == b'none': elif self._compengine == b'none':