This is an archive of the discontinued Mercurial Phabricator instance.

Differential D10755

docket: move the uid logic in the `revlogutils.docket` module
ClosedPublic

Authored by marmoute on May 20 2021, 10:19 AM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG7ea39d633cf3: docket: move the uid logic in the `revlogutils.docket` module
Summary

We want to use it for revlog-v2 (& Co), it seems more logical to have the logic
lives inside the docket file than the nodemap file.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

marmoute created this revision.May 20 2021, 10:19 AM
pulkit accepted this revision.May 23 2021, 3:40 PM
This revision is now accepted and ready to land.May 23 2021, 3:40 PM

Revision Contents
Changeset List

Diff 28189

mercurial/revlogutils/docket.py

# * a small docket file, containing metadata and a pointer, # * a small docket file, containing metadata and a pointer,
# #
# * an index file, containing fixed width information about revisions, # * an index file, containing fixed width information about revisions,
# #
# * a data file, containing variable width data for these revisions, # * a data file, containing variable width data for these revisions,
from __future__ import absolute_import from __future__ import absolute_import
import errno
import os
import random
import struct import struct
from .. import ( from .. import (
encoding,
error, error,
node,
pycompat,
util, util,
) )
from . import ( from . import (
constants, constants,
) )
def make_uid(id_size=8):
"""return a new unique identifier.
The identifier is random and composed of ascii characters."""
# size we "hex" the result we need half the number of bits to have a final
# uuid of size ID_SIZE
return node.hex(os.urandom(id_size // 2))
# some special test logic to avoid anoying random output in the test
stable_docket_file = encoding.environ.get(b'HGTEST_UUIDFILE')
if stable_docket_file:
def make_uid(id_size=8):
try:
with open(stable_docket_file, mode='rb') as f:
seed = f.read().strip()
except IOError as inst:
if inst.errno != errno.ENOENT:
raise
seed = b'04' # chosen by a fair dice roll. garanteed to be random
if pycompat.ispy3:
iter_seed = iter(seed)
else:
iter_seed = (ord(c) for c in seed)
# some basic circular sum hashing on 64 bits
int_seed = 0
low_mask = int('1' * 35, 2)
for i in iter_seed:
high_part = int_seed >> 35
low_part = (int_seed & low_mask) << 28
int_seed = high_part + low_part + i
r = random.Random()
if pycompat.ispy3:
r.seed(int_seed, version=1)
else:
r.seed(int_seed)
# once we drop python 3.8 support we can simply use r.randbytes
raw = r.getrandbits(id_size * 4)
assert id_size == 8
p = struct.pack('>L', raw)
new = node.hex(p)
with open(stable_docket_file, 'wb') as f:
f.write(new)
return new
# Docket format # Docket format
# #
# * 4 bytes: revlog version # * 4 bytes: revlog version
# | This is mandatory as docket must be compatible with the previous # | This is mandatory as docket must be compatible with the previous
# | revlog index header. # | revlog index header.
# * 8 bytes: size of index-data # * 8 bytes: size of index-data
# * 8 bytes: pending size of index-data # * 8 bytes: pending size of index-data
# * 8 bytes: size of data # * 8 bytes: size of data

mercurial/revlogutils/nodemap.py

# nodemap.py - nodemap related code and utilities # nodemap.py - nodemap related code and utilities
# #
# Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net> # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net>
# Copyright 2019 George Racinet <georges.racinet@octobus.net> # Copyright 2019 George Racinet <georges.racinet@octobus.net>
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
from __future__ import absolute_import from __future__ import absolute_import
import errno import errno
import os
import random
import re import re
import struct import struct
from ..node import hex from ..node import hex
from .. import ( from .. import (
encoding,
error, error,
pycompat,
util, util,
) )
from . import docket as docket_mod
class NodeMap(dict): class NodeMap(dict):
def __missing__(self, x): def __missing__(self, x):
raise error.RevlogError(b'unknown node: %s' % x) raise error.RevlogError(b'unknown node: %s' % x)
def persisted_data(revlog): def persisted_data(revlog):
# The docket file contains information to find, qualify and validate the raw # The docket file contains information to find, qualify and validate the raw
# data. Its content is currently very light, but it will expand as the on disk # data. Its content is currently very light, but it will expand as the on disk
# nodemap gains the necessary features to be used in production. # nodemap gains the necessary features to be used in production.
ONDISK_VERSION = 1 ONDISK_VERSION = 1
S_VERSION = struct.Struct(">B") S_VERSION = struct.Struct(">B")
S_HEADER = struct.Struct(">BQQQQ") S_HEADER = struct.Struct(">BQQQQ")
ID_SIZE = 8
def _make_uid():
"""return a new unique identifier.
The identifier is random and composed of ascii characters."""
# size we "hex" the result we need half the number of bits to have a final
# uuid of size ID_SIZE
return hex(os.urandom(ID_SIZE // 2))
# some special test logic to avoid anoying random output in the test
stable_docket_file = encoding.environ.get(b'HGTEST_DOCKETIDFILE')
if stable_docket_file:
def _make_uid():
try:
with open(stable_docket_file, mode='rb') as f:
seed = f.read().strip()
except IOError as inst:
if inst.errno != errno.ENOENT:
raise
seed = b'4' # chosen by a fair dice roll. garanteed to be random
if pycompat.ispy3:
iter_seed = iter(seed)
else:
iter_seed = (ord(c) for c in seed)
# some basic circular sum hashing on 64 bits
int_seed = 0
low_mask = int('1' * 35, 2)
for i in iter_seed:
high_part = int_seed >> 35
low_part = (int_seed & low_mask) << 28
int_seed = high_part + low_part + i
r = random.Random()
if pycompat.ispy3:
r.seed(int_seed, version=1)
else:
r.seed(int_seed)
# once we drop python 3.8 support we can simply use r.randbytes
raw = r.getrandbits(ID_SIZE * 4)
assert ID_SIZE == 8
p = struct.pack('>L', raw)
new = hex(p)
with open(stable_docket_file, 'wb') as f:
f.write(new)
return new
class NodeMapDocket(object): class NodeMapDocket(object):
"""metadata associated with persistent nodemap data """metadata associated with persistent nodemap data
The persistent data may come from disk or be on their way to disk. The persistent data may come from disk or be on their way to disk.
""" """
def __init__(self, uid=None): def __init__(self, uid=None):
if uid is None: if uid is None:
uid = _make_uid() uid = docket_mod.make_uid()
# a unique identifier for the data file: # a unique identifier for the data file:
# - When new data are appended, it is preserved. # - When new data are appended, it is preserved.
# - When a new data file is created, a new identifier is generated. # - When a new data file is created, a new identifier is generated.
self.uid = uid self.uid = uid
# the tipmost revision stored in the data file. This revision and all # the tipmost revision stored in the data file. This revision and all
# revision before it are expected to be encoded in the data file. # revision before it are expected to be encoded in the data file.
self.tip_rev = None self.tip_rev = None
# the node of that tipmost revision, if it mismatch the current index # the node of that tipmost revision, if it mismatch the current index

tests/run-tests.py

def defineport(i): def defineport(i):
offset = '' if i == 0 else '%s' % i offset = '' if i == 0 else '%s' % i
env["HGPORT%s" % offset] = '%s' % (self._startport + i) env["HGPORT%s" % offset] = '%s' % (self._startport + i)
env = os.environ.copy() env = os.environ.copy()
env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or '' env['PYTHONUSERBASE'] = sysconfig.get_config_var('userbase') or ''
env['HGEMITWARNINGS'] = '1' env['HGEMITWARNINGS'] = '1'
env['TESTTMP'] = _bytes2sys(self._testtmp) env['TESTTMP'] = _bytes2sys(self._testtmp)
docket_id_file = os.path.join(_bytes2sys(self._testtmp), 'DOCKETID') uid_file = os.path.join(_bytes2sys(self._testtmp), 'UID')
env['HGTEST_DOCKETIDFILE'] = docket_id_file env['HGTEST_UUIDFILE'] = uid_file
env['TESTNAME'] = self.name env['TESTNAME'] = self.name
env['HOME'] = _bytes2sys(self._testtmp) env['HOME'] = _bytes2sys(self._testtmp)
if os.name == 'nt': if os.name == 'nt':
env['REALUSERPROFILE'] = env['USERPROFILE'] env['REALUSERPROFILE'] = env['USERPROFILE']
# py3.8+ ignores HOME: https://bugs.python.org/issue36264 # py3.8+ ignores HOME: https://bugs.python.org/issue36264
env['USERPROFILE'] = env['HOME'] env['USERPROFILE'] = env['HOME']
formated_timeout = _bytes2sys(b"%d" % default_defaults['timeout'][1]) formated_timeout = _bytes2sys(b"%d" % default_defaults['timeout'][1])
env['HGTEST_TIMEOUT_DEFAULT'] = formated_timeout env['HGTEST_TIMEOUT_DEFAULT'] = formated_timeout