This is an archive of the discontinued Mercurial Phabricator instance.

Differential D1353

clindex: wrapper for future changelog index overrides
ClosedPublic

Authored by quark on Nov 9 2017, 2:28 PM.
Tags
None
Subscribers
durham

Details

Reviewers
durham
Group Reviewers
Restricted Project
Commits
rFBHGX7c74370081c0: clindex: wrapper for future changelog index overrides
Summary

This extension is a thin wrapper around the native changelog.index object
that allows us to incrementally replace its methods. Since some index
methods (ex. nodemap.__getitem__) are called very frequently, Cython
features are used intentionally to avoid overhead. This also makes it easier
to integrate logic with C interface in the future.

As a side effect, this patch enforce index to be conceptually separate
from nodemap. So changelog.index[node] could be made illegal in the
future, which seems to be a good thing.

Test Plan

Run hg sl with and without the extension in a large repo. Check traceprof
outputs. Notice the performance difference around index methods are roughly
10%, which seems acceptable:

Without the extension:

25      \ node (4823 times)                  changelog.py:361
18       | node (4931 times)                 revlog.py:631

With the extension:

27      \ node (4823 times)                  changelog.py:361
19       | node (4931 times)                 revlog.py:631

Also run rt --extra-config-opt=extensions.clindex= from core hg and make
sure changes are all caused by having an extra extension enabled.

Diff Detail

Repository
rFBHGX Facebook Mercurial Extensions
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

quark created this revision.Nov 9 2017, 2:28 PM
Herald added a reviewer: Restricted Project. · View Herald TranscriptNov 9 2017, 2:28 PM
quark updated this revision to Diff 3382.Nov 9 2017, 2:37 PM
durham accepted this revision.Nov 14 2017, 7:04 PM
durham added a subscriber: durham.
durham added inline comments.
hgext3rd/clindex.pyx
7

I'd also mention why this is useful.

setup.py
334

I'd probably put this first. The list isn't quite alphabetical, but it's pretty close.

This revision is now accepted and ready to land.Nov 14 2017, 7:04 PM
Closed by commit rFBHGX7c74370081c0: clindex: wrapper for future changelog index overrides (authored by quark). · Explain WhyNov 20 2017, 9:54 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
AMhgext3rd/clindex.pyx (103 lines)
Msetup.py (12 lines)

Diff 3641

hgext3rd/clindex.pyx

  • This file was added.
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""alternative changelog index
durhamUnsubmitted
Not Done

I'd also mention why this is useful.

durham: I'd also mention why this is useful.
This extension replaces certain parts of changelog index algorithms to make it
more efficient when changelog is large.
"""
from __future__ import absolute_import
from mercurial import (
changelog,
error,
extensions,
policy,
revlog,
)
origindex = policy.importmod('parsers').index
# cdef is important for performance because it avoids dict lookups:
# - `self._origindex` becomes `some_c_struct_pointer->_origindex`
# - `__getitem__`, `__len__` will be using `PyMappingMethods` APIs
cdef class clindex(object):
cdef _origindex
def __init__(self, data, inlined):
assert not inlined
self._origindex = origindex(data, inlined)
def ancestors(self, *revs):
return self._origindex.ancestors(*revs)
def commonancestorsheads(self, *revs):
return self._origindex.commonancestorsheads(*revs)
def __getitem__(self, int rev):
return self._origindex[rev]
def computephasesmapsets(self, roots):
return self._origindex.computephasesmapsets(roots)
def reachableroots2(self, int minroot, heads, roots, includepath):
return self._origindex.reachableroots2(minroot, heads, roots,
includepath)
def headrevs(self):
return self._origindex.headrevs()
def headrevsfiltered(self, filtered):
return self._origindex.headrevsfiltered(filtered)
def deltachain(self, rev, stoprev, generaldelta):
return self._origindex.deltachain(rev, stoprev, generaldelta)
def insert(self, int rev, entry):
return self._origindex.insert(rev, entry)
def partialmatch(self, hexnode):
return self._origindex.partialmatch(hexnode)
def __len__(self):
return len(self._origindex)
@property
def nodemap(self):
return nodemap(self._origindex)
cdef class nodemap(object):
cdef _origindex
def __init__(self, origindex):
self._origindex = origindex
def __getitem__(self, node):
return self._origindex[node]
def __setitem__(self, node, rev):
self._origindex[node] = rev
def __contains__(self, node):
return node in self._origindex
def get(self, node, default=None):
return self._origindex.get(node, default)
def _parseindex(orig, self, data, inline):
if inline:
# clindex does not support inline
return orig(self, data, inline)
index = clindex(data, inline)
return index, index.nodemap, None
def _changeloginit(orig, self, *args, **kwargs):
with extensions.wrappedfunction(revlog.revlogio, 'parseindex', _parseindex):
orig(self, *args, **kwargs)
def uisetup(ui):
extensions.wrapfunction(changelog.changelog, '__init__', _changeloginit)

setup.py

libraries=[SHA1_LIBRARY], libraries=[SHA1_LIBRARY],
extra_compile_args=filter(None, [ extra_compile_args=filter(None, [
STDC99, STDC99,
WALL, WALL,
WSTRICTPROTOTYPES, WSTRICTPROTOTYPES,
] + cflags), ] + cflags),
), ),
], ],
'clindex': [
Extension('hgext3rd.clindex',
sources=['hgext3rd/clindex.pyx'],
include_dirs=['hgext3rd'],
extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]),
),
],
'linelog' : [ 'linelog' : [
Extension('linelog', Extension('linelog',
sources=['linelog/pyext/linelog.pyx'], sources=['linelog/pyext/linelog.pyx'],
extra_compile_args=filter(None, [ extra_compile_args=filter(None, [
STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC, STDC99, WALL, WEXTRA, WCONVERSION, PEDANTIC,
]), ]),
), ),
], ],
Extension('hgext3rd.traceprof', Extension('hgext3rd.traceprof',
sources=['hgext3rd/traceprof.pyx'], sources=['hgext3rd/traceprof.pyx'],
include_dirs=['hgext3rd'], include_dirs=['hgext3rd'],
extra_compile_args=filter(None, [ extra_compile_args=filter(None, [
OPTIMIZATION, STDCPP0X, WALL, WEXTRA, WCONVERSION, PEDANTIC, OPTIMIZATION, STDCPP0X, WALL, WEXTRA, WCONVERSION, PEDANTIC,
PRODUCEDEBUGSYMBOLS PRODUCEDEBUGSYMBOLS
]), ]),
), ),
] ],
} }
durhamUnsubmitted
Not Done

I'd probably put this first. The list isn't quite alphabetical, but it's pretty close.

durham: I'd probably put this first. The list isn't quite alphabetical, but it's pretty close.
allnames = availablepackages + availableextmodules.keys() + availablepymodules allnames = availablepackages + availableextmodules.keys() + availablepymodules
COMPONENTS = sorted(name.split('.')[-1] for name in allnames) COMPONENTS = sorted(name.split('.')[-1] for name in allnames)
if not components: if not components:
components = COMPONENTS components = COMPONENTS
dependencies = { dependencies = {
components.append(dep) components.append(dep)
processdep = True processdep = True
if iswindows: if iswindows:
# The modules that successfully compile on Windows # The modules that successfully compile on Windows
cythonmodules = ['linelog'] cythonmodules = ['linelog']
else: else:
cythonmodules = [ cythonmodules = [
'clindex',
'linelog', 'linelog',
'patchrmdir', 'patchrmdir',
'traceprof', 'traceprof',
] ]
if USECYTHON: if USECYTHON:
# see http://cython.readthedocs.io/en/latest/src/reference/compilation.html # see http://cython.readthedocs.io/en/latest/src/reference/compilation.html
compileroptions = { compileroptions = {