This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3440

demandimport: make module ignores a set (API)
ClosedPublic

Authored by indygreg on May 5 2018, 9:58 PM.

Details

Reviewers
martinvonz
Group Reviewers
hg-reviewers
Commits
rHG670eb4fa1b86: demandimport: make module ignores a set (API)
Summary

The list of modules to ignore is used for membership testing. Yet
it is defined as a list.

Sets are more efficient for membership testing. So this commit converts
the module list to a set.

Since we took an API hit, I renamed the variable to further clarify
the change.

This appears to reduce the CPU time for running 300 invocations of
hg log -r. -T '{rev}' on my i7-6700K:

before: 18.64s
after: 18.44s

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

indygreg created this revision.May 5 2018, 9:58 PM
martinvonz accepted this revision.May 6 2018, 12:14 AM
This revision is now accepted and ready to land.May 6 2018, 12:14 AM
Closed by commit rHG670eb4fa1b86: demandimport: make module ignores a set (API) (authored by indygreg). · Explain WhyMay 6 2018, 12:41 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 8463

hgdemandimport/__init__.py

import os import os
import sys import sys
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:
from . import demandimportpy3 as demandimport from . import demandimportpy3 as demandimport
else: else:
from . import demandimportpy2 as demandimport from . import demandimportpy2 as demandimport
# Extensions can add to this list if necessary. # Full module names which can't be lazy imported.
ignore = [ # Extensions can add to this set.
IGNORES = {
'__future__', '__future__',
'_hashlib', '_hashlib',
# ImportError during pkg_resources/__init__.py:fixup_namespace_package # ImportError during pkg_resources/__init__.py:fixup_namespace_package
'_imp', '_imp',
'_xmlplus', '_xmlplus',
'fcntl', 'fcntl',
'nt', # pathlib2 tests the existence of built-in 'nt' module 'nt', # pathlib2 tests the existence of built-in 'nt' module
'win32com.gen_py', 'win32com.gen_py',
'rfc822', 'rfc822',
'mimetools', 'mimetools',
'sqlalchemy.events', # has import-time side effects (issue5085) 'sqlalchemy.events', # has import-time side effects (issue5085)
# setuptools 8 expects this module to explode early when not on windows # setuptools 8 expects this module to explode early when not on windows
'distutils.msvc9compiler', 'distutils.msvc9compiler',
'__builtin__', '__builtin__',
'builtins', 'builtins',
'urwid.command_map', # for pudb 'urwid.command_map', # for pudb
] }
_pypy = '__pypy__' in sys.builtin_module_names _pypy = '__pypy__' in sys.builtin_module_names
if _pypy: if _pypy:
ignore.extend([
# _ctypes.pointer is shadowed by "from ... import pointer" (PyPy 5) # _ctypes.pointer is shadowed by "from ... import pointer" (PyPy 5)
'_ctypes.pointer', IGNORES.add('_ctypes.pointer')
])
demandimport.init(ignore) demandimport.init(IGNORES)
# Re-export. # Re-export.
isenabled = demandimport.isenabled isenabled = demandimport.isenabled
disable = demandimport.disable disable = demandimport.disable
deactivated = demandimport.deactivated deactivated = demandimport.deactivated
def enable(): def enable():
# chg pre-imports modules so do not enable demandimport for it # chg pre-imports modules so do not enable demandimport for it
if ('CHGINTERNALMARK' not in os.environ if ('CHGINTERNALMARK' not in os.environ
and os.environ.get('HGDEMANDIMPORT') != 'disable'): and os.environ.get('HGDEMANDIMPORT') != 'disable'):
demandimport.enable() demandimport.enable()

hgdemandimport/demandimportpy2.py

@property @property
def __doc__(self): def __doc__(self):
self._load() self._load()
return self._module.__doc__ return self._module.__doc__
_pypy = '__pypy__' in sys.builtin_module_names _pypy = '__pypy__' in sys.builtin_module_names
def _demandimport(name, globals=None, locals=None, fromlist=None, level=-1): def _demandimport(name, globals=None, locals=None, fromlist=None, level=-1):
if locals is None or name in ignore or fromlist == ('*',): if locals is None or name in ignores or fromlist == ('*',):
# these cases we can't really delay # these cases we can't really delay
return _hgextimport(_origimport, name, globals, locals, fromlist, level) return _hgextimport(_origimport, name, globals, locals, fromlist, level)
elif not fromlist: elif not fromlist:
# import a [as b] # import a [as b]
if '.' in name: # a.b if '.' in name: # a.b
base, rest = name.split('.', 1) base, rest = name.split('.', 1)
# email.__init__ loading email.mime # email.__init__ loading email.mime
if globals and globals.get('__name__', None) == base: if globals and globals.get('__name__', None) == base:
if symbol is nothing: if symbol is nothing:
if nonpkg: if nonpkg:
# do not try relative import, which would raise ValueError, # do not try relative import, which would raise ValueError,
# and leave unknown attribute as the default __import__() # and leave unknown attribute as the default __import__()
# would do. the missing attribute will be detected later # would do. the missing attribute will be detected later
# while processing the import statement. # while processing the import statement.
return return
mn = '%s.%s' % (mod.__name__, attr) mn = '%s.%s' % (mod.__name__, attr)
if mn in ignore: if mn in ignores:
importfunc = _origimport importfunc = _origimport
else: else:
importfunc = _demandmod importfunc = _demandmod
symbol = importfunc(attr, mod.__dict__, locals, level=1) symbol = importfunc(attr, mod.__dict__, locals, level=1)
setattr(mod, attr, symbol) setattr(mod, attr, symbol)
# Record the importing module references this symbol so we can # Record the importing module references this symbol so we can
# replace the symbol with the actual module instance at load # replace the symbol with the actual module instance at load
mod = _hgextimport(_origimport, name, globals, locals) mod = _hgextimport(_origimport, name, globals, locals)
mod = chainmodules(mod, name) mod = chainmodules(mod, name)
for x in fromlist: for x in fromlist:
processfromitem(mod, x) processfromitem(mod, x)
return mod return mod
ignore = [] ignores = set()
def init(ignorelist): def init(ignoreset):
global ignore global ignores
ignore = ignorelist ignores = ignoreset
def isenabled(): def isenabled():
return builtins.__import__ == _demandimport return builtins.__import__ == _demandimport
def enable(): def enable():
"enable global demand-loading of modules" "enable global demand-loading of modules"
builtins.__import__ = _demandimport builtins.__import__ = _demandimport

hgdemandimport/demandimportpy3.py

_deactivated = False _deactivated = False
class _lazyloaderex(importlib.util.LazyLoader): class _lazyloaderex(importlib.util.LazyLoader):
"""This is a LazyLoader except it also follows the _deactivated global and """This is a LazyLoader except it also follows the _deactivated global and
the ignore list. the ignore list.
""" """
def exec_module(self, module): def exec_module(self, module):
"""Make the module load lazily.""" """Make the module load lazily."""
if _deactivated or module.__name__ in ignore: if _deactivated or module.__name__ in ignores:
self.loader.exec_module(module) self.loader.exec_module(module)
else: else:
super().exec_module(module) super().exec_module(module)
# This is 3.6+ because with Python 3.5 it isn't possible to lazily load # This is 3.6+ because with Python 3.5 it isn't possible to lazily load
# extensions. See the discussion in https://bugs.python.org/issue26186 for more. # extensions. See the discussion in https://bugs.python.org/issue26186 for more.
_extensions_loader = _lazyloaderex.factory( _extensions_loader = _lazyloaderex.factory(
importlib.machinery.ExtensionFileLoader) importlib.machinery.ExtensionFileLoader)
_bytecode_loader = _lazyloaderex.factory( _bytecode_loader = _lazyloaderex.factory(
importlib.machinery.SourcelessFileLoader) importlib.machinery.SourcelessFileLoader)
_source_loader = _lazyloaderex.factory(importlib.machinery.SourceFileLoader) _source_loader = _lazyloaderex.factory(importlib.machinery.SourceFileLoader)
def _makefinder(path): def _makefinder(path):
return importlib.machinery.FileFinder( return importlib.machinery.FileFinder(
path, path,
# This is the order in which loaders are passed in in core Python. # This is the order in which loaders are passed in in core Python.
(_extensions_loader, importlib.machinery.EXTENSION_SUFFIXES), (_extensions_loader, importlib.machinery.EXTENSION_SUFFIXES),
(_source_loader, importlib.machinery.SOURCE_SUFFIXES), (_source_loader, importlib.machinery.SOURCE_SUFFIXES),
(_bytecode_loader, importlib.machinery.BYTECODE_SUFFIXES), (_bytecode_loader, importlib.machinery.BYTECODE_SUFFIXES),
) )
ignore = [] ignores = set()
def init(ignorelist): def init(ignoreset):
global ignore global ignores
ignore = ignorelist ignores = ignoreset
def isenabled(): def isenabled():
return _makefinder in sys.path_hooks and not _deactivated return _makefinder in sys.path_hooks and not _deactivated
def disable(): def disable():
try: try:
while True: while True:
sys.path_hooks.remove(_makefinder) sys.path_hooks.remove(_makefinder)

hgext/convert/bzr.py

from mercurial.i18n import _ from mercurial.i18n import _
from mercurial import ( from mercurial import (
demandimport, demandimport,
error error
) )
from . import common from . import common
# these do not work with demandimport, blacklist # these do not work with demandimport, blacklist
demandimport.ignore.extend([ demandimport.IGNORES.update([
'bzrlib.transactions', 'bzrlib.transactions',
'bzrlib.urlutils', 'bzrlib.urlutils',
'ElementPath', 'ElementPath',
]) ])
try: try:
# bazaar imports # bazaar imports
import bzrlib.bzrdir import bzrlib.bzrdir

hgext/highlight/highlight.py

# highlight.py - highlight extension implementation file # highlight.py - highlight extension implementation file
# #
# Copyright 2007-2009 Adam Hupp <adam@hupp.org> and others # Copyright 2007-2009 Adam Hupp <adam@hupp.org> and others
# #
# This software may be used and distributed according to the terms of the # This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version. # GNU General Public License version 2 or any later version.
# #
# The original module was split in an interface and an implementation # The original module was split in an interface and an implementation
# file to defer pygments loading and speedup extension setup. # file to defer pygments loading and speedup extension setup.
from __future__ import absolute_import from __future__ import absolute_import
from mercurial import demandimport from mercurial import demandimport
demandimport.ignore.extend(['pkgutil', 'pkg_resources', '__main__']) demandimport.IGNORES.update(['pkgutil', 'pkg_resources', '__main__'])
from mercurial import ( from mercurial import (
encoding, encoding,
) )
from mercurial.utils import ( from mercurial.utils import (
stringutil, stringutil,
) )