This is an archive of the discontinued Mercurial Phabricator instance.

Differential D6575

pycompat: make fewer assumptions about sys.executable
ClosedPublic

Authored by rdamazio on Jun 25 2019, 10:37 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG49998d5ba66a: pycompat: make fewer assumptions about sys.executable
Summary

There are many Python "bundlers" which create an archive to run a Python binary
from, and they may not set sys.executable at all - handle that case properly,
especially to run tests.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

rdamazio created this revision.Jun 25 2019, 10:37 PM
rdamazio updated this revision to Diff 15663.Jun 25 2019, 11:05 PM
pulkit accepted this revision.Jun 27 2019, 11:41 AM
This revision is now accepted and ready to land.Jun 27 2019, 11:41 AM
rdamazio added a commit: rHG49998d5ba66a: pycompat: make fewer assumptions about sys.executable.Jun 27 2019, 12:19 PM
Closed by commit rHG49998d5ba66a: pycompat: make fewer assumptions about sys.executable (authored by rdamazio). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mmercurial/chgserver.py (4 lines)
Mmercurial/debugcommands.py (2 lines)
Mtests/run-tests.py (27 lines)

Diff 15675

mercurial/chgserver.py

- python binary - python binary
""" """
modules = [m for n, m in extensions.extensions(ui)] modules = [m for n, m in extensions.extensions(ui)]
try: try:
from . import __version__ from . import __version__
modules.append(__version__) modules.append(__version__)
except ImportError: except ImportError:
pass pass
files = [pycompat.sysexecutable] files = []
if pycompat.sysexecutable:
files.append(pycompat.sysexecutable)
for m in modules: for m in modules:
try: try:
files.append(pycompat.fsencode(inspect.getabsfile(m))) files.append(pycompat.fsencode(inspect.getabsfile(m)))
except TypeError: except TypeError:
pass pass
return sorted(set(files)) return sorted(set(files))
def _mtimehash(paths): def _mtimehash(paths):

mercurial/debugcommands.py

except LookupError as inst: except LookupError as inst:
err = stringutil.forcebytestr(inst) err = stringutil.forcebytestr(inst)
problems += 1 problems += 1
fm.condwrite(err, 'encodingerror', _(" %s\n" fm.condwrite(err, 'encodingerror', _(" %s\n"
" (check that your locale is properly set)\n"), err) " (check that your locale is properly set)\n"), err)
# Python # Python
fm.write('pythonexe', _("checking Python executable (%s)\n"), fm.write('pythonexe', _("checking Python executable (%s)\n"),
pycompat.sysexecutable) pycompat.sysexecutable or _("unknown"))
fm.write('pythonver', _("checking Python version (%s)\n"), fm.write('pythonver', _("checking Python version (%s)\n"),
("%d.%d.%d" % sys.version_info[:3])) ("%d.%d.%d" % sys.version_info[:3]))
fm.write('pythonlib', _("checking Python lib (%s)...\n"), fm.write('pythonlib', _("checking Python lib (%s)...\n"),
os.path.dirname(pycompat.fsencode(os.__file__))) os.path.dirname(pycompat.fsencode(os.__file__)))
security = set(sslutil.supportedprotocols) security = set(sslutil.supportedprotocols)
if sslutil.hassni: if sslutil.hassni:
security.add('sni') security.add('sni')

tests/run-tests.py

time.sleep(.1) time.sleep(.1)
p.timeout = True p.timeout = True
if p.returncode is None: if p.returncode is None:
terminate(p) terminate(p)
threading.Thread(target=t).start() threading.Thread(target=t).start()
return p return p
PYTHON = _bytespath(sys.executable.replace('\\', '/')) if sys.executable:
sysexecutable = sys.executable
elif os.environ.get('PYTHONEXECUTABLE'):
sysexecutable = os.environ['PYTHONEXECUTABLE']
elif os.environ.get('PYTHON'):
sysexecutable = os.environ['PYTHON']
else:
raise AssertionError('Could not find Python interpreter')
PYTHON = _bytespath(sysexecutable.replace('\\', '/'))
IMPL_PATH = b'PYTHONPATH' IMPL_PATH = b'PYTHONPATH'
if 'java' in sys.platform: if 'java' in sys.platform:
IMPL_PATH = b'JYTHONPATH' IMPL_PATH = b'JYTHONPATH'
defaults = { defaults = {
'jobs': ('HGTEST_JOBS', multiprocessing.cpu_count()), 'jobs': ('HGTEST_JOBS', multiprocessing.cpu_count()),
'timeout': ('HGTEST_TIMEOUT', 180), 'timeout': ('HGTEST_TIMEOUT', 180),
'slowtimeout': ('HGTEST_SLOWTIMEOUT', 1500), 'slowtimeout': ('HGTEST_SLOWTIMEOUT', 1500),
env['HOME'] = _strpath(self._testtmp) env['HOME'] = _strpath(self._testtmp)
# This number should match portneeded in _getport # This number should match portneeded in _getport
for port in xrange(3): for port in xrange(3):
# This list should be parallel to _portmap in _getreplacements # This list should be parallel to _portmap in _getreplacements
defineport(port) defineport(port)
env["HGRCPATH"] = _strpath(os.path.join(self._threadtmp, b'.hgrc')) env["HGRCPATH"] = _strpath(os.path.join(self._threadtmp, b'.hgrc'))
env["DAEMON_PIDS"] = _strpath(os.path.join(self._threadtmp, env["DAEMON_PIDS"] = _strpath(os.path.join(self._threadtmp,
b'daemon.pids')) b'daemon.pids'))
env["HGEDITOR"] = ('"' + sys.executable + '"' env["HGEDITOR"] = ('"' + sysexecutable + '"'
+ ' -c "import sys; sys.exit(0)"') + ' -c "import sys; sys.exit(0)"')
env["HGUSER"] = "test" env["HGUSER"] = "test"
env["HGENCODING"] = "ascii" env["HGENCODING"] = "ascii"
env["HGENCODINGMODE"] = "strict" env["HGENCODINGMODE"] = "strict"
env["HGHOSTNAME"] = "test-hostname" env["HGHOSTNAME"] = "test-hostname"
env['HGIPV6'] = str(int(self._useipv6)) env['HGIPV6'] = str(int(self._useipv6))
# See contrib/catapipe.py for how to use this functionality. # See contrib/catapipe.py for how to use this functionality.
if 'HGTESTCATAPULTSERVERPIPE' not in env: if 'HGTESTCATAPULTSERVERPIPE' not in env:
pread(bisectcmd + ['--bad', '.']) pread(bisectcmd + ['--bad', '.'])
pread(bisectcmd + ['--good', self._runner.options.known_good_rev]) pread(bisectcmd + ['--good', self._runner.options.known_good_rev])
# TODO: we probably need to forward more options # TODO: we probably need to forward more options
# that alter hg's behavior inside the tests. # that alter hg's behavior inside the tests.
opts = '' opts = ''
withhg = self._runner.options.with_hg withhg = self._runner.options.with_hg
if withhg: if withhg:
opts += ' --with-hg=%s ' % shellquote(_strpath(withhg)) opts += ' --with-hg=%s ' % shellquote(_strpath(withhg))
rtc = '%s %s %s %s' % (sys.executable, sys.argv[0], opts, rtc = '%s %s %s %s' % (sysexecutable, sys.argv[0], opts,
test) test)
data = pread(bisectcmd + ['--command', rtc]) data = pread(bisectcmd + ['--command', rtc])
m = re.search( m = re.search(
(br'\nThe first (?P<goodbad>bad|good) revision ' (br'\nThe first (?P<goodbad>bad|good) revision '
br'is:\nchangeset: +\d+:(?P<node>[a-f0-9]+)\n.*\n' br'is:\nchangeset: +\d+:(?P<node>[a-f0-9]+)\n.*\n'
br'summary: +(?P<summary>[^\n]+)\n'), br'summary: +(?P<summary>[^\n]+)\n'),
data, (re.MULTILINE | re.DOTALL)) data, (re.MULTILINE | re.DOTALL))
if m is None: if m is None:
"""Configure the environment to use the appropriate Python in tests.""" """Configure the environment to use the appropriate Python in tests."""
# Tests must use the same interpreter as us or bad things will happen. # Tests must use the same interpreter as us or bad things will happen.
pyexename = sys.platform == 'win32' and b'python.exe' or b'python' pyexename = sys.platform == 'win32' and b'python.exe' or b'python'
# os.symlink() is a thing with py3 on Windows, but it requires # os.symlink() is a thing with py3 on Windows, but it requires
# Administrator rights. # Administrator rights.
if getattr(os, 'symlink', None) and os.name != 'nt': if getattr(os, 'symlink', None) and os.name != 'nt':
vlog("# Making python executable in test path a symlink to '%s'" % vlog("# Making python executable in test path a symlink to '%s'" %
sys.executable) sysexecutable)
mypython = os.path.join(self._tmpbindir, pyexename) mypython = os.path.join(self._tmpbindir, pyexename)
try: try:
if os.readlink(mypython) == sys.executable: if os.readlink(mypython) == sysexecutable:
return return
os.unlink(mypython) os.unlink(mypython)
except OSError as err: except OSError as err:
if err.errno != errno.ENOENT: if err.errno != errno.ENOENT:
raise raise
if self._findprogram(pyexename) != sys.executable: if self._findprogram(pyexename) != sysexecutable:
try: try:
os.symlink(sys.executable, mypython) os.symlink(sysexecutable, mypython)
self._createdfiles.append(mypython) self._createdfiles.append(mypython)
except OSError as err: except OSError as err:
# child processes may race, which is harmless # child processes may race, which is harmless
if err.errno != errno.EEXIST: if err.errno != errno.EEXIST:
raise raise
else: else:
exedir, exename = os.path.split(sys.executable) exedir, exename = os.path.split(sysexecutable)
vlog("# Modifying search path to find %s as %s in '%s'" % vlog("# Modifying search path to find %s as %s in '%s'" %
(exename, pyexename, exedir)) (exename, pyexename, exedir))
path = os.environ['PATH'].split(os.pathsep) path = os.environ['PATH'].split(os.pathsep)
while exedir in path: while exedir in path:
path.remove(exedir) path.remove(exedir)
os.environ['PATH'] = os.pathsep.join([exedir] + path) os.environ['PATH'] = os.pathsep.join([exedir] + path)
if not self._findprogram(pyexename): if not self._findprogram(pyexename):
print("WARNING: Cannot find %s in search path" % pyexename) print("WARNING: Cannot find %s in search path" % pyexename)
compiler = '--compiler ' + self.options.compiler compiler = '--compiler ' + self.options.compiler
if self.options.pure: if self.options.pure:
pure = b"--pure" pure = b"--pure"
else: else:
pure = b"" pure = b""
# Run installer in hg root # Run installer in hg root
script = os.path.realpath(sys.argv[0]) script = os.path.realpath(sys.argv[0])
exe = sys.executable exe = sysexecutable
if PYTHON3: if PYTHON3:
compiler = _bytespath(compiler) compiler = _bytespath(compiler)
script = _bytespath(script) script = _bytespath(script)
exe = _bytespath(exe) exe = _bytespath(exe)
hgroot = os.path.dirname(os.path.dirname(script)) hgroot = os.path.dirname(os.path.dirname(script))
self._hgroot = hgroot self._hgroot = hgroot
os.chdir(hgroot) os.chdir(hgroot)
nohome = b'--home=""' nohome = b'--home=""'