This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5045

hgdemandimport: remove unused functionality from vendored modules
AbandonedPublic

Authored by indygreg on Oct 13 2018, 3:28 AM.

Details

Reviewers
baymax
Group Reviewers
hg-reviewers
Summary

We'll never use this. It is just bloating the files.

The PSF license section 3 stipulates that derived works...
"include in any such work a brief summary of the changes made to Python."
While I think commit messages and version control count, I'm going one
step further and documenting things inline.

This change makes test-check-module-imports.t happy.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

indygreg created this revision.Oct 13 2018, 3:28 AM
baymax requested changes to this revision.Jan 24 2020, 12:32 PM

There seems to have been no activities on this Diff for the past 3 Months.

By policy, we are automatically moving it out of the need-review state.

Please, move it back to need-review without hesitation if this diff should still be discussed.

:baymax:need-review-idle:

This revision now requires changes to proceed.Jan 24 2020, 12:32 PM
indygreg abandoned this revision.Jan 24 2020, 11:50 PM

Revision Contents
Changeset List

PathPackages
Mhgdemandimport/py3token.py (76 lines)
Mhgdemandimport/py3tokenize.py (117 lines)

Diff 12000

hgdemandimport/py3token.py

# Vendored copy of Lib/token.py from cpython. # Vendored copy of Lib/token.py from cpython with modifications.
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
# Rights Reserved # Rights Reserved
# #
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
# -------------------------------------------- # --------------------------------------------
# #
# Licensee. This License Agreement does not grant permission to use PSF # Licensee. This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote # trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party. # products or services of Licensee, or any third party.
# #
# 8. By copying, installing or otherwise using Python, Licensee # 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License # agrees to be bound by the terms and conditions of this License
# Agreement. # Agreement.
# Changes from official Python source code:
#
# * _main() and related functionality removed.
"""Token constants (from "token.h").""" """Token constants (from "token.h")."""
__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
# This file is automatically generated; please don't muck it up! # This file is automatically generated; please don't muck it up!
# #
# To update the symbols in this file, 'cd' to the top directory of # To update the symbols in this file, 'cd' to the top directory of
# the python source tree after building the interpreter and run: # the python source tree after building the interpreter and run:
def ISTERMINAL(x): def ISTERMINAL(x):
return x < NT_OFFSET return x < NT_OFFSET
def ISNONTERMINAL(x): def ISNONTERMINAL(x):
return x >= NT_OFFSET return x >= NT_OFFSET
def ISEOF(x): def ISEOF(x):
return x == ENDMARKER return x == ENDMARKER
def _main():
import re
import sys
args = sys.argv[1:]
inFileName = args and args[0] or "Include/token.h"
outFileName = "Lib/token.py"
if len(args) > 1:
outFileName = args[1]
try:
fp = open(inFileName)
except OSError as err:
sys.stdout.write("I/O error: %s\n" % str(err))
sys.exit(1)
with fp:
lines = fp.read().split("\n")
prog = re.compile(
r"#define[ \t][ \t]*([A-Z0-9][A-Z0-9_]*)[ \t][ \t]*([0-9][0-9]*)",
re.IGNORECASE)
comment_regex = re.compile(
r"^\s*/\*\s*(.+?)\s*\*/\s*$",
re.IGNORECASE)
tokens = {}
prev_val = None
for line in lines:
match = prog.match(line)
if match:
name, val = match.group(1, 2)
val = int(val)
tokens[val] = {'token': name} # reverse so we can sort them...
prev_val = val
else:
comment_match = comment_regex.match(line)
if comment_match and prev_val is not None:
comment = comment_match.group(1)
tokens[prev_val]['comment'] = comment
keys = sorted(tokens.keys())
# load the output skeleton from the target:
try:
fp = open(outFileName)
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(2)
with fp:
format = fp.read().split("\n")
try:
start = format.index("#--start constants--") + 1
end = format.index("#--end constants--")
except ValueError:
sys.stderr.write("target does not contain format markers")
sys.exit(3)
lines = []
for key in keys:
lines.append("%s = %d" % (tokens[key]["token"], key))
if "comment" in tokens[key]:
lines.append("# %s" % tokens[key]["comment"])
format[start:end] = lines
try:
fp = open(outFileName, 'w')
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(4)
with fp:
fp.write("\n".join(format))
if __name__ == "__main__":
_main()

hgdemandimport/py3tokenize.py

# Vendored copy of Lib/tokenize.py from cpython. # Vendored copy of Lib/tokenize.py from cpython with modifications.
# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All
# Rights Reserved # Rights Reserved
# #
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
# -------------------------------------------- # --------------------------------------------
# #
# Licensee. This License Agreement does not grant permission to use PSF # Licensee. This License Agreement does not grant permission to use PSF
# trademarks or trade name in a trademark sense to endorse or promote # trademarks or trade name in a trademark sense to endorse or promote
# products or services of Licensee, or any third party. # products or services of Licensee, or any third party.
# #
# 8. By copying, installing or otherwise using Python, Licensee # 8. By copying, installing or otherwise using Python, Licensee
# agrees to be bound by the terms and conditions of this License # agrees to be bound by the terms and conditions of this License
# Agreement. # Agreement.
"""Tokenization help for Python programs. # Changes from official Python source code:
#
tokenize(readline) is a generator that breaks a stream of bytes into # * Removed main() and related functionality.
Python tokens. It decodes the bytes according to PEP-0263 for # * Removed generate_tokens().
determining source file encoding. # * Removed open().
# * Removed module docstring.
It accepts a readline-like method which is called repeatedly to get the
next line of input (or b"" for EOF). It generates 5-tuples with these
members:
the token type (see token.py)
the token (a string)
the starting (row, column) indices of the token (a 2-tuple of ints)
the ending (row, column) indices of the token (a 2-tuple of ints)
the original line (string)
It is designed to match the working of the Python tokenizer exactly, except
that it produces COMMENT tokens for comments and gives type OP for all
operators. Additionally, all token lists start with an ENCODING token
which tells you which encoding was used to decode the bytes stream.
"""
__author__ = 'Ka-Ping Yee <ping@lfw.org>' __author__ = 'Ka-Ping Yee <ping@lfw.org>'
__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
'Michael Foord') 'Michael Foord')
from builtins import open as _builtin_open
from codecs import lookup, BOM_UTF8 from codecs import lookup, BOM_UTF8
import collections import collections
from io import TextIOWrapper
from itertools import chain from itertools import chain
import itertools as _itertools import itertools as _itertools
import re import re
import sys
from token import * from token import *
cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
import token import token
__all__ = token.__all__ + ["tokenize", "detect_encoding", __all__ = token.__all__ + ["tokenize", "detect_encoding",
"untokenize", "TokenInfo"] "untokenize", "TokenInfo"]
encoding = find_cookie(second) encoding = find_cookie(second)
if encoding: if encoding:
return encoding, [first, second] return encoding, [first, second]
return default, [first, second] return default, [first, second]
def open(filename):
"""Open a file in read only mode using the encoding detected by
detect_encoding().
"""
buffer = _builtin_open(filename, 'rb')
try:
encoding, lines = detect_encoding(buffer.readline)
buffer.seek(0)
text = TextIOWrapper(buffer, encoding, line_buffering=True)
text.mode = 'r'
return text
except:
buffer.close()
raise
def tokenize(readline): def tokenize(readline):
""" """
The tokenize() generator requires one argument, readline, which The tokenize() generator requires one argument, readline, which
must be a callable object which provides the same interface as the must be a callable object which provides the same interface as the
readline() method of built-in file objects. Each call to the function readline() method of built-in file objects. Each call to the function
should return one line of input as bytes. Alternatively, readline should return one line of input as bytes. Alternatively, readline
can be a callable function terminating with StopIteration: can be a callable function terminating with StopIteration:
readline = open(myfile, 'rb').__next__ # Example of alternate readline readline = open(myfile, 'rb').__next__ # Example of alternate readline
pos += 1 pos += 1
# Add an implicit NEWLINE if the input doesn't end in one # Add an implicit NEWLINE if the input doesn't end in one
if last_line and last_line[-1] not in '\r\n': if last_line and last_line[-1] not in '\r\n':
yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
for indent in indents[1:]: # pop remaining indent levels for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
# An undocumented, backwards compatible, API for all the places in the standard
# library that expect to be able to use tokenize with strings
def generate_tokens(readline):
return _tokenize(readline, None)
def main():
import argparse
# Helper error handling routines
def perror(message):
print(message, file=sys.stderr)
def error(message, filename=None, location=None):
if location:
args = (filename,) + location + (message,)
perror("%s:%d:%d: error: %s" % args)
elif filename:
perror("%s: error: %s" % (filename, message))
else:
perror("error: %s" % message)
sys.exit(1)
# Parse the arguments and options
parser = argparse.ArgumentParser(prog='python -m tokenize')
parser.add_argument(dest='filename', nargs='?',
metavar='filename.py',
help='the file to tokenize; defaults to stdin')
parser.add_argument('-e', '--exact', dest='exact', action='store_true',
help='display token names using the exact type')
args = parser.parse_args()
try:
# Tokenize the input
if args.filename:
filename = args.filename
with _builtin_open(filename, 'rb') as f:
tokens = list(tokenize(f.readline))
else:
filename = "<stdin>"
tokens = _tokenize(sys.stdin.readline, None)
# Output the tokenization
for token in tokens:
token_type = token.type
if args.exact:
token_type = token.exact_type
token_range = "%d,%d-%d,%d:" % (token.start + token.end)
print("%-20s%-15s%-15r" %
(token_range, tok_name[token_type], token.string))
except IndentationError as err:
line, column = err.args[1][1:3]
error(err.args[0], filename, (line, column))
except TokenError as err:
line, column = err.args[1]
error(err.args[0], filename, (line, column))
except SyntaxError as err:
error(err, filename)
except OSError as err:
error(err)
except KeyboardInterrupt:
print("interrupted\n")
except Exception as err:
perror("unexpected error: %s" % err)
raise
if __name__ == "__main__":
main()