diff --git a/contrib/check-py3-tokenize.py b/contrib/check-py3-tokenize.py new file mode 100755 --- /dev/null +++ b/contrib/check-py3-tokenize.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# +# check-py3-tokenize - check that our Python 3 tokenizer works +# +# Copyright Gregory Szorc +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import, print_function + +import io +import sys + +from hgdemandimport import ( + py3tokenize, +) + +def checkfile(path): + with open(path, 'rb') as fh: + b = io.BytesIO(fh.read()) + + b.seek(0) + + try: + tokens = list(py3tokenize.tokenize(b.readline)) + except Exception as e: + print('%s failed to tokenize: %s' % (path, e)) + return False + + try: + py3tokenize.untokenize(tokens) + except Exception as e: + print('%s failed to untokenize: %' % (path, e)) + return False + + return True + +if __name__ == '__main__': + argv = sys.argv + + if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2): + print('Usage: %s {-|file [file] [file] ...}') + sys.exit(1) + + if argv[1] == '-': + argv = argv[:1] + argv.extend(l.rstrip() for l in sys.stdin.readlines()) + + code = 0 + + for path in argv[1:]: + if not checkfile(path): + code = 1 + + sys.exit(code) diff --git a/contrib/import-checker.py b/contrib/import-checker.py --- a/contrib/import-checker.py +++ b/contrib/import-checker.py @@ -24,6 +24,7 @@ '__future__', 'bzrlib', 'hgclient', + 'hgdemandimport', 'mercurial', 'mercurial.hgweb.common', 'mercurial.hgweb.request', diff --git a/tests/test-check-py3-tokenize.t b/tests/test-check-py3-tokenize.t new file mode 100644 --- /dev/null +++ b/tests/test-check-py3-tokenize.t @@ -0,0 +1,8 @@ +#require test-repo + + $ . "$TESTDIR/helpers-testrepo.sh" + $ cd "$TESTDIR"/.. + + $ testrepohg files 'set:**.py' \ + > 'tests/**.t' \ + > | sed 's-\\-/-g' | "$PYTHON" contrib/check-py3-tokenize.py -