diff --git a/hgdemandimport/py3tokenize.py b/hgdemandimport/py3tokenize.py
--- a/hgdemandimport/py3tokenize.py
+++ b/hgdemandimport/py3tokenize.py
@@ -63,6 +63,7 @@
 # * absolute_import added.
 # * Removed re.ASCII.
 # * Various backports to work on Python 2.7.
+# * Dropped support for non-ASCII identifiers (PEP 3131 support).
 
 from __future__ import absolute_import
 
@@ -75,6 +76,7 @@
 from itertools import chain
 import itertools as _itertools
 import re
+import string
 from .py3token import *
 
 cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)')
@@ -524,6 +526,7 @@
 
 def _tokenize(readline, encoding):
     lnum = parenlev = continued = 0
+    namechars = string.ascii_letters + '_'
     numchars = '0123456789'
     contstr, needcont = '', 0
     contline = None
@@ -682,7 +685,8 @@
                     else:                                  # ordinary string
                         yield TokenInfo(STRING, token, spos, epos, line)
 
-                elif initial.isidentifier():               # ordinary name
+                # This doesn't conform to PEP 3131.
+                elif initial in namechars:                 # ordinary name
                     yield TokenInfo(NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     continued = 1