diff --git a/tests/test-doctest.py b/tests/test-doctest.py
--- a/tests/test-doctest.py
+++ b/tests/test-doctest.py
@@ -1,10 +1,12 @@
 # this is hack to make sure no escape characters are inserted into the output
 
 from __future__ import absolute_import
+from __future__ import print_function
 
 import doctest
 import os
 import re
+import subprocess
 import sys
 
 ispy3 = sys.version_info[0] >= 3
@@ -49,46 +51,116 @@
     runner.summarize()
 
 
-testmod('mercurial.changelog')
-testmod('mercurial.cmdutil')
-testmod('mercurial.color')
-testmod('mercurial.config')
-testmod('mercurial.dagparser', optionflags=doctest.NORMALIZE_WHITESPACE)
-testmod('mercurial.encoding')
-testmod('mercurial.fancyopts')
-testmod('mercurial.formatter')
-testmod('mercurial.hg')
-testmod('mercurial.hgweb.hgwebdir_mod')
-testmod('mercurial.match')
-testmod('mercurial.mdiff')
-testmod('mercurial.minirst')
-testmod('mercurial.parser')
-testmod('mercurial.patch')
-testmod('mercurial.pathutil')
-testmod('mercurial.pycompat')
-testmod('mercurial.revlogutils.deltas')
-testmod('mercurial.revset')
-testmod('mercurial.revsetlang')
-testmod('mercurial.simplemerge')
-testmod('mercurial.smartset')
-testmod('mercurial.store')
-testmod('mercurial.subrepo')
-testmod('mercurial.templater')
-testmod('mercurial.ui')
-testmod('mercurial.util')
-testmod('mercurial.util', testtarget='platform')  # windows.py or posix.py
-testmod('mercurial.utils.dateutil')
-testmod('mercurial.utils.stringutil')
-testmod('hgext.convert.convcmd')
-testmod('hgext.convert.cvsps')
-testmod('hgext.convert.filemap')
-testmod('hgext.convert.p4')
-testmod('hgext.convert.subversion')
-testmod('hgext.fix')
-testmod('hgext.mq')
-# Helper scripts in tests/ that have doctests:
-testmod('drawdag')
-testmod('test-run-tests')
+DONT_RUN = []
+
+# Exceptions to the defaults for a given detected module. The value for each
+# module name is a list of dicts that specify the kwargs to pass to testmod.
+# testmod is called once per item in the list, so an empty list will cause the
+# module to not be tested.
+testmod_arg_overrides = {
+    'i18n.check-translation': DONT_RUN,  # may require extra installation
+    'mercurial.dagparser': [{'optionflags': doctest.NORMALIZE_WHITESPACE}],
+    'mercurial.keepalive': DONT_RUN,  # >>> is an example, not a doctest
+    'mercurial.posix': DONT_RUN,  # run by mercurial.platform
+    'mercurial.statprof': DONT_RUN,  # >>> is an example, not a doctest
+    'mercurial.util': [{}, {'testtarget': 'platform'}],  # run twice!
+    'mercurial.windows': DONT_RUN,  # run by mercurial.platform
+    'tests.test-url': [{'optionflags': doctest.NORMALIZE_WHITESPACE}],
+}
+
+doctest_indicator = '\n\\s*>>> '
+fileset = 'set:(**.py and grep("%s"))' % doctest_indicator
+
+files = subprocess.check_output(
+    "hg files --print0 '%s'" % fileset,
+    shell=True,
+    cwd=os.path.dirname(os.environ['TESTDIR']),
+).split(b'\0')
+
+mods_tested = set()
+for f in files:
+    if not f:
+        continue
+
+    if ispy3:
+        f = f.decode()
+
+    modname = f.replace('.py', '').replace('\\', '.').replace('/', '.')
+
+    # Third-party modules aren't our responsibility to test, and the modules in
+    # contrib generally do not have doctests in a good state, plus they're hard
+    # to import if this test is running with py2, so we just skip both for now.
+    if modname.startswith('mercurial.thirdparty.') or modname.startswith(
+        'contrib.'
+    ):
+        continue
+
+    for kwargs in testmod_arg_overrides.get(modname, [{}]):
+        mods_tested.add((modname, '%r' % (kwargs,)))
+        if modname.startswith('tests.'):
+            # On py2, we can't import from tests.foo, but it works on both py2
+            # and py3 with the way that PYTHONPATH is setup to import without
+            # the 'tests.' prefix, so we do that.
+            modname = modname[len('tests.') :]
+
+        testmod(modname, **kwargs)
 
-# Disabled since it requires extra modules that might not be installed.
-# testmod('i18n.check-translation')
+# Meta-test: let's make sure that we actually ran what we expected to, above.
+# Each item in the set is a 2-tuple of module name and stringified kwargs passed
+# to testmod.
+expected_mods_tested = set(
+    [
+        ('hgext.convert.convcmd', '{}'),
+        ('hgext.convert.cvsps', '{}'),
+        ('hgext.convert.filemap', '{}'),
+        ('hgext.convert.p4', '{}'),
+        ('hgext.convert.subversion', '{}'),
+        ('hgext.fix', '{}'),
+        ('hgext.mq', '{}'),
+        ('mercurial.changelog', '{}'),
+        ('mercurial.cmdutil', '{}'),
+        ('mercurial.color', '{}'),
+        ('mercurial.config', '{}'),
+        ('mercurial.dagparser', "{'optionflags': 4}"),
+        ('mercurial.encoding', '{}'),
+        ('mercurial.fancyopts', '{}'),
+        ('mercurial.formatter', '{}'),
+        ('mercurial.hg', '{}'),
+        ('mercurial.hgweb.hgwebdir_mod', '{}'),
+        ('mercurial.match', '{}'),
+        ('mercurial.mdiff', '{}'),
+        ('mercurial.minirst', '{}'),
+        ('mercurial.parser', '{}'),
+        ('mercurial.patch', '{}'),
+        ('mercurial.pathutil', '{}'),
+        ('mercurial.pycompat', '{}'),
+        ('mercurial.revlogutils.deltas', '{}'),
+        ('mercurial.revset', '{}'),
+        ('mercurial.revsetlang', '{}'),
+        ('mercurial.simplemerge', '{}'),
+        ('mercurial.smartset', '{}'),
+        ('mercurial.store', '{}'),
+        ('mercurial.subrepo', '{}'),
+        ('mercurial.templater', '{}'),
+        ('mercurial.ui', '{}'),
+        ('mercurial.util', "{'testtarget': 'platform'}"),
+        ('mercurial.util', '{}'),
+        ('mercurial.utils.dateutil', '{}'),
+        ('mercurial.utils.stringutil', '{}'),
+        ('tests.drawdag', '{}'),
+        ('tests.test-run-tests', '{}'),
+        ('tests.test-url', "{'optionflags': 4}"),
+    ]
+)
+
+unexpectedly_run = mods_tested.difference(expected_mods_tested)
+not_run = expected_mods_tested.difference(mods_tested)
+
+if unexpectedly_run:
+    print('Unexpectedly ran (probably need to add to list):')
+    for r in sorted(unexpectedly_run):
+        print('  %r' % (r,))
+if not_run:
+    print('Expected to run, but was not run (doctest removed?):')
+    for r in sorted(not_run):
+        print('  %r' % (r,))