This is an archive of the discontinued Mercurial Phabricator instance.

Differential D7592

fuzz: add support for fuzzing under either Python 2 or 3
ClosedPublic

Authored by durin42 on Dec 9 2019, 10:21 PM.

Details

Reviewers
pulkit
Group Reviewers
hg-reviewers
Commits
rHG8766728dbce6: fuzz: add support for fuzzing under either Python 2 or 3
Summary

This was more of a hairball than I hoped, but it appears to work. The hg-py3
branch of my oss-fuzz fork on github has the remaining changes to switch us to
Python 3, but we may as well retain Python 2 fuzzing support for at least a
little while.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Dec 9 2019, 10:21 PM
durin42 added a comment.Dec 9 2019, 10:26 PM

Note: please review the embedded Python segments carefully, as it's a little fidgety to get right and verify I didn't do something Really Dumb.

pulkit accepted this revision.Dec 12 2019, 7:22 AM
This revision is now accepted and ready to land.Dec 12 2019, 7:22 AM
durin42 added a commit: rHG8766728dbce6: fuzz: add support for fuzzing under either Python 2 or 3.Dec 12 2019, 7:52 AM
Closed by commit rHG8766728dbce6: fuzz: add support for fuzzing under either Python 2 or 3 (authored by durin42). · Explain Why
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

Diff 18630

contrib/fuzz/dirs.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h" #include "pyutil.h"
#include <string> #include <string>
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import dirs
try: try:
files = mdata.split('\n') files = mdata.split('\n')
d = dirs(files) d = parsers.dirs(files)
list(d) list(d)
'a' in d 'a' in d
if files: if files:
files[0] in d files[0] in d
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
// Don't allow fuzzer inputs larger than 100k, since we'll just bog // Don't allow fuzzer inputs larger than 100k, since we'll just bog
// down and not accomplish much. // down and not accomplish much.
if (Size > 100000) { if (Size > 100000) {

contrib/fuzz/dirstate.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <string> #include <string>
#include "pyutil.h" #include "pyutil.h"
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import parse_dirstate
try: try:
dmap = {} dmap = {}
copymap = {} copymap = {}
p = parse_dirstate(dmap, copymap, data) p = parsers.parse_dirstate(dmap, copymap, data)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
PyObject *text = PyObject *text =
PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size); PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
PyObject *locals = PyDict_New(); PyObject *locals = PyDict_New();

contrib/fuzz/fm1readmarkers.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <string> #include <string>
#include "pyutil.h" #include "pyutil.h"
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import fm1readmarkers
def maybeint(s, default): def maybeint(s, default):
try: try:
return int(s) return int(s)
except ValueError: except ValueError:
return default return default
try: try:
parts = data.split('\0', 2) parts = data.split('\0', 2)
if len(parts) == 3: if len(parts) == 3:
offset, stop, data = parts offset, stop, data = parts
elif len(parts) == 2: elif len(parts) == 2:
stop, data = parts stop, data = parts
offset = 0 offset = 0
else: else:
offset = stop = 0 offset = stop = 0
offset, stop = maybeint(offset, 0), maybeint(stop, len(data)) offset, stop = maybeint(offset, 0), maybeint(stop, len(data))
fm1readmarkers(data, offset, stop) parsers.fm1readmarkers(data, offset, stop)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
PyObject *text = PyObject *text =
PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size); PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
PyObject *locals = PyDict_New(); PyObject *locals = PyDict_New();

contrib/fuzz/fncache.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h" #include "pyutil.h"
#include <iostream> #include <iostream>
#include <string> #include <string>
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import (
isasciistr,
asciilower,
asciiupper,
encodedir,
pathencode,
lowerencode,
)
try: try:
for fn in ( for fn in (
isasciistr, parsers.isasciistr,
asciilower, parsers.asciilower,
asciiupper, parsers.asciiupper,
encodedir, parsers.encodedir,
pathencode, parsers.pathencode,
lowerencode, parsers.lowerencode,
): ):
try: try:
fn(data) fn(data)
except UnicodeDecodeError: except UnicodeDecodeError:
pass # some functions emit this exception pass # some functions emit this exception
except AttributeError: except AttributeError:
# pathencode needs hashlib, which fails to import because the time # pathencode needs hashlib, which fails to import because the time
# module fails to import. We should try and fix that some day, but # module fails to import. We should try and fix that some day, but
# for now we at least get coverage on non-hashencoded codepaths. # for now we at least get coverage on non-hashencoded codepaths.
if fn != pathencode: if fn != pathencode:
raise raise
# uncomment this for debugging exceptions # uncomment this for debugging exceptions
# except Exception as e: # except Exception as e:
# raise Exception('%r: %r' % (fn, e)) # raise Exception('%r: %r' % (fn, e))
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print(e) # print(e)
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
if (!code) { if (!code) {
std::cerr << "failed to compile Python code!" << std::endl; std::cerr << "failed to compile Python code!" << std::endl;
} }
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {

contrib/fuzz/jsonescapeu8fast.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h" #include "pyutil.h"
#include <iostream> #include <iostream>
#include <string> #include <string>
#include "FuzzedDataProvider.h" #include "FuzzedDataProvider.h"
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import jsonescapeu8fast
try: try:
jsonescapeu8fast(data, paranoid) parsers.jsonescapeu8fast(data, paranoid)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print(e) # print(e)
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
if (!code) { if (!code) {
std::cerr << "failed to compile Python code!" << std::endl; std::cerr << "failed to compile Python code!" << std::endl;
} }
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {

contrib/fuzz/manifest.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h" #include "pyutil.h"
#include <string> #include <string>
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import lazymanifest
try: try:
lm = lazymanifest(mdata) lm = parsers.lazymanifest(mdata)
# iterate the whole thing, which causes the code to fully parse # iterate the whole thing, which causes the code to fully parse
# every line in the manifest # every line in the manifest
for e, _, _ in lm.iterentries(): for e, _, _ in lm.iterentries():
# also exercise __getitem__ et al # also exercise __getitem__ et al
lm[e] lm[e]
e in lm e in lm
(e + 'nope') in lm (e + 'nope') in lm
lm[b'xyzzy'] = (b'\0' * 20, 'x') lm[b'xyzzy'] = (b'\0' * 20, 'x')
# do an insert, text should change # do an insert, text should change
assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata) assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
cloned = lm.filtercopy(lambda x: x != 'xyzzy') cloned = lm.filtercopy(lambda x: x != 'xyzzy')
assert cloned.text() == mdata, 'cloned text should equal mdata' assert cloned.text() == mdata, 'cloned text should equal mdata'
cloned.diff(lm) cloned.diff(lm)
del lm[b'xyzzy'] del lm[b'xyzzy']
cloned.diff(lm) cloned.diff(lm)
# should be back to the same # should be back to the same
assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata) assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
// Don't allow fuzzer inputs larger than 100k, since we'll just bog // Don't allow fuzzer inputs larger than 100k, since we'll just bog
// down and not accomplish much. // down and not accomplish much.
if (Size > 100000) { if (Size > 100000) {

contrib/fuzz/pyutil.h

#include <Python.h> #include <Python.h>
#if PY_MAJOR_VERSION >= 3
#define PYCODETYPE PyObject
#else
#define PYCODETYPE PyCodeObject
#endif
namespace contrib namespace contrib
{ {
void initpy(const char *cselfpath); void initpy(const char *cselfpath);
PyObject *pyglobals(); PyObject *pyglobals();
} /* namespace contrib */ } /* namespace contrib */

contrib/fuzz/pyutil.cc

#include "pyutil.h" #include "pyutil.h"
#include <iostream> #include <iostream>
#include <string> #include <string>
namespace contrib namespace contrib
{ {
#if PY_MAJOR_VERSION >= 3
#define HG_FUZZER_PY3 1
PyMODINIT_FUNC PyInit_parsers(void);
#else
PyMODINIT_FUNC initparsers(void);
#endif
static char cpypath[8192] = "\0"; static char cpypath[8192] = "\0";
static PyObject *mainmod; static PyObject *mainmod;
static PyObject *globals; static PyObject *globals;
/* TODO: use Python 3 for this fuzzing? */
PyMODINIT_FUNC initparsers(void);
void initpy(const char *cselfpath) void initpy(const char *cselfpath)
{ {
#ifdef HG_FUZZER_PY3
const std::string subdir = "/sanpy/lib/python3.7";
#else
const std::string subdir = "/sanpy/lib/python2.7"; const std::string subdir = "/sanpy/lib/python2.7";
#endif
/* HACK ALERT: we need a full Python installation built without /* HACK ALERT: we need a full Python installation built without
pymalloc and with ASAN, so we dump one in pymalloc and with ASAN, so we dump one in
$OUT/sanpy/lib/python2.7. This helps us wire that up. */ $OUT/sanpy/lib/python2.7. This helps us wire that up. */
std::string selfpath(cselfpath); std::string selfpath(cselfpath);
std::string pypath; std::string pypath;
auto pos = selfpath.rfind("/"); auto pos = selfpath.rfind("/");
if (pos == std::string::npos) { if (pos == std::string::npos) {
char wd[8192]; char wd[8192];
if (!getcwd(wd, 8192)) { if (!getcwd(wd, 8192)) {
std::cerr << "Failed to call getcwd: errno " << errno std::cerr << "Failed to call getcwd: errno " << errno
<< std::endl; << std::endl;
exit(1); exit(1);
} }
pypath = std::string(wd) + subdir; pypath = std::string(wd) + subdir;
} else { } else {
pypath = selfpath.substr(0, pos) + subdir; pypath = selfpath.substr(0, pos) + subdir;
} }
strncpy(cpypath, pypath.c_str(), pypath.size()); strncpy(cpypath, pypath.c_str(), pypath.size());
setenv("PYTHONPATH", cpypath, 1); setenv("PYTHONPATH", cpypath, 1);
setenv("PYTHONNOUSERSITE", "1", 1); setenv("PYTHONNOUSERSITE", "1", 1);
/* prevent Python from looking up users in the fuzz environment */ /* prevent Python from looking up users in the fuzz environment */
setenv("PYTHONUSERBASE", cpypath, 1); setenv("PYTHONUSERBASE", cpypath, 1);
#ifdef HG_FUZZER_PY3
std::wstring wcpypath(pypath.begin(), pypath.end());
Py_SetPythonHome(wcpypath.c_str());
#else
Py_SetPythonHome(cpypath); Py_SetPythonHome(cpypath);
#endif
Py_InitializeEx(0); Py_InitializeEx(0);
mainmod = PyImport_AddModule("__main__"); mainmod = PyImport_AddModule("__main__");
globals = PyModule_GetDict(mainmod); globals = PyModule_GetDict(mainmod);
#ifdef HG_FUZZER_PY3
PyObject *mod = PyInit_parsers();
#else
initparsers(); initparsers();
PyObject *mod = PyImport_ImportModule("parsers");
#endif
PyDict_SetItemString(globals, "parsers", mod);
} }
PyObject *pyglobals() PyObject *pyglobals()
{ {
return globals; return globals;
} }
} // namespace contrib } // namespace contrib

contrib/fuzz/revlog.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <string> #include <string>
#include "pyutil.h" #include "pyutil.h"
extern "C" { extern "C" {
static PyCodeObject *code; static PYCODETYPE *code;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
contrib::initpy(*argv[0]); contrib::initpy(*argv[0]);
code = (PyCodeObject *)Py_CompileString(R"py( code = (PYCODETYPE *)Py_CompileString(R"py(
from parsers import parse_index2
for inline in (True, False): for inline in (True, False):
try: try:
index, cache = parse_index2(data, inline) index, cache = parsers.parse_index2(data, inline)
index.slicechunktodensity(list(range(len(index))), 0.5, 262144) index.slicechunktodensity(list(range(len(index))), 0.5, 262144)
index.stats() index.stats()
index.findsnapshots({}, 0) index.findsnapshots({}, 0)
10 in index 10 in index
for rev in range(len(index)): for rev in range(len(index)):
index.reachableroots(0, [len(index)-1], [rev]) index.reachableroots(0, [len(index)-1], [rev])
node = index[rev][7] node = index[rev][7]
partial = index.shortest(node) partial = index.shortest(node)
index.partialmatch(node[:partial]) index.partialmatch(node[:partial])
index.deltachain(rev, None, True) index.deltachain(rev, None, True)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
// Don't allow fuzzer inputs larger than 60k, since we'll just bog // Don't allow fuzzer inputs larger than 60k, since we'll just bog
// down and not accomplish much. // down and not accomplish much.
if (Size > 60000) { if (Size > 60000) {