( )⚙ D5461 fuzz: extract Python initialization to utility package

This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5461

fuzz: extract Python initialization to utility package
ClosedPublic

Authored by durin42 on Dec 20 2018, 1:41 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGef103c96ed33: fuzz: extract Python initialization to utility package
Summary

Avoids code duplication between fuzzers of parsers.so.

Diff Detail

Repository
rHG Mercurial
Lint
Lint Skipped
Unit
Unit Tests Skipped

Event Timeline

durin42 created this revision.Dec 20 2018, 1:41 AM
Closed by commit rHGef103c96ed33: fuzz: extract Python initialization to utility package (authored by durin42). · Explain WhyDec 20 2018, 7:33 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mcontrib/fuzz/Makefile (9 lines)
Mcontrib/fuzz/manifest.cc (36 lines)
AMcontrib/fuzz/pyutil.h (9 lines)
AMcontrib/fuzz/pyutil.cc (49 lines)

Diff 12910

contrib/fuzz/Makefile

CC = clang CC = clang
CXX = clang++ CXX = clang++
all: bdiff mpatch xdiff all: bdiff mpatch xdiff
fuzzutil.o: fuzzutil.cc fuzzutil.h fuzzutil.o: fuzzutil.cc fuzzutil.h
$(CXX) $(CXXFLAGS) -g -O1 \ $(CXX) $(CXXFLAGS) -g -O1 \
-std=c++17 \ -std=c++17 \
-I../../mercurial -c -o fuzzutil.o fuzzutil.cc -I../../mercurial -c -o fuzzutil.o fuzzutil.cc
fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzutil.h fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzutil.h
$(CXX) $(CXXFLAGS) -std=c++17 \ $(CXX) $(CXXFLAGS) -std=c++17 \
-I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc -I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc
pyutil.o: pyutil.cc pyutil.h
$(CXX) $(CXXFLAGS) -g -O1 \
`$$OUT/sanpy/bin/python-config --cflags` \
-I../../mercurial -c -o pyutil.o pyutil.cc
bdiff.o: ../../mercurial/bdiff.c bdiff.o: ../../mercurial/bdiff.c
$(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \ $(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \
../../mercurial/bdiff.c ../../mercurial/bdiff.c
bdiff: bdiff.cc bdiff.o fuzzutil.o bdiff: bdiff.cc bdiff.o fuzzutil.o
$(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \ $(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
-std=c++17 \ -std=c++17 \
-I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff -I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff
-I../../mercurial \ -I../../mercurial \
-c -o pathencode.o ../../mercurial/cext/pathencode.c -c -o pathencode.o ../../mercurial/cext/pathencode.c
revlog.o: sanpy ../../mercurial/cext/revlog.c revlog.o: sanpy ../../mercurial/cext/revlog.c
$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \ $(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
-I../../mercurial \ -I../../mercurial \
-c -o revlog.o ../../mercurial/cext/revlog.c -c -o revlog.o ../../mercurial/cext/revlog.c
manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \ $(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
-Wno-register -Wno-macro-redefined \ -Wno-register -Wno-macro-redefined \
-I../../mercurial manifest.cc \ -I../../mercurial manifest.cc \
manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o \ manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
-lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \ -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
-o $$OUT/manifest_fuzzer -o $$OUT/manifest_fuzzer
manifest_corpus.zip: manifest_corpus.zip:
python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip
clean: clean:
$(RM) *.o *_fuzzer \ $(RM) *.o *_fuzzer \
bdiff \ bdiff \
mpatch \ mpatch \
xdiff xdiff
oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip
.PHONY: all clean oss-fuzz sanpy .PHONY: all clean oss-fuzz sanpy

contrib/fuzz/manifest.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h"
#include <string> #include <string>
extern "C" { extern "C" {
/* TODO: use Python 3 for this fuzzing? */
PyMODINIT_FUNC initparsers(void);
static char cpypath[8192] = "\0";
static PyCodeObject *code; static PyCodeObject *code;
static PyObject *mainmod;
static PyObject *globals;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
const std::string subdir = "/sanpy/lib/python2.7"; contrib::initpy(*argv[0]);
/* HACK ALERT: we need a full Python installation built without
pymalloc and with ASAN, so we dump one in
$OUT/sanpy/lib/python2.7. This helps us wire that up. */
std::string selfpath(*argv[0]);
std::string pypath;
auto pos = selfpath.rfind("/");
if (pos == std::string::npos) {
char wd[8192];
getcwd(wd, 8192);
pypath = std::string(wd) + subdir;
} else {
pypath = selfpath.substr(0, pos) + subdir;
}
strncpy(cpypath, pypath.c_str(), pypath.size());
setenv("PYTHONPATH", cpypath, 1);
setenv("PYTHONNOUSERSITE", "1", 1);
/* prevent Python from looking up users in the fuzz environment */
setenv("PYTHONUSERBASE", cpypath, 1);
Py_SetPythonHome(cpypath);
Py_InitializeEx(0);
initparsers();
code = (PyCodeObject *)Py_CompileString(R"py( code = (PyCodeObject *)Py_CompileString(R"py(
from parsers import lazymanifest from parsers import lazymanifest
try: try:
lm = lazymanifest(mdata) lm = lazymanifest(mdata)
# iterate the whole thing, which causes the code to fully parse # iterate the whole thing, which causes the code to fully parse
# every line in the manifest # every line in the manifest
list(lm.iterentries()) list(lm.iterentries())
lm[b'xyzzy'] = (b'\0' * 20, 'x') lm[b'xyzzy'] = (b'\0' * 20, 'x')
# do an insert, text should change # do an insert, text should change
assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata) assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
del lm[b'xyzzy'] del lm[b'xyzzy']
# should be back to the same # should be back to the same
assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata) assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
mainmod = PyImport_AddModule("__main__");
globals = PyModule_GetDict(mainmod);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
PyObject *mtext = PyObject *mtext =
PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size); PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
PyObject *locals = PyDict_New(); PyObject *locals = PyDict_New();
PyDict_SetItemString(locals, "mdata", mtext); PyDict_SetItemString(locals, "mdata", mtext);
PyObject *res = PyEval_EvalCode(code, globals, locals); PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
if (!res) { if (!res) {
PyErr_Print(); PyErr_Print();
} }
Py_XDECREF(res); Py_XDECREF(res);
Py_DECREF(locals); Py_DECREF(locals);
Py_DECREF(mtext); Py_DECREF(mtext);
return 0; // Non-zero return values are reserved for future use. return 0; // Non-zero return values are reserved for future use.
} }
} }

contrib/fuzz/pyutil.h

  • This file was added.
#include <Python.h>
namespace contrib
{
void initpy(const char *cselfpath);
PyObject *pyglobals();
} // namespace contrib

contrib/fuzz/pyutil.cc

  • This file was added.
#include "pyutil.h"
#include <string>
namespace contrib
{
static char cpypath[8192] = "\0";
static PyObject *mainmod;
static PyObject *globals;
/* TODO: use Python 3 for this fuzzing? */
PyMODINIT_FUNC initparsers(void);
void initpy(const char *cselfpath)
{
const std::string subdir = "/sanpy/lib/python2.7";
/* HACK ALERT: we need a full Python installation built without
pymalloc and with ASAN, so we dump one in
$OUT/sanpy/lib/python2.7. This helps us wire that up. */
std::string selfpath(cselfpath);
std::string pypath;
auto pos = selfpath.rfind("/");
if (pos == std::string::npos) {
char wd[8192];
getcwd(wd, 8192);
pypath = std::string(wd) + subdir;
} else {
pypath = selfpath.substr(0, pos) + subdir;
}
strncpy(cpypath, pypath.c_str(), pypath.size());
setenv("PYTHONPATH", cpypath, 1);
setenv("PYTHONNOUSERSITE", "1", 1);
/* prevent Python from looking up users in the fuzz environment */
setenv("PYTHONUSERBASE", cpypath, 1);
Py_SetPythonHome(cpypath);
Py_InitializeEx(0);
mainmod = PyImport_AddModule("__main__");
globals = PyModule_GetDict(mainmod);
initparsers();
}
PyObject *pyglobals()
{
return globals;
}
} // namespace contrib