This is an archive of the discontinued Mercurial Phabricator instance.

Differential D5461

fuzz: extract Python initialization to utility package
ClosedPublic

Authored by durin42 on Dec 20 2018, 1:41 AM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGef103c96ed33: fuzz: extract Python initialization to utility package
Summary

Avoids code duplication between fuzzers of parsers.so.

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

durin42 created this revision.Dec 20 2018, 1:41 AM
Closed by commit rHGef103c96ed33: fuzz: extract Python initialization to utility package (authored by durin42). · Explain WhyDec 20 2018, 7:33 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathPackages
Mcontrib/fuzz/Makefile (9 lines)
Mcontrib/fuzz/manifest.cc (36 lines)
AMcontrib/fuzz/pyutil.h (9 lines)
AMcontrib/fuzz/pyutil.cc (49 lines)

Diff 12922

contrib/fuzz/Makefile

CC = clang CC = clang
CXX = clang++ CXX = clang++
all: bdiff mpatch xdiff all: bdiff mpatch xdiff
fuzzutil.o: fuzzutil.cc fuzzutil.h fuzzutil.o: fuzzutil.cc fuzzutil.h
$(CXX) $(CXXFLAGS) -g -O1 \ $(CXX) $(CXXFLAGS) -g -O1 \
-std=c++17 \ -std=c++17 \
-I../../mercurial -c -o fuzzutil.o fuzzutil.cc -I../../mercurial -c -o fuzzutil.o fuzzutil.cc
fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzutil.h fuzzutil-oss-fuzz.o: fuzzutil.cc fuzzutil.h
$(CXX) $(CXXFLAGS) -std=c++17 \ $(CXX) $(CXXFLAGS) -std=c++17 \
-I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc -I../../mercurial -c -o fuzzutil-oss-fuzz.o fuzzutil.cc
pyutil.o: pyutil.cc pyutil.h
$(CXX) $(CXXFLAGS) -g -O1 \
`$$OUT/sanpy/bin/python-config --cflags` \
-I../../mercurial -c -o pyutil.o pyutil.cc
bdiff.o: ../../mercurial/bdiff.c bdiff.o: ../../mercurial/bdiff.c
$(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \ $(CC) $(CFLAGS) -fsanitize=fuzzer-no-link,address -c -o bdiff.o \
../../mercurial/bdiff.c ../../mercurial/bdiff.c
bdiff: bdiff.cc bdiff.o fuzzutil.o bdiff: bdiff.cc bdiff.o fuzzutil.o
$(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \ $(CXX) $(CXXFLAGS) -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
-std=c++17 \ -std=c++17 \
-I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff -I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff
-I../../mercurial \ -I../../mercurial \
-c -o pathencode.o ../../mercurial/cext/pathencode.c -c -o pathencode.o ../../mercurial/cext/pathencode.c
revlog.o: sanpy ../../mercurial/cext/revlog.c revlog.o: sanpy ../../mercurial/cext/revlog.c
$(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \ $(CC) $(CFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
-I../../mercurial \ -I../../mercurial \
-c -o revlog.o ../../mercurial/cext/revlog.c -c -o revlog.o ../../mercurial/cext/revlog.c
manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o manifest_fuzzer: sanpy manifest.cc manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o
$(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \ $(CXX) $(CXXFLAGS) `$$OUT/sanpy/bin/python-config --cflags` \
-Wno-register -Wno-macro-redefined \ -Wno-register -Wno-macro-redefined \
-I../../mercurial manifest.cc \ -I../../mercurial manifest.cc \
manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o \ manifest.o charencode.o parsers.o dirs.o pathencode.o revlog.o pyutil.o \
-lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \ -lFuzzingEngine `$$OUT/sanpy/bin/python-config --ldflags` \
-o $$OUT/manifest_fuzzer -o $$OUT/manifest_fuzzer
manifest_corpus.zip: manifest_corpus.zip:
python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip python manifest_corpus.py $$OUT/manifest_fuzzer_seed_corpus.zip
clean: clean:
$(RM) *.o *_fuzzer \ $(RM) *.o *_fuzzer \
bdiff \ bdiff \
mpatch \ mpatch \
xdiff xdiff
oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip oss-fuzz: bdiff_fuzzer mpatch_fuzzer mpatch_corpus.zip xdiff_fuzzer manifest_fuzzer manifest_corpus.zip
.PHONY: all clean oss-fuzz sanpy .PHONY: all clean oss-fuzz sanpy

contrib/fuzz/manifest.cc

#include <Python.h> #include <Python.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include "pyutil.h"
#include <string> #include <string>
extern "C" { extern "C" {
/* TODO: use Python 3 for this fuzzing? */
PyMODINIT_FUNC initparsers(void);
static char cpypath[8192] = "\0";
static PyCodeObject *code; static PyCodeObject *code;
static PyObject *mainmod;
static PyObject *globals;
extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
{ {
const std::string subdir = "/sanpy/lib/python2.7"; contrib::initpy(*argv[0]);
/* HACK ALERT: we need a full Python installation built without
pymalloc and with ASAN, so we dump one in
$OUT/sanpy/lib/python2.7. This helps us wire that up. */
std::string selfpath(*argv[0]);
std::string pypath;
auto pos = selfpath.rfind("/");
if (pos == std::string::npos) {
char wd[8192];
getcwd(wd, 8192);
pypath = std::string(wd) + subdir;
} else {
pypath = selfpath.substr(0, pos) + subdir;
}
strncpy(cpypath, pypath.c_str(), pypath.size());
setenv("PYTHONPATH", cpypath, 1);
setenv("PYTHONNOUSERSITE", "1", 1);
/* prevent Python from looking up users in the fuzz environment */
setenv("PYTHONUSERBASE", cpypath, 1);
Py_SetPythonHome(cpypath);
Py_InitializeEx(0);
initparsers();
code = (PyCodeObject *)Py_CompileString(R"py( code = (PyCodeObject *)Py_CompileString(R"py(
from parsers import lazymanifest from parsers import lazymanifest
try: try:
lm = lazymanifest(mdata) lm = lazymanifest(mdata)
# iterate the whole thing, which causes the code to fully parse # iterate the whole thing, which causes the code to fully parse
# every line in the manifest # every line in the manifest
list(lm.iterentries()) list(lm.iterentries())
lm[b'xyzzy'] = (b'\0' * 20, 'x') lm[b'xyzzy'] = (b'\0' * 20, 'x')
# do an insert, text should change # do an insert, text should change
assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata) assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
del lm[b'xyzzy'] del lm[b'xyzzy']
# should be back to the same # should be back to the same
assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata) assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
except Exception as e: except Exception as e:
pass pass
# uncomment this print if you're editing this Python code # uncomment this print if you're editing this Python code
# to debug failures. # to debug failures.
# print e # print e
)py", )py",
"fuzzer", Py_file_input); "fuzzer", Py_file_input);
mainmod = PyImport_AddModule("__main__");
globals = PyModule_GetDict(mainmod);
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
PyObject *mtext = PyObject *mtext =
PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size); PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
PyObject *locals = PyDict_New(); PyObject *locals = PyDict_New();
PyDict_SetItemString(locals, "mdata", mtext); PyDict_SetItemString(locals, "mdata", mtext);
PyObject *res = PyEval_EvalCode(code, globals, locals); PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
if (!res) { if (!res) {
PyErr_Print(); PyErr_Print();
} }
Py_XDECREF(res); Py_XDECREF(res);
Py_DECREF(locals); Py_DECREF(locals);
Py_DECREF(mtext); Py_DECREF(mtext);
return 0; // Non-zero return values are reserved for future use. return 0; // Non-zero return values are reserved for future use.
} }
} }

contrib/fuzz/pyutil.h

  • This file was added.
#include <Python.h>
namespace contrib
{
void initpy(const char *cselfpath);
PyObject *pyglobals();
} /* namespace contrib */

contrib/fuzz/pyutil.cc

  • This file was added.
#include "pyutil.h"
#include <string>
namespace contrib
{
static char cpypath[8192] = "\0";
static PyObject *mainmod;
static PyObject *globals;
/* TODO: use Python 3 for this fuzzing? */
PyMODINIT_FUNC initparsers(void);
void initpy(const char *cselfpath)
{
const std::string subdir = "/sanpy/lib/python2.7";
/* HACK ALERT: we need a full Python installation built without
pymalloc and with ASAN, so we dump one in
$OUT/sanpy/lib/python2.7. This helps us wire that up. */
std::string selfpath(cselfpath);
std::string pypath;
auto pos = selfpath.rfind("/");
if (pos == std::string::npos) {
char wd[8192];
getcwd(wd, 8192);
pypath = std::string(wd) + subdir;
} else {
pypath = selfpath.substr(0, pos) + subdir;
}
strncpy(cpypath, pypath.c_str(), pypath.size());
setenv("PYTHONPATH", cpypath, 1);
setenv("PYTHONNOUSERSITE", "1", 1);
/* prevent Python from looking up users in the fuzz environment */
setenv("PYTHONUSERBASE", cpypath, 1);
Py_SetPythonHome(cpypath);
Py_InitializeEx(0);
mainmod = PyImport_AddModule("__main__");
globals = PyModule_GetDict(mainmod);
initparsers();
}
PyObject *pyglobals()
{
return globals;
}
} // namespace contrib