This is an archive of the discontinued Mercurial Phabricator instance.

Differential D3675

fuzz: extract some common utilities and use modern C++ idioms
ClosedPublic

Authored by durin42 on May 30 2018, 5:28 PM.

Details

Reviewers
None
Group Reviewers
hg-reviewers
Commits
rHGfa0ddd5e8fff: fuzz: extract some common utilities and use modern C++ idioms
Summary

Alex Gaynor suggested we should probably copy the left and right sides
of diffs to new blocks so we can detect over-reads in the diffing
code, and I agree. Once I got into that, I realized we should do
things with C++17 idioms rather than keep using malloc() and
free(). This change is the result. I tried to split it more than this
and failed.

Everything still compiles and works in the oss-fuzz container, so I
think we can count on C++17 being available!

Diff Detail

Repository
rHG Mercurial
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Revision Contents
Changeset List

PathPackages
Mcontrib/fuzz/Makefile (34 lines)
Mcontrib/fuzz/bdiff.cc (21 lines)
AMcontrib/fuzz/fuzzutil.h (24 lines)
AMcontrib/fuzz/fuzzutil.cc (26 lines)
Mcontrib/fuzz/xdiff.cc (20 lines)

Diff 8936

contrib/fuzz/Makefile

fuzzutil.o: fuzzutil.cc fuzzutil.h
$$CXX $$CXXFLAGS -g -O1 -fsanitize=fuzzer-no-link,address \
-std=c++17 \
-I../../mercurial -c -o fuzzutil.o fuzzutil.cc
bdiff.o: ../../mercurial/bdiff.c bdiff.o: ../../mercurial/bdiff.c
clang -g -O1 -fsanitize=fuzzer-no-link,address -c -o bdiff.o \ $$CC $$CFLAGS -fsanitize=fuzzer-no-link,address -c -o bdiff.o \
../../mercurial/bdiff.c ../../mercurial/bdiff.c
bdiff: bdiff.cc bdiff.o bdiff: bdiff.cc bdiff.o fuzzutil.o
clang -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \ $$CXX $$CXXFLAGS -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
-I../../mercurial bdiff.cc bdiff.o -o bdiff -std=c++17 \
-I../../mercurial bdiff.cc bdiff.o fuzzutil.o -o bdiff
bdiff-oss-fuzz.o: ../../mercurial/bdiff.c bdiff-oss-fuzz.o: ../../mercurial/bdiff.c
$$CC $$CFLAGS -c -o bdiff-oss-fuzz.o ../../mercurial/bdiff.c $$CC $$CFLAGS -c -o bdiff-oss-fuzz.o ../../mercurial/bdiff.c
bdiff_fuzzer: bdiff.cc bdiff-oss-fuzz.o bdiff_fuzzer: bdiff.cc bdiff-oss-fuzz.o fuzzutil.o
$$CXX $$CXXFLAGS -std=c++11 -I../../mercurial bdiff.cc \ $$CXX $$CXXFLAGS -std=c++17 -I../../mercurial bdiff.cc \
bdiff-oss-fuzz.o -lFuzzingEngine -o $$OUT/bdiff_fuzzer bdiff-oss-fuzz.o fuzzutil.o -lFuzzingEngine -o $$OUT/bdiff_fuzzer
x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h
clang -g -O1 -fsanitize=fuzzer-no-link,address -c \ $$CC -g -O1 -fsanitize=fuzzer-no-link,address -c \
-o $@ \ -o $@ \
$< $<
xdiff: xdiff.cc xdiffi.o xprepare.o xutils.o xdiff: xdiff.cc xdiffi.o xprepare.o xutils.o fuzzutil.o
clang -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \ $$CXX $$CXXFLAGS -DHG_FUZZER_INCLUDE_MAIN=1 -g -O1 -fsanitize=fuzzer-no-link,address \
-I../../mercurial xdiff.cc \ -I../../mercurial xdiff.cc \
xdiffi.o xprepare.o xutils.o -o xdiff xdiffi.o xprepare.o xutils.o fuzzutil.o -o xdiff
fuzz-x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h fuzz-x%.o: ../../mercurial/thirdparty/xdiff/x%.c ../../mercurial/thirdparty/xdiff/*.h
$$CC $$CFLAGS -c \ $$CC $$CFLAGS -c \
-o $@ \ -o $@ \
$< $<
xdiff_fuzzer: xdiff.cc fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o xdiff_fuzzer: xdiff.cc fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o fuzzutil.o
$$CXX $$CXXFLAGS -std=c++11 -I../../mercurial xdiff.cc \ $$CXX $$CXXFLAGS -std=c++17 -I../../mercurial xdiff.cc \
fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o \ fuzz-xdiffi.o fuzz-xprepare.o fuzz-xutils.o fuzzutil.o \
-lFuzzingEngine -o $$OUT/xdiff_fuzzer -lFuzzingEngine -o $$OUT/xdiff_fuzzer
all: bdiff xdiff all: bdiff xdiff
oss-fuzz: bdiff_fuzzer xdiff_fuzzer oss-fuzz: bdiff_fuzzer xdiff_fuzzer
.PHONY: all oss-fuzz .PHONY: all oss-fuzz

contrib/fuzz/bdiff.cc

/* /*
* bdiff.cc - fuzzer harness for bdiff.c * bdiff.cc - fuzzer harness for bdiff.c
* *
* Copyright 2018, Google Inc. * Copyright 2018, Google Inc.
* *
* This software may be used and distributed according to the terms of * This software may be used and distributed according to the terms of
* the GNU General Public License, incorporated herein by reference. * the GNU General Public License, incorporated herein by reference.
*/ */
#include <memory>
#include <stdlib.h> #include <stdlib.h>
#include "fuzzutil.h"
extern "C" { extern "C" {
#include "bdiff.h" #include "bdiff.h"
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
if (!Size) { auto maybe_inputs = SplitInputs(Data, Size);
if (!maybe_inputs) {
return 0; return 0;
} }
// figure out a random point in [0, Size] to split our input. auto inputs = std::move(maybe_inputs.value());
size_t split = Data[0] / 255.0 * Size;
// left input to diff is data[1:split]
const uint8_t *left = Data + 1;
// which has len split-1
size_t left_size = split - 1;
// right starts at the next byte after left ends
const uint8_t *right = left + left_size;
size_t right_size = Size - split;
struct bdiff_line *a, *b; struct bdiff_line *a, *b;
int an = bdiff_splitlines((const char *)left, split - 1, &a); int an = bdiff_splitlines(inputs.left.get(), inputs.left_size, &a);
int bn = bdiff_splitlines((const char *)right, right_size, &b); int bn = bdiff_splitlines(inputs.right.get(), inputs.right_size, &b);
struct bdiff_hunk l; struct bdiff_hunk l;
bdiff_diff(a, an, b, bn, &l); bdiff_diff(a, an, b, bn, &l);
free(a); free(a);
free(b); free(b);
bdiff_freehunks(l.next); bdiff_freehunks(l.next);
return 0; // Non-zero return values are reserved for future use. return 0; // Non-zero return values are reserved for future use.
} }

contrib/fuzz/fuzzutil.h

  • This file was added.
#ifndef CONTRIB_FUZZ_FUZZUTIL_H
#define CONTRIB_FUZZ_FUZZUTIL_H
#include <iostream>
#include <memory>
#include <optional>
#include <stdint.h>
/* set DEBUG to 1 for a few debugging prints, or 2 for a lot */
#define DEBUG 0
#define LOG(level) \
if (level <= DEBUG) \
std::cout
struct two_inputs {
std::unique_ptr<char[]> right;
size_t right_size;
std::unique_ptr<char[]> left;
size_t left_size;
};
/* Split a non-zero-length input into two inputs. */
std::optional<two_inputs> SplitInputs(const uint8_t *Data, size_t Size);
#endif /* CONTRIB_FUZZ_FUZZUTIL_H */

contrib/fuzz/fuzzutil.cc

  • This file was added.
#include "fuzzutil.h"
#include <utility>
std::optional<two_inputs> SplitInputs(const uint8_t *Data, size_t Size)
{
if (!Size) {
return std::nullopt;
}
// figure out a random point in [0, Size] to split our input.
size_t left_size = (Data[0] / 255.0) * (Size - 1);
// Copy inputs to new allocations so if bdiff over-reads
// AddressSanitizer can detect it.
std::unique_ptr<char[]> left(new char[left_size]);
memcpy(left.get(), Data + 1, left_size);
// right starts at the next byte after left ends
size_t right_size = Size - (left_size + 1);
std::unique_ptr<char[]> right(new char[right_size]);
memcpy(right.get(), Data + 1 + left_size, right_size);
LOG(2) << "inputs are " << left_size << " and " << right_size
<< " bytes" << std::endl;
two_inputs result = {std::move(right), right_size, std::move(left),
left_size};
return result;
}

contrib/fuzz/xdiff.cc

/* /*
* xdiff.cc - fuzzer harness for thirdparty/xdiff * xdiff.cc - fuzzer harness for thirdparty/xdiff
* *
* Copyright 2018, Google Inc. * Copyright 2018, Google Inc.
* *
* This software may be used and distributed according to the terms of * This software may be used and distributed according to the terms of
* the GNU General Public License, incorporated herein by reference. * the GNU General Public License, incorporated herein by reference.
*/ */
#include "thirdparty/xdiff/xdiff.h" #include "thirdparty/xdiff/xdiff.h"
#include <inttypes.h> #include <inttypes.h>
#include <stdlib.h> #include <stdlib.h>
#include "fuzzutil.h"
extern "C" { extern "C" {
int hunk_consumer(long a1, long a2, long b1, long b2, void *priv) int hunk_consumer(long a1, long a2, long b1, long b2, void *priv)
{ {
// TODO: probably also test returning -1 from this when things break? // TODO: probably also test returning -1 from this when things break?
return 0; return 0;
} }
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
{ {
if (!Size) { auto maybe_inputs = SplitInputs(Data, Size);
if (!maybe_inputs) {
return 0; return 0;
} }
// figure out a random point in [0, Size] to split our input. auto inputs = std::move(maybe_inputs.value());
size_t split = Data[0] / 255.0 * Size;
mmfile_t a, b; mmfile_t a, b;
// `a` input to diff is data[1:split] a.ptr = inputs.left.get();
a.ptr = (char *)Data + 1; a.size = inputs.left_size;
// which has len split-1 b.ptr = inputs.right.get();
a.size = split - 1; b.size = inputs.right_size;
// `b` starts at the next byte after `a` ends
b.ptr = a.ptr + a.size;
b.size = Size - split;
xpparam_t xpp = { xpparam_t xpp = {
XDF_INDENT_HEURISTIC, /* flags */ XDF_INDENT_HEURISTIC, /* flags */
}; };
xdemitconf_t xecfg = { xdemitconf_t xecfg = {
XDL_EMIT_BDIFFHUNK, /* flags */ XDL_EMIT_BDIFFHUNK, /* flags */
hunk_consumer, /* hunk_consume_func */ hunk_consumer, /* hunk_consume_func */
}; };
xdemitcb_t ecb = { xdemitcb_t ecb = {