author | Alex Gaynor <agaynor@mozilla.com> |
Thu, 29 Mar 2018 14:18:36 -0400 | |
changeset 410731 | 900b21685bc45f123eb2daa9e468e0aadee3282d |
parent 410730 | 1caa3b80f1faf1c0fe5cc30ff572df340280b9a4 |
child 410732 | 336801376f35e2410f27d1f4fb1dc9bc90584e9a |
push id | 33735 |
push user | shindli@mozilla.com |
push date | Fri, 30 Mar 2018 09:55:46 +0000 |
treeherder | mozilla-central@3f37287132bf [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | decoder |
bugs | 1450047 |
milestone | 61.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
new file mode 100644 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerClangCounters.cpp @@ -0,0 +1,49 @@ +//===- FuzzerExtraCounters.cpp - Extra coverage counters ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Coverage counters from Clang's SourceBasedCodeCoverage. +//===----------------------------------------------------------------------===// + +// Support for SourceBasedCodeCoverage is experimental: +// * Works only for the main binary, not DSOs yet. +// * Works only on Linux. +// * Does not implement print_pcs/print_coverage yet. +// * Is not fully evaluated for performance and sensitivity. +// We expect large performance drop due to 64-bit counters, +// and *maybe* better sensitivity due to more fine-grained counters. +// Preliminary comparison on a single benchmark (RE2) shows +// a bit worse sensitivity though. + +#include "FuzzerDefs.h" + +#if LIBFUZZER_LINUX +__attribute__((weak)) extern uint64_t __start___llvm_prf_cnts; +__attribute__((weak)) extern uint64_t __stop___llvm_prf_cnts; +namespace fuzzer { +uint64_t *ClangCountersBegin() { return &__start___llvm_prf_cnts; } +uint64_t *ClangCountersEnd() { return &__stop___llvm_prf_cnts; } +} // namespace fuzzer +#else +// TODO: Implement on Mac (if the data shows it's worth it). +//__attribute__((visibility("hidden"))) +//extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts"); +//__attribute__((visibility("hidden"))) +//extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts"); +namespace fuzzer { +uint64_t *ClangCountersBegin() { return nullptr; } +uint64_t *ClangCountersEnd() { return nullptr; } +} // namespace fuzzer +#endif + +namespace fuzzer { +ATTRIBUTE_NO_SANITIZE_ALL +void ClearClangCounters() { // hand-written memset, don't asan-ify. + for (auto P = ClangCountersBegin(); P < ClangCountersEnd(); P++) + *P = 0; +} +}
new file mode 100644 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCommand.h @@ -0,0 +1,180 @@ +//===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// FuzzerCommand represents a command to run in a subprocess. It allows callers +// to manage command line arguments and output and error streams. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_COMMAND_H +#define LLVM_FUZZER_COMMAND_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include <algorithm> +#include <sstream> +#include <string> +#include <vector> + +namespace fuzzer { + +class Command final { +public: + // This command line flag is used to indicate that the remaining command line + // is immutable, meaning this flag effectively marks the end of the mutable + // argument list. + static inline const char *ignoreRemainingArgs() { + static const char *kIgnoreRemaining = "-ignore_remaining_args=1"; + return kIgnoreRemaining; + } + + Command() : CombinedOutAndErr(false) {} + + explicit Command(const Vector<std::string> &ArgsToAdd) + : Args(ArgsToAdd), CombinedOutAndErr(false) {} + + explicit Command(const Command &Other) + : Args(Other.Args), CombinedOutAndErr(Other.CombinedOutAndErr), + OutputFile(Other.OutputFile) {} + + Command &operator=(const Command &Other) { + Args = Other.Args; + CombinedOutAndErr = Other.CombinedOutAndErr; + OutputFile = Other.OutputFile; + return *this; + } + + ~Command() {} + + // Returns true if the given Arg is present in Args. Only checks up to + // "-ignore_remaining_args=1". + bool hasArgument(const std::string &Arg) const { + auto i = endMutableArgs(); + return std::find(Args.begin(), i, Arg) != i; + } + + // Gets all of the current command line arguments, **including** those after + // "-ignore-remaining-args=1". + const Vector<std::string> &getArguments() const { return Args; } + + // Adds the given argument before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArgument(const std::string &Arg) { + Args.insert(endMutableArgs(), Arg); + } + + // Adds all given arguments before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArguments(const Vector<std::string> &ArgsToAdd) { + Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); + } + + // Removes the given argument from the command argument list. Ignores any + // occurrences after "-ignore_remaining_args=1", if present. + void removeArgument(const std::string &Arg) { + auto i = endMutableArgs(); + Args.erase(std::remove(Args.begin(), i, Arg), i); + } + + // Like hasArgument, but checks for "-[Flag]=...". + bool hasFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + return std::any_of(Args.begin(), endMutableArgs(), IsMatch); + } + + // Returns the value of the first instance of a given flag, or an empty string + // if the flag isn't present. Ignores any occurrences after + // "-ignore_remaining_args=1", if present. + std::string getFlagValue(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + auto j = std::find_if(Args.begin(), i, IsMatch); + std::string result; + if (j != i) { + result = j->substr(Arg.length()); + } + return result; + } + + // Like AddArgument, but adds "-[Flag]=[Value]". + void addFlag(const std::string &Flag, const std::string &Value) { + addArgument("-" + Flag + "=" + Value); + } + + // Like RemoveArgument, but removes "-[Flag]=...". + void removeFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + Args.erase(std::remove_if(Args.begin(), i, IsMatch), i); + } + + // Returns whether the command's stdout is being written to an output file. + bool hasOutputFile() const { return !OutputFile.empty(); } + + // Returns the currently set output file. + const std::string &getOutputFile() const { return OutputFile; } + + // Configures the command to redirect its output to the name file. + void setOutputFile(const std::string &FileName) { OutputFile = FileName; } + + // Returns whether the command's stderr is redirected to stdout. + bool isOutAndErrCombined() const { return CombinedOutAndErr; } + + // Sets whether to redirect the command's stderr to its stdout. + void combineOutAndErr(bool combine = true) { CombinedOutAndErr = combine; } + + // Returns a string representation of the command. On many systems this will + // be the equivalent command line. + std::string toString() const { + std::stringstream SS; + for (auto arg : getArguments()) + SS << arg << " "; + if (hasOutputFile()) + SS << ">" << getOutputFile() << " "; + if (isOutAndErrCombined()) + SS << "2>&1 "; + std::string result = SS.str(); + if (!result.empty()) + result = result.substr(0, result.length() - 1); + return result; + } + +private: + Command(Command &&Other) = delete; + Command &operator=(Command &&Other) = delete; + + Vector<std::string>::iterator endMutableArgs() { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + Vector<std::string>::const_iterator endMutableArgs() const { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + // The command arguments. Args[0] is the command name. + Vector<std::string> Args; + + // True indicates stderr is redirected to stdout. + bool CombinedOutAndErr; + + // If not empty, stdout is redirected to the named file. + std::string OutputFile; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_COMMAND_H
--- a/tools/fuzzing/libfuzzer/FuzzerCorpus.h +++ b/tools/fuzzing/libfuzzer/FuzzerCorpus.h @@ -29,24 +29,28 @@ struct InputInfo { uint8_t Sha1[kSHA1NumBytes]; // Checksum. // Number of features that this input has and no smaller input has. size_t NumFeatures = 0; size_t Tmp = 0; // Used by ValidateFeatureSet. // Stats. size_t NumExecutedMutations = 0; size_t NumSuccessfullMutations = 0; bool MayDeleteFile = false; + bool Reduced = false; + Vector<uint32_t> UniqFeatureSet; + float FeatureFrequencyScore = 1.0; }; class InputCorpus { static const size_t kFeatureSetSize = 1 << 21; public: InputCorpus(const std::string &OutputCorpus) : OutputCorpus(OutputCorpus) { memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); + memset(FeatureFrequency, 0, sizeof(FeatureFrequency)); } ~InputCorpus() { for (auto II : Inputs) delete II; } size_t size() const { return Inputs.size(); } size_t SizeInBytes() const { size_t Res = 0; @@ -63,45 +67,92 @@ class InputCorpus { size_t MaxInputSize() const { size_t Res = 0; for (auto II : Inputs) Res = std::max(Res, II->U.size()); return Res; } bool empty() const { return Inputs.empty(); } const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; } - void AddToCorpus(const Unit &U, size_t NumFeatures, - bool MayDeleteFile = false) { + void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile, + const Vector<uint32_t> &FeatureSet) { assert(!U.empty()); - uint8_t Hash[kSHA1NumBytes]; if (FeatureDebug) Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures); - ComputeSHA1(U.data(), U.size(), Hash); - Hashes.insert(Sha1ToString(Hash)); Inputs.push_back(new InputInfo()); InputInfo &II = *Inputs.back(); II.U = U; II.NumFeatures = NumFeatures; II.MayDeleteFile = MayDeleteFile; - memcpy(II.Sha1, Hash, kSHA1NumBytes); + II.UniqFeatureSet = FeatureSet; + std::sort(II.UniqFeatureSet.begin(), II.UniqFeatureSet.end()); + ComputeSHA1(U.data(), U.size(), II.Sha1); + Hashes.insert(Sha1ToString(II.Sha1)); UpdateCorpusDistribution(); + PrintCorpus(); // ValidateFeatureSet(); } + // Debug-only + void PrintUnit(const Unit &U) { + if (!FeatureDebug) return; + for (uint8_t C : U) { + if (C != 'F' && C != 'U' && C != 'Z') + C = '.'; + Printf("%c", C); + } + } + + // Debug-only + void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) { + if (!FeatureDebug) return; + Printf("{"); + for (uint32_t Feature: FeatureSet) + Printf("%u,", Feature); + Printf("}"); + } + + // Debug-only + void PrintCorpus() { + if (!FeatureDebug) return; + Printf("======= CORPUS:\n"); + int i = 0; + for (auto II : Inputs) { + if (std::find(II->U.begin(), II->U.end(), 'F') != II->U.end()) { + Printf("[%2d] ", i); + Printf("%s sz=%zd ", Sha1ToString(II->Sha1).c_str(), II->U.size()); + PrintUnit(II->U); + Printf(" "); + PrintFeatureSet(II->UniqFeatureSet); + Printf("\n"); + } + i++; + } + } + + void Replace(InputInfo *II, const Unit &U) { + assert(II->U.size() > U.size()); + Hashes.erase(Sha1ToString(II->Sha1)); + DeleteFile(*II); + ComputeSHA1(U.data(), U.size(), II->Sha1); + Hashes.insert(Sha1ToString(II->Sha1)); + II->U = U; + II->Reduced = true; + UpdateCorpusDistribution(); + } + bool HasUnit(const Unit &U) { return Hashes.count(Hash(U)); } bool HasUnit(const std::string &H) { return Hashes.count(H); } InputInfo &ChooseUnitToMutate(Random &Rand) { InputInfo &II = *Inputs[ChooseUnitIdxToMutate(Rand)]; assert(!II.U.empty()); return II; }; // Returns an index of random unit from the corpus to mutate. - // Hypothesis: units added to the corpus last are more likely to be - // interesting. This function gives more weight to the more recent units. size_t ChooseUnitIdxToMutate(Random &Rand) { size_t Idx = static_cast<size_t>(CorpusDistribution(Rand)); assert(Idx < Inputs.size()); return Idx; } void PrintStats() { for (size_t i = 0; i < Inputs.size(); i++) { @@ -119,26 +170,30 @@ class InputCorpus { } Printf("\n\t"); for (size_t i = 0; i < Inputs.size(); i++) if (size_t N = Inputs[i]->NumFeatures) Printf(" %zd=>%zd ", i, N); Printf("\n"); } + void DeleteFile(const InputInfo &II) { + if (!OutputCorpus.empty() && II.MayDeleteFile) + RemoveFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1))); + } + void DeleteInput(size_t Idx) { InputInfo &II = *Inputs[Idx]; - if (!OutputCorpus.empty() && II.MayDeleteFile) - RemoveFile(DirPlusFile(OutputCorpus, Sha1ToString(II.Sha1))); + DeleteFile(II); Unit().swap(II.U); if (FeatureDebug) Printf("EVICTED %zd\n", Idx); } - void AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { + bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink) { assert(NewSize); Idx = Idx % kFeatureSetSize; uint32_t OldSize = GetFeature(Idx); if (OldSize == 0 || (Shrink && OldSize > NewSize)) { if (OldSize > 0) { size_t OldIdx = SmallestElementPerFeature[Idx]; InputInfo &II = *Inputs[OldIdx]; assert(II.NumFeatures > 0); @@ -148,77 +203,100 @@ class InputCorpus { } else { NumAddedFeatures++; } NumUpdatedFeatures++; if (FeatureDebug) Printf("ADD FEATURE %zd sz %d\n", Idx, NewSize); SmallestElementPerFeature[Idx] = Inputs.size(); InputSizesPerFeature[Idx] = NewSize; - CountingFeatures = true; + return true; } + return false; + } + + void UpdateFeatureFrequency(size_t Idx) { + FeatureFrequency[Idx % kFeatureSetSize]++; + } + float GetFeatureFrequency(size_t Idx) const { + return FeatureFrequency[Idx % kFeatureSetSize]; + } + void UpdateFeatureFrequencyScore(InputInfo *II) { + const float kMin = 0.01, kMax = 100.; + II->FeatureFrequencyScore = kMin; + for (auto Idx : II->UniqFeatureSet) + II->FeatureFrequencyScore += 1. / (GetFeatureFrequency(Idx) + 1.); + II->FeatureFrequencyScore = Min(II->FeatureFrequencyScore, kMax); } size_t NumFeatures() const { return NumAddedFeatures; } size_t NumFeatureUpdates() const { return NumUpdatedFeatures; } - void ResetFeatureSet() { - assert(Inputs.empty()); - memset(InputSizesPerFeature, 0, sizeof(InputSizesPerFeature)); - memset(SmallestElementPerFeature, 0, sizeof(SmallestElementPerFeature)); - } - private: static const bool FeatureDebug = false; size_t GetFeature(size_t Idx) const { return InputSizesPerFeature[Idx]; } void ValidateFeatureSet() { - if (!CountingFeatures) return; if (FeatureDebug) PrintFeatureSet(); for (size_t Idx = 0; Idx < kFeatureSetSize; Idx++) if (GetFeature(Idx)) Inputs[SmallestElementPerFeature[Idx]]->Tmp++; for (auto II: Inputs) { if (II->Tmp != II->NumFeatures) Printf("ZZZ %zd %zd\n", II->Tmp, II->NumFeatures); assert(II->Tmp == II->NumFeatures); II->Tmp = 0; } } // Updates the probability distribution for the units in the corpus. // Must be called whenever the corpus or unit weights are changed. + // + // Hypothesis: units added to the corpus last are more interesting. + // + // Hypothesis: inputs with infrequent features are more interesting. void UpdateCorpusDistribution() { size_t N = Inputs.size(); + assert(N); Intervals.resize(N + 1); Weights.resize(N); std::iota(Intervals.begin(), Intervals.end(), 0); - if (CountingFeatures) + for (size_t i = 0; i < N; i++) + Weights[i] = Inputs[i]->NumFeatures + ? (i + 1) * Inputs[i]->FeatureFrequencyScore + : 0.; + if (FeatureDebug) { for (size_t i = 0; i < N; i++) - Weights[i] = Inputs[i]->NumFeatures * (i + 1); - else - std::iota(Weights.begin(), Weights.end(), 1); + Printf("%zd ", Inputs[i]->NumFeatures); + Printf("NUM\n"); + for (size_t i = 0; i < N; i++) + Printf("%f ", Inputs[i]->FeatureFrequencyScore); + Printf("SCORE\n"); + for (size_t i = 0; i < N; i++) + Printf("%f ", Weights[i]); + Printf("Weights\n"); + } CorpusDistribution = std::piecewise_constant_distribution<double>( Intervals.begin(), Intervals.end(), Weights.begin()); } std::piecewise_constant_distribution<double> CorpusDistribution; - std::vector<double> Intervals; - std::vector<double> Weights; + Vector<double> Intervals; + Vector<double> Weights; std::unordered_set<std::string> Hashes; - std::vector<InputInfo*> Inputs; + Vector<InputInfo*> Inputs; - bool CountingFeatures = false; size_t NumAddedFeatures = 0; size_t NumUpdatedFeatures = 0; uint32_t InputSizesPerFeature[kFeatureSetSize]; uint32_t SmallestElementPerFeature[kFeatureSetSize]; + float FeatureFrequency[kFeatureSetSize]; std::string OutputCorpus; }; } // namespace fuzzer #endif // LLVM_FUZZER_CORPUS
--- a/tools/fuzzing/libfuzzer/FuzzerDefs.h +++ b/tools/fuzzing/libfuzzer/FuzzerDefs.h @@ -13,39 +13,71 @@ #define LLVM_FUZZER_DEFS_H #include <cassert> #include <cstddef> #include <cstdint> #include <cstring> #include <string> #include <vector> +#include <set> +#include <memory> // Platform detection. #ifdef __linux__ #define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 #define LIBFUZZER_LINUX 1 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 #define LIBFUZZER_WINDOWS 0 #elif __APPLE__ #define LIBFUZZER_APPLE 1 +#define LIBFUZZER_FUCHSIA 0 #define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_WINDOWS 0 +#elif __NetBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 1 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_WINDOWS 0 +#elif __FreeBSD__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 1 #define LIBFUZZER_WINDOWS 0 #elif _WIN32 #define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 0 #define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 #define LIBFUZZER_WINDOWS 1 +#elif __Fuchsia__ +#define LIBFUZZER_APPLE 0 +#define LIBFUZZER_FUCHSIA 1 +#define LIBFUZZER_LINUX 0 +#define LIBFUZZER_NETBSD 0 +#define LIBFUZZER_FREEBSD 0 +#define LIBFUZZER_WINDOWS 0 #else #error "Support for your platform has not been implemented" #endif #ifndef __has_attribute # define __has_attribute(x) 0 #endif -#define LIBFUZZER_POSIX LIBFUZZER_APPLE || LIBFUZZER_LINUX +#define LIBFUZZER_POSIX (LIBFUZZER_APPLE || LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD) #ifdef __x86_64 # if __has_attribute(target) # define ATTRIBUTE_TARGET_POPCNT __attribute__((target("popcnt"))) # else # define ATTRIBUTE_TARGET_POPCNT # endif #else @@ -97,18 +129,33 @@ class MutationDispatcher; struct FuzzingOptions; class InputCorpus; struct InputInfo; struct ExternalFunctions; // Global interface to functions that may or may not be available. extern ExternalFunctions *EF; -typedef std::vector<uint8_t> Unit; -typedef std::vector<Unit> UnitVector; +// We are using a custom allocator to give a different symbol name to STL +// containers in order to avoid ODR violations. +template<typename T> + class fuzzer_allocator: public std::allocator<T> { + public: + template<class Other> + struct rebind { typedef fuzzer_allocator<Other> other; }; + }; + +template<typename T> +using Vector = std::vector<T, fuzzer_allocator<T>>; + +template<typename T> +using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>; + +typedef Vector<uint8_t> Unit; +typedef Vector<Unit> UnitVector; typedef int (*UserCallback)(const uint8_t *Data, size_t Size); int FuzzerDriver(int *argc, char ***argv, UserCallback Callback); struct ScopedDoingMyOwnMemOrStr { ScopedDoingMyOwnMemOrStr() { DoingMyOwnMemOrStr++; } ~ScopedDoingMyOwnMemOrStr() { DoingMyOwnMemOrStr--; } static int DoingMyOwnMemOrStr; @@ -118,11 +165,15 @@ inline uint8_t Bswap(uint8_t x) { retu inline uint16_t Bswap(uint16_t x) { return __builtin_bswap16(x); } inline uint32_t Bswap(uint32_t x) { return __builtin_bswap32(x); } inline uint64_t Bswap(uint64_t x) { return __builtin_bswap64(x); } uint8_t *ExtraCountersBegin(); uint8_t *ExtraCountersEnd(); void ClearExtraCounters(); +uint64_t *ClangCountersBegin(); +uint64_t *ClangCountersEnd(); +void ClearClangCounters(); + } // namespace fuzzer #endif // LLVM_FUZZER_DEFS_H
--- a/tools/fuzzing/libfuzzer/FuzzerDictionary.h +++ b/tools/fuzzing/libfuzzer/FuzzerDictionary.h @@ -110,18 +110,18 @@ class Dictionary { size_t size() const { return Size; } private: DictionaryEntry DE[kMaxDictSize]; size_t Size = 0; }; // Parses one dictionary entry. -// If successfull, write the enty to Unit and returns true, +// If successful, write the enty to Unit and returns true, // otherwise returns false. bool ParseOneDictionaryEntry(const std::string &Str, Unit *U); // Parses the dictionary file, fills Units, returns true iff all lines -// were parsed succesfully. -bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units); +// were parsed successfully. +bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units); } // namespace fuzzer #endif // LLVM_FUZZER_DICTIONARY_H
--- a/tools/fuzzing/libfuzzer/FuzzerDriver.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerDriver.cpp @@ -4,27 +4,29 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // FuzzerDriver and flag parsing. //===----------------------------------------------------------------------===// +#include "FuzzerCommand.h" #include "FuzzerCorpus.h" +#include "FuzzerIO.h" #include "FuzzerInterface.h" #include "FuzzerInternal.h" -#include "FuzzerIO.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" #include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <atomic> #include <chrono> +#include <cstdlib> #include <cstring> #include <mutex> #include <string> #include <thread> // This function should be present in the libFuzzer so that the client // binary can test for its existence. extern "C" __attribute__((used)) void __libfuzzer_is_present() {} @@ -68,17 +70,17 @@ static const FlagDescription FlagDescrip #undef FUZZER_FLAG_INT #undef FUZZER_FLAG_UNSIGNED #undef FUZZER_FLAG_STRING }; static const size_t kNumFlags = sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); -static std::vector<std::string> *Inputs; +static Vector<std::string> *Inputs; static std::string *ProgName; static void PrintHelp() { Printf("Usage:\n"); auto Prog = ProgName->c_str(); Printf("\nTo run fuzzing pass 0 or more directories.\n"); Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog); @@ -144,17 +146,17 @@ static bool ParseOneFlag(const char *Par for (size_t F = 0; F < kNumFlags; F++) { const char *Name = FlagDescriptions[F].Name; const char *Str = FlagValue(Param, Name); if (Str) { if (FlagDescriptions[F].IntFlag) { int Val = MyStol(Str); *FlagDescriptions[F].IntFlag = Val; if (Flags.verbosity >= 2) - Printf("Flag: %s %d\n", Name, Val);; + Printf("Flag: %s %d\n", Name, Val); return true; } else if (FlagDescriptions[F].UIntFlag) { unsigned int Val = std::stoul(Str); *FlagDescriptions[F].UIntFlag = Val; if (Flags.verbosity >= 2) Printf("Flag: %s %u\n", Name, Val); return true; } else if (FlagDescriptions[F].StrFlag) { @@ -169,83 +171,93 @@ static bool ParseOneFlag(const char *Par } } Printf("\n\nWARNING: unrecognized flag '%s'; " "use -help=1 to list all flags\n\n", Param); return true; } // We don't use any library to minimize dependencies. -static void ParseFlags(const std::vector<std::string> &Args) { +static void ParseFlags(const Vector<std::string> &Args) { for (size_t F = 0; F < kNumFlags; F++) { if (FlagDescriptions[F].IntFlag) *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; if (FlagDescriptions[F].UIntFlag) *FlagDescriptions[F].UIntFlag = static_cast<unsigned int>(FlagDescriptions[F].Default); if (FlagDescriptions[F].StrFlag) *FlagDescriptions[F].StrFlag = nullptr; } - Inputs = new std::vector<std::string>; + Inputs = new Vector<std::string>; for (size_t A = 1; A < Args.size(); A++) { - if (ParseOneFlag(Args[A].c_str())) continue; + if (ParseOneFlag(Args[A].c_str())) { + if (Flags.ignore_remaining_args) + break; + continue; + } Inputs->push_back(Args[A]); } } static std::mutex Mu; static void PulseThread() { while (true) { SleepSeconds(600); std::lock_guard<std::mutex> Lock(Mu); Printf("pulse...\n"); } } -static void WorkerThread(const std::string &Cmd, std::atomic<unsigned> *Counter, +static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter, unsigned NumJobs, std::atomic<bool> *HasErrors) { while (true) { unsigned C = (*Counter)++; if (C >= NumJobs) break; std::string Log = "fuzz-" + std::to_string(C) + ".log"; - std::string ToRun = Cmd + " > " + Log + " 2>&1\n"; - if (Flags.verbosity) - Printf("%s", ToRun.c_str()); - int ExitCode = ExecuteCommand(ToRun); + Command Cmd(BaseCmd); + Cmd.setOutputFile(Log); + Cmd.combineOutAndErr(); + if (Flags.verbosity) { + std::string CommandLine = Cmd.toString(); + Printf("%s\n", CommandLine.c_str()); + } + int ExitCode = ExecuteCommand(Cmd); if (ExitCode != 0) *HasErrors = true; std::lock_guard<std::mutex> Lock(Mu); Printf("================== Job %u exited with exit code %d ============\n", C, ExitCode); fuzzer::CopyFileToErr(Log); } } -std::string CloneArgsWithoutX(const std::vector<std::string> &Args, +std::string CloneArgsWithoutX(const Vector<std::string> &Args, const char *X1, const char *X2) { std::string Cmd; for (auto &S : Args) { if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) continue; Cmd += S + " "; } return Cmd; } -static int RunInMultipleProcesses(const std::vector<std::string> &Args, +static int RunInMultipleProcesses(const Vector<std::string> &Args, unsigned NumWorkers, unsigned NumJobs) { std::atomic<unsigned> Counter(0); std::atomic<bool> HasErrors(false); - std::string Cmd = CloneArgsWithoutX(Args, "jobs", "workers"); - std::vector<std::thread> V; + Command Cmd(Args); + Cmd.removeFlag("jobs"); + Cmd.removeFlag("workers"); + Vector<std::thread> V; std::thread Pulse(PulseThread); Pulse.detach(); for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, Cmd, &Counter, NumJobs, &HasErrors)); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); for (auto &T : V) T.join(); return HasErrors ? 1 : 0; } static void RssThread(Fuzzer *F, size_t RssLimitMb) { while (true) { SleepSeconds(1); @@ -260,17 +272,17 @@ static void StartRssThread(Fuzzer *F, si std::thread T(RssThread, F, RssLimitMb); T.detach(); } int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { Unit U = FileToVector(InputFilePath); if (MaxLen && MaxLen < U.size()) U.resize(MaxLen); - F->RunOne(U.data(), U.size()); + F->ExecuteCallback(U.data(), U.size()); F->TryDetectingAMemoryLeak(U.data(), U.size(), true); return 0; } static bool AllInputsAreFiles() { if (Inputs->empty()) return false; for (auto &Path : *Inputs) if (!IsFile(Path)) @@ -284,45 +296,44 @@ static std::string GetDedupTokenFromFile if (Beg == std::string::npos) return ""; auto End = S.find('\n', Beg); if (End == std::string::npos) return ""; return S.substr(Beg, End - Beg); } -int CleanseCrashInput(const std::vector<std::string> &Args, +int CleanseCrashInput(const Vector<std::string> &Args, const FuzzingOptions &Options) { if (Inputs->size() != 1 || !Flags.exact_artifact_path) { Printf("ERROR: -cleanse_crash should be given one input file and" " -exact_artifact_path\n"); exit(1); } std::string InputFilePath = Inputs->at(0); std::string OutputFilePath = Flags.exact_artifact_path; - std::string BaseCmd = - CloneArgsWithoutX(Args, "cleanse_crash", "cleanse_crash"); + Command Cmd(Args); + Cmd.removeFlag("cleanse_crash"); - auto InputPos = BaseCmd.find(" " + InputFilePath + " "); - assert(InputPos != std::string::npos); - BaseCmd.erase(InputPos, InputFilePath.size() + 1); + assert(Cmd.hasArgument(InputFilePath)); + Cmd.removeArgument(InputFilePath); auto LogFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); auto TmpFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".repro"); - auto LogFileRedirect = " > " + LogFilePath + " 2>&1 "; - - auto Cmd = BaseCmd + " " + TmpFilePath + LogFileRedirect; + Cmd.addArgument(TmpFilePath); + Cmd.setOutputFile(LogFilePath); + Cmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; auto U = FileToVector(CurrentFilePath); size_t Size = U.size(); - const std::vector<uint8_t> ReplacementBytes = {' ', 0xff}; + const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { bool Changed = false; for (size_t Idx = 0; Idx < Size; Idx++) { Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts, Idx, Size); uint8_t OriginalByte = U[Idx]; if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(), ReplacementBytes.end(), @@ -344,67 +355,71 @@ int CleanseCrashInput(const std::vector< } } if (!Changed) break; } RemoveFile(LogFilePath); return 0; } -int MinimizeCrashInput(const std::vector<std::string> &Args, +int MinimizeCrashInput(const Vector<std::string> &Args, const FuzzingOptions &Options) { if (Inputs->size() != 1) { Printf("ERROR: -minimize_crash should be given one input file\n"); exit(1); } std::string InputFilePath = Inputs->at(0); - std::string BaseCmd = - CloneArgsWithoutX(Args, "minimize_crash", "exact_artifact_path"); - auto InputPos = BaseCmd.find(" " + InputFilePath + " "); - assert(InputPos != std::string::npos); - BaseCmd.erase(InputPos, InputFilePath.size() + 1); + Command BaseCmd(Args); + BaseCmd.removeFlag("minimize_crash"); + BaseCmd.removeFlag("exact_artifact_path"); + assert(BaseCmd.hasArgument(InputFilePath)); + BaseCmd.removeArgument(InputFilePath); if (Flags.runs <= 0 && Flags.max_total_time == 0) { Printf("INFO: you need to specify -runs=N or " "-max_total_time=N with -minimize_crash=1\n" "INFO: defaulting to -max_total_time=600\n"); - BaseCmd += " -max_total_time=600"; + BaseCmd.addFlag("max_total_time", "600"); } auto LogFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - auto LogFileRedirect = " > " + LogFilePath + " 2>&1 "; + BaseCmd.setOutputFile(LogFilePath); + BaseCmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; while (true) { Unit U = FileToVector(CurrentFilePath); Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", CurrentFilePath.c_str(), U.size()); - auto Cmd = BaseCmd + " " + CurrentFilePath + LogFileRedirect; + Command Cmd(BaseCmd); + Cmd.addArgument(CurrentFilePath); - Printf("CRASH_MIN: executing: %s\n", Cmd.c_str()); + std::string CommandLine = Cmd.toString(); + Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); int ExitCode = ExecuteCommand(Cmd); if (ExitCode == 0) { Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); exit(1); } Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " "it further\n", CurrentFilePath.c_str(), U.size()); auto DedupToken1 = GetDedupTokenFromFile(LogFilePath); if (!DedupToken1.empty()) Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); std::string ArtifactPath = Flags.exact_artifact_path ? Flags.exact_artifact_path : Options.ArtifactPrefix + "minimized-from-" + Hash(U); - Cmd += " -minimize_crash_internal_step=1 -exact_artifact_path=" + - ArtifactPath; - Printf("CRASH_MIN: executing: %s\n", Cmd.c_str()); + Cmd.addFlag("minimize_crash_internal_step", "1"); + Cmd.addFlag("exact_artifact_path", ArtifactPath); + CommandLine = Cmd.toString(); + Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); ExitCode = ExecuteCommand(Cmd); CopyFileToErr(LogFilePath); if (ExitCode == 0) { if (Flags.exact_artifact_path) { CurrentFilePath = Flags.exact_artifact_path; WriteToFile(U, CurrentFilePath); } Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", @@ -436,47 +451,45 @@ int MinimizeCrashInputInternalStep(Fuzze assert(Inputs->size() == 1); std::string InputFilePath = Inputs->at(0); Unit U = FileToVector(InputFilePath); Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); if (U.size() < 2) { Printf("INFO: The input is small enough, exiting\n"); exit(0); } - Corpus->AddToCorpus(U, 0); F->SetMaxInputLen(U.size()); F->SetMaxMutationLen(U.size() - 1); F->MinimizeCrashLoop(U); Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); exit(0); return 0; } -int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit>& Dict, +int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, UnitVector& Corpus) { Printf("Started dictionary minimization (up to %d tests)\n", Dict.size() * Corpus.size() * 2); // Scores and usage count for each dictionary unit. - std::vector<int> Scores(Dict.size()); - std::vector<int> Usages(Dict.size()); + Vector<int> Scores(Dict.size()); + Vector<int> Usages(Dict.size()); - std::vector<size_t> InitialFeatures; - std::vector<size_t> ModifiedFeatures; + Vector<size_t> InitialFeatures; + Vector<size_t> ModifiedFeatures; for (auto &C : Corpus) { // Get coverage for the testcase without modifications. F->ExecuteCallback(C.data(), C.size()); InitialFeatures.clear(); - TPC.CollectFeatures([&](size_t Feature) -> bool { + TPC.CollectFeatures([&](size_t Feature) { InitialFeatures.push_back(Feature); - return true; }); for (size_t i = 0; i < Dict.size(); ++i) { - auto Data = C; + Vector<uint8_t> Data = C; auto StartPos = std::search(Data.begin(), Data.end(), Dict[i].begin(), Dict[i].end()); // Skip dictionary unit, if the testcase does not contain it. if (StartPos == Data.end()) continue; ++Usages[i]; while (StartPos != Data.end()) { @@ -487,19 +500,18 @@ int AnalyzeDictionary(Fuzzer *F, const s StartPos = std::search(EndPos, Data.end(), Dict[i].begin(), Dict[i].end()); } // Get coverage for testcase with masked occurrences of dictionary unit. F->ExecuteCallback(Data.data(), Data.size()); ModifiedFeatures.clear(); - TPC.CollectFeatures([&](size_t Feature) -> bool { + TPC.CollectFeatures([&](size_t Feature) { ModifiedFeatures.push_back(Feature); - return true; }); if (InitialFeatures == ModifiedFeatures) --Scores[i]; else Scores[i] += 2; } } @@ -520,17 +532,17 @@ int AnalyzeDictionary(Fuzzer *F, const s int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { using namespace fuzzer; assert(argc && argv && "Argument pointers cannot be nullptr"); std::string Argv0((*argv)[0]); EF = new ExternalFunctions(); if (EF->LLVMFuzzerInitialize) EF->LLVMFuzzerInitialize(argc, argv); - const std::vector<std::string> Args(*argv, *argv + *argc); + const Vector<std::string> Args(*argv, *argv + *argc); assert(!Args.empty()); ProgName = new std::string(Args[0]); if (Argv0 != *ProgName) { Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); exit(1); } ParseFlags(Args); if (Flags.help) { @@ -547,66 +559,70 @@ int FuzzerDriver(int *argc, char ***argv Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); if (Flags.workers > 1) Printf("Running %u workers\n", Flags.workers); } if (Flags.workers > 0 && Flags.jobs > 0) return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs); - const size_t kMaxSaneLen = 1 << 20; - const size_t kMinDefaultLen = 64; FuzzingOptions Options; Options.Verbosity = Flags.verbosity; Options.MaxLen = Flags.max_len; - Options.ExperimentalLenControl = Flags.experimental_len_control; - if (Flags.experimental_len_control && Flags.max_len == 64) - Options.MaxLen = 1 << 20; + Options.LenControl = Flags.len_control; Options.UnitTimeoutSec = Flags.timeout; Options.ErrorExitCode = Flags.error_exitcode; Options.TimeoutExitCode = Flags.timeout_exitcode; Options.MaxTotalTimeSec = Flags.max_total_time; Options.DoCrossOver = Flags.cross_over; Options.MutateDepth = Flags.mutate_depth; + Options.ReduceDepth = Flags.reduce_depth; Options.UseCounters = Flags.use_counters; - Options.UseIndirCalls = Flags.use_indir_calls; Options.UseMemmem = Flags.use_memmem; Options.UseCmp = Flags.use_cmp; Options.UseValueProfile = Flags.use_value_profile; Options.Shrink = Flags.shrink; + Options.ReduceInputs = Flags.reduce_inputs; Options.ShuffleAtStartUp = Flags.shuffle; Options.PreferSmall = Flags.prefer_small; Options.ReloadIntervalSec = Flags.reload; Options.OnlyASCII = Flags.only_ascii; Options.DetectLeaks = Flags.detect_leaks; + Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval; Options.TraceMalloc = Flags.trace_malloc; Options.RssLimitMb = Flags.rss_limit_mb; + Options.MallocLimitMb = Flags.malloc_limit_mb; + if (!Options.MallocLimitMb) + Options.MallocLimitMb = Options.RssLimitMb; if (Flags.runs >= 0) Options.MaxNumberOfRuns = Flags.runs; if (!Inputs->empty() && !Flags.minimize_crash_internal_step) Options.OutputCorpus = (*Inputs)[0]; Options.ReportSlowUnits = Flags.report_slow_units; if (Flags.artifact_prefix) Options.ArtifactPrefix = Flags.artifact_prefix; if (Flags.exact_artifact_path) Options.ExactArtifactPath = Flags.exact_artifact_path; - std::vector<Unit> Dictionary; + Vector<Unit> Dictionary; if (Flags.dict) if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) return 1; if (Flags.verbosity > 0 && !Dictionary.empty()) Printf("Dictionary: %zd entries\n", Dictionary.size()); bool DoPlainRun = AllInputsAreFiles(); Options.SaveArtifacts = !DoPlainRun || Flags.minimize_crash_internal_step; Options.PrintNewCovPcs = Flags.print_pcs; + Options.PrintNewCovFuncs = Flags.print_funcs; Options.PrintFinalStats = Flags.print_final_stats; Options.PrintCorpusStats = Flags.print_corpus_stats; Options.PrintCoverage = Flags.print_coverage; Options.DumpCoverage = Flags.dump_coverage; + Options.UseClangCoverage = Flags.use_clang_coverage; + Options.UseFeatureFrequency = Flags.use_feature_frequency; if (Flags.exit_on_src_pos) Options.ExitOnSrcPos = Flags.exit_on_src_pos; if (Flags.exit_on_item) Options.ExitOnItem = Flags.exit_on_item; unsigned Seed = Flags.seed; // Initialize Seed. if (Seed == 0) @@ -629,18 +645,22 @@ int FuzzerDriver(int *argc, char ***argv Options.HandleAbrt = Flags.handle_abrt; Options.HandleBus = Flags.handle_bus; Options.HandleFpe = Flags.handle_fpe; Options.HandleIll = Flags.handle_ill; Options.HandleInt = Flags.handle_int; Options.HandleSegv = Flags.handle_segv; Options.HandleTerm = Flags.handle_term; Options.HandleXfsz = Flags.handle_xfsz; + Options.HandleUsr1 = Flags.handle_usr1; + Options.HandleUsr2 = Flags.handle_usr2; SetSignalHandler(Options); + std::atexit(Fuzzer::StaticExitCallback); + if (Flags.minimize_crash) return MinimizeCrashInput(Args, Options); if (Flags.minimize_crash_internal_step) return MinimizeCrashInputInternalStep(F, Corpus); if (Flags.cleanse_crash) return CleanseCrashInput(Args, Options); @@ -652,17 +672,17 @@ int FuzzerDriver(int *argc, char ***argv return 1; } Printf("INFO: EQUIVALENCE SERVER UP\n"); while (true) { SMR.WaitClient(); size_t Size = SMR.ReadByteArraySize(); SMR.WriteByteArray(nullptr, 0); const Unit tmp(SMR.GetByteArray(), SMR.GetByteArray() + Size); - F->RunOne(tmp.data(), tmp.size()); + F->ExecuteCallback(tmp.data(), tmp.size()); SMR.PostServer(); } return 0; } if (auto Name = Flags.use_equivalence_server) { if (!SMR.Open(Name)) { Printf("ERROR: can't open shared memory region\n"); @@ -689,64 +709,54 @@ int FuzzerDriver(int *argc, char ***argv "*** NOTE: fuzzing was not performed, you have only\n" "*** executed the target code on a fixed set of inputs.\n" "***\n"); F->PrintFinalStats(); exit(0); } if (Flags.merge) { - if (Options.MaxLen == 0) - F->SetMaxInputLen(kMaxSaneLen); - if (Flags.merge_control_file) - F->CrashResistantMergeInternalStep(Flags.merge_control_file); - else - F->CrashResistantMerge(Args, *Inputs, - Flags.load_coverage_summary, - Flags.save_coverage_summary); + F->CrashResistantMerge(Args, *Inputs, + Flags.load_coverage_summary, + Flags.save_coverage_summary, + Flags.merge_control_file); exit(0); } - size_t TemporaryMaxLen = Options.MaxLen ? Options.MaxLen : kMaxSaneLen; - - UnitVector InitialCorpus; - for (auto &Inp : *Inputs) { - Printf("Loading corpus dir: %s\n", Inp.c_str()); - ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, - TemporaryMaxLen, /*ExitOnError=*/false); + if (Flags.merge_inner) { + const size_t kDefaultMaxMergeLen = 1 << 20; + if (Options.MaxLen == 0) + F->SetMaxInputLen(kDefaultMaxMergeLen); + assert(Flags.merge_control_file); + F->CrashResistantMergeInternalStep(Flags.merge_control_file); + exit(0); } if (Flags.analyze_dict) { + size_t MaxLen = INT_MAX; // Large max length. + UnitVector InitialCorpus; + for (auto &Inp : *Inputs) { + Printf("Loading corpus dir: %s\n", Inp.c_str()); + ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, + MaxLen, /*ExitOnError=*/false); + } + if (Dictionary.empty() || Inputs->empty()) { Printf("ERROR: can't analyze dict without dict and corpus provided\n"); return 1; } if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { Printf("Dictionary analysis failed\n"); exit(1); } - Printf("Dictionary analysis suceeded\n"); + Printf("Dictionary analysis succeeded\n"); exit(0); } - if (Options.MaxLen == 0) { - size_t MaxLen = 0; - for (auto &U : InitialCorpus) - MaxLen = std::max(U.size(), MaxLen); - F->SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxLen), kMaxSaneLen)); - } - - if (InitialCorpus.empty()) { - InitialCorpus.push_back(Unit({'\n'})); // Valid ASCII input. - if (Options.Verbosity) - Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); - } - F->ShuffleAndMinimize(&InitialCorpus); - InitialCorpus.clear(); // Don't need this memory any more. - F->Loop(); + F->Loop(*Inputs); if (Flags.verbosity) Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), F->secondsSinceProcessStartUp()); F->PrintFinalStats(); exit(0); // Don't let F destroy itself. }
--- a/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctions.def @@ -28,16 +28,17 @@ EXT_FUNC(LLVMFuzzerCustomCrossOver, size // Sanitizer functions EXT_FUNC(__lsan_enable, void, (), false); EXT_FUNC(__lsan_disable, void, (), false); EXT_FUNC(__lsan_do_recoverable_leak_check, int, (), false); EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int, (void (*malloc_hook)(const volatile void *, size_t), void (*free_hook)(const volatile void *)), false); +EXT_FUNC(__sanitizer_purge_allocator, void, (), false); EXT_FUNC(__sanitizer_print_memory_profile, int, (size_t, size_t), false); EXT_FUNC(__sanitizer_print_stack_trace, void, (), true); EXT_FUNC(__sanitizer_symbolize_pc, void, (void *, const char *fmt, char *out_buf, size_t out_buf_size), false); EXT_FUNC(__sanitizer_get_module_and_offset_for_pc, int, (void *pc, char *module_path, size_t module_path_len,void **pc_offset), false); EXT_FUNC(__sanitizer_set_death_callback, void, (void (*)(void)), true);
--- a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsDlsymWin.cpp @@ -9,16 +9,18 @@ // Implementation using dynamic loading for Windows. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_WINDOWS #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" #include "Windows.h" + +// This must be included after Windows.h. #include "Psapi.h" namespace fuzzer { ExternalFunctions::ExternalFunctions() { HMODULE Modules[1024]; DWORD BytesNeeded; HANDLE CurrentProcess = GetCurrentProcess();
--- a/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerExtFunctionsWeak.cpp @@ -8,17 +8,17 @@ //===----------------------------------------------------------------------===// // Implementation for Linux. This relies on the linker's support for weak // symbols. We don't use this approach on Apple platforms because it requires // clients of LibFuzzer to pass ``-U _<symbol_name>`` to the linker to allow // weak symbols to be undefined. That is a complication we don't want to expose // to clients right now. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" -#if LIBFUZZER_LINUX +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUCHSIA || LIBFUZZER_FREEBSD #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" extern "C" { // Declare these symbols as weak to allow them to be optionally defined. #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ __attribute__((weak)) RETURN_TYPE NAME FUNC_SIG @@ -36,18 +36,19 @@ static void CheckFnPtr(void *FnPtr, cons } } namespace fuzzer { ExternalFunctions::ExternalFunctions() { #define EXT_FUNC(NAME, RETURN_TYPE, FUNC_SIG, WARN) \ this->NAME = ::NAME; \ - CheckFnPtr((void *)::NAME, #NAME, WARN); + CheckFnPtr(reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(::NAME)), \ + #NAME, WARN); #include "FuzzerExtFunctions.def" #undef EXT_FUNC } } // namespace fuzzer -#endif // LIBFUZZER_LINUX +#endif // LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FUSCHIA || LIBFUZZER_FREEBSD
--- a/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerExtraCounters.cpp @@ -6,17 +6,17 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Extra coverage counters defined by user code. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" -#if LIBFUZZER_LINUX +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD __attribute__((weak)) extern uint8_t __start___libfuzzer_extra_counters; __attribute__((weak)) extern uint8_t __stop___libfuzzer_extra_counters; namespace fuzzer { uint8_t *ExtraCountersBegin() { return &__start___libfuzzer_extra_counters; } uint8_t *ExtraCountersEnd() { return &__stop___libfuzzer_extra_counters; } ATTRIBUTE_NO_SANITIZE_ALL void ClearExtraCounters() { // hand-written memset, don't asan-ify.
--- a/tools/fuzzing/libfuzzer/FuzzerFlags.def +++ b/tools/fuzzing/libfuzzer/FuzzerFlags.def @@ -12,20 +12,25 @@ //===----------------------------------------------------------------------===// FUZZER_FLAG_INT(verbosity, 1, "Verbosity level.") FUZZER_FLAG_UNSIGNED(seed, 0, "Random seed. If 0, seed is generated.") FUZZER_FLAG_INT(runs, -1, "Number of individual test runs (-1 for infinite runs).") FUZZER_FLAG_INT(max_len, 0, "Maximum length of the test input. " "If 0, libFuzzer tries to guess a good value based on the corpus " "and reports it. ") -FUZZER_FLAG_INT(experimental_len_control, 0, "experimental flag") +FUZZER_FLAG_INT(len_control, 1000, "Try generating small inputs first, " + "then try larger inputs over time. Specifies the rate at which the length " + "limit is increased (smaller == faster). If 0, immediately try inputs with " + "size up to max_len.") FUZZER_FLAG_INT(cross_over, 1, "If 1, cross over inputs.") FUZZER_FLAG_INT(mutate_depth, 5, "Apply this number of consecutive mutations to each input.") +FUZZER_FLAG_INT(reduce_depth, 0, "Experimental/internal. " + "Reduce depth if mutations lose unique features") FUZZER_FLAG_INT(shuffle, 1, "Shuffle inputs at startup") FUZZER_FLAG_INT(prefer_small, 1, "If 1, always prefer smaller inputs during the corpus shuffle.") FUZZER_FLAG_INT( timeout, 1200, "Timeout in seconds (if positive). " "If one unit runs more than this number of seconds the process will abort.") FUZZER_FLAG_INT(error_exitcode, 77, "When libFuzzer itself reports a bug " @@ -33,17 +38,22 @@ FUZZER_FLAG_INT(error_exitcode, 77, "Whe FUZZER_FLAG_INT(timeout_exitcode, 77, "When libFuzzer reports a timeout " "this exit code will be used.") FUZZER_FLAG_INT(max_total_time, 0, "If positive, indicates the maximal total " "time in seconds to run the fuzzer.") FUZZER_FLAG_INT(help, 0, "Print help.") FUZZER_FLAG_INT(merge, 0, "If 1, the 2-nd, 3-rd, etc corpora will be " "merged into the 1-st corpus. Only interesting units will be taken. " "This flag can be used to minimize a corpus.") -FUZZER_FLAG_STRING(merge_control_file, "internal flag") +FUZZER_FLAG_STRING(merge_inner, "internal flag") +FUZZER_FLAG_STRING(merge_control_file, + "Specify a control file used for the merge process. " + "If a merge process gets killed it tries to leave this file " + "in a state suitable for resuming the merge. " + "By default a temporary file will be used.") FUZZER_FLAG_STRING(save_coverage_summary, "Experimental:" " save coverage summary to a given file." " Used with -merge=1") FUZZER_FLAG_STRING(load_coverage_summary, "Experimental:" " load coverage summary from a given file." " Treat this coverage as belonging to the first corpus. " " Used with -merge=1") FUZZER_FLAG_INT(minimize_crash, 0, "If 1, minimizes the provided" @@ -54,23 +64,24 @@ FUZZER_FLAG_INT(minimize_crash, 0, "If 1 " the minimized input triggers the same crash." ) FUZZER_FLAG_INT(cleanse_crash, 0, "If 1, tries to cleanse the provided" " crash input to make it contain fewer original bytes." " Use with -exact_artifact_path to specify the output." ) FUZZER_FLAG_INT(minimize_crash_internal_step, 0, "internal flag") FUZZER_FLAG_INT(use_counters, 1, "Use coverage counters") -FUZZER_FLAG_INT(use_indir_calls, 1, "Use indirect caller-callee counters") FUZZER_FLAG_INT(use_memmem, 1, "Use hints from intercepting memmem, strstr, etc") FUZZER_FLAG_INT(use_value_profile, 0, "Experimental. Use value profile to guide fuzzing.") FUZZER_FLAG_INT(use_cmp, 1, "Use CMP traces to guide mutations") -FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus elements.") +FUZZER_FLAG_INT(shrink, 0, "Experimental. Try to shrink corpus inputs.") +FUZZER_FLAG_INT(reduce_inputs, 1, + "Try to reduce the size of inputs while preserving their full feature sets") FUZZER_FLAG_UNSIGNED(jobs, 0, "Number of jobs to run. If jobs >= 1 we spawn" " this number of jobs in separate worker processes" " with stdout/stderr redirected to fuzz-JOB.log.") FUZZER_FLAG_UNSIGNED(workers, 0, "Number of simultaneous worker processes to run the jobs." " If zero, \"min(jobs,NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload, 1, "Reload the main corpus every <N> seconds to get new units" @@ -84,51 +95,59 @@ FUZZER_FLAG_STRING(artifact_prefix, "Wri "timeout, or slow inputs) as " "$(artifact_prefix)file") FUZZER_FLAG_STRING(exact_artifact_path, "Write the single artifact on failure (crash, timeout) " "as $(exact_artifact_path). This overrides -artifact_prefix " "and will not use checksum in the file name. Do not " "use the same path for several parallel processes.") FUZZER_FLAG_INT(print_pcs, 0, "If 1, print out newly covered PCs.") +FUZZER_FLAG_INT(print_funcs, 2, "If >=1, print out at most this number of " + "newly covered functions.") FUZZER_FLAG_INT(print_final_stats, 0, "If 1, print statistics at exit.") FUZZER_FLAG_INT(print_corpus_stats, 0, "If 1, print statistics on corpus elements at exit.") FUZZER_FLAG_INT(print_coverage, 0, "If 1, print coverage information as text" " at exit.") -FUZZER_FLAG_INT(dump_coverage, 0, "If 1, dump coverage information as a" +FUZZER_FLAG_INT(dump_coverage, 0, "Deprecated." + " If 1, dump coverage information as a" " .sancov file at exit.") FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.") FUZZER_FLAG_INT(handle_bus, 1, "If 1, try to intercept SIGBUS.") FUZZER_FLAG_INT(handle_abrt, 1, "If 1, try to intercept SIGABRT.") FUZZER_FLAG_INT(handle_ill, 1, "If 1, try to intercept SIGILL.") FUZZER_FLAG_INT(handle_fpe, 1, "If 1, try to intercept SIGFPE.") FUZZER_FLAG_INT(handle_int, 1, "If 1, try to intercept SIGINT.") FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.") FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.") +FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.") +FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.") FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; " "if 2, close stderr; if 3, close both. " - "Be careful, this will also close e.g. asan's stderr/stdout.") + "Be careful, this will also close e.g. stderr of asan.") FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled " "try to detect memory leaks during fuzzing (i.e. not only at shut down).") +FUZZER_FLAG_INT(purge_allocator_interval, 1, "Purge allocator caches and " + "quarantines every <N> seconds. When rss_limit_mb is specified (>0), " + "purging starts when RSS exceeds 50% of rss_limit_mb. Pass " + "purge_allocator_interval=-1 to disable this functionality.") FUZZER_FLAG_INT(trace_malloc, 0, "If >= 1 will print all mallocs/frees. " "If >= 2 will also print stack traces.") FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon" "reaching this limit of RSS memory usage.") +FUZZER_FLAG_INT(malloc_limit_mb, 0, "If non-zero, the fuzzer will exit " + "if the target tries to allocate this number of Mb with one malloc call. " + "If zero (default) same limit as rss_limit_mb is applied.") FUZZER_FLAG_STRING(exit_on_src_pos, "Exit if a newly found PC originates" " from the given source location. Example: -exit_on_src_pos=foo.cc:123. " "Used primarily for testing libFuzzer itself.") FUZZER_FLAG_STRING(exit_on_item, "Exit if an item with a given sha1 sum" " was added to the corpus. " "Used primarily for testing libFuzzer itself.") +FUZZER_FLAG_INT(ignore_remaining_args, 0, "If 1, ignore all arguments passed " + "after this one. Useful for fuzzers that need to do their own " + "argument parsing.") FUZZER_FLAG_STRING(run_equivalence_server, "Experimental") FUZZER_FLAG_STRING(use_equivalence_server, "Experimental") FUZZER_FLAG_INT(analyze_dict, 0, "Experimental") - -FUZZER_DEPRECATED_FLAG(exit_on_first) -FUZZER_DEPRECATED_FLAG(save_minimized_corpus) -FUZZER_DEPRECATED_FLAG(sync_command) -FUZZER_DEPRECATED_FLAG(sync_timeout) -FUZZER_DEPRECATED_FLAG(test_single_input) -FUZZER_DEPRECATED_FLAG(drill) -FUZZER_DEPRECATED_FLAG(truncate_units) -FUZZER_DEPRECATED_FLAG(output_csv) +FUZZER_FLAG_INT(use_clang_coverage, 0, "Experimental") +FUZZER_FLAG_INT(use_feature_frequency, 0, "Experimental/internal")
deleted file mode 100644 --- a/tools/fuzzing/libfuzzer/FuzzerFnAdapter.h +++ /dev/null @@ -1,187 +0,0 @@ -//===- FuzzerAdapter.h - Arbitrary function Fuzzer adapter -------*- C++ -*===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// W A R N I N G : E X P E R I M E N T A L. -// -// Defines an adapter to fuzz functions with (almost) arbitrary signatures. -//===----------------------------------------------------------------------===// - -#ifndef LLVM_FUZZER_ADAPTER_H -#define LLVM_FUZZER_ADAPTER_H - -#include <stddef.h> -#include <stdint.h> - -#include <algorithm> -#include <string> -#include <tuple> -#include <vector> - -namespace fuzzer { - -/// Unpacks bytes from \p Data according to \p F argument types -/// and calls the function. -/// Use to automatically adapt LLVMFuzzerTestOneInput interface to -/// a specific function. -/// Supported argument types: primitive types, std::vector<uint8_t>. -template <typename Fn> bool Adapt(Fn F, const uint8_t *Data, size_t Size); - -// The implementation performs several steps: -// - function argument types are obtained (Args...) -// - data is unpacked into std::tuple<Args...> one by one -// - function is called with std::tuple<Args...> containing arguments. -namespace impl { - -// Single argument unpacking. - -template <typename T> -size_t UnpackPrimitive(const uint8_t *Data, size_t Size, T *Value) { - if (Size < sizeof(T)) - return Size; - *Value = *reinterpret_cast<const T *>(Data); - return Size - sizeof(T); -} - -/// Unpacks into a given Value and returns the Size - num_consumed_bytes. -/// Return value equal to Size signals inability to unpack the data (typically -/// because there are not enough bytes). -template <typename T> -size_t UnpackSingle(const uint8_t *Data, size_t Size, T *Value); - -#define UNPACK_SINGLE_PRIMITIVE(Type) \ - template <> \ - size_t UnpackSingle<Type>(const uint8_t *Data, size_t Size, Type *Value) { \ - return UnpackPrimitive(Data, Size, Value); \ - } - -UNPACK_SINGLE_PRIMITIVE(char) -UNPACK_SINGLE_PRIMITIVE(signed char) -UNPACK_SINGLE_PRIMITIVE(unsigned char) - -UNPACK_SINGLE_PRIMITIVE(short int) -UNPACK_SINGLE_PRIMITIVE(unsigned short int) - -UNPACK_SINGLE_PRIMITIVE(int) -UNPACK_SINGLE_PRIMITIVE(unsigned int) - -UNPACK_SINGLE_PRIMITIVE(long int) -UNPACK_SINGLE_PRIMITIVE(unsigned long int) - -UNPACK_SINGLE_PRIMITIVE(bool) -UNPACK_SINGLE_PRIMITIVE(wchar_t) - -UNPACK_SINGLE_PRIMITIVE(float) -UNPACK_SINGLE_PRIMITIVE(double) -UNPACK_SINGLE_PRIMITIVE(long double) - -#undef UNPACK_SINGLE_PRIMITIVE - -template <> -size_t UnpackSingle<std::vector<uint8_t>>(const uint8_t *Data, size_t Size, - std::vector<uint8_t> *Value) { - if (Size < 1) - return Size; - size_t Len = std::min(static_cast<size_t>(*Data), Size - 1); - std::vector<uint8_t> V(Data + 1, Data + 1 + Len); - Value->swap(V); - return Size - Len - 1; -} - -template <> -size_t UnpackSingle<std::string>(const uint8_t *Data, size_t Size, - std::string *Value) { - if (Size < 1) - return Size; - size_t Len = std::min(static_cast<size_t>(*Data), Size - 1); - std::string S(Data + 1, Data + 1 + Len); - Value->swap(S); - return Size - Len - 1; -} - -// Unpacking into arbitrary tuple. - -// Recursion guard. -template <int N, typename TupleT> -typename std::enable_if<N == std::tuple_size<TupleT>::value, bool>::type -UnpackImpl(const uint8_t *Data, size_t Size, TupleT *Tuple) { - return true; -} - -// Unpack tuple elements starting from Nth. -template <int N, typename TupleT> -typename std::enable_if<N < std::tuple_size<TupleT>::value, bool>::type -UnpackImpl(const uint8_t *Data, size_t Size, TupleT *Tuple) { - size_t NewSize = UnpackSingle(Data, Size, &std::get<N>(*Tuple)); - if (NewSize == Size) { - return false; - } - - return UnpackImpl<N + 1, TupleT>(Data + (Size - NewSize), NewSize, Tuple); -} - -// Unpacks into arbitrary tuple and returns true if successful. -template <typename... Args> -bool Unpack(const uint8_t *Data, size_t Size, std::tuple<Args...> *Tuple) { - return UnpackImpl<0, std::tuple<Args...>>(Data, Size, Tuple); -} - -// Helper integer sequence templates. - -template <int...> struct Seq {}; - -template <int N, int... S> struct GenSeq : GenSeq<N - 1, N - 1, S...> {}; - -// GenSeq<N>::type is Seq<0, 1, ..., N-1> -template <int... S> struct GenSeq<0, S...> { typedef Seq<S...> type; }; - -// Function signature introspection. - -template <typename T> struct FnTraits {}; - -template <typename ReturnType, typename... Args> -struct FnTraits<ReturnType (*)(Args...)> { - enum { Arity = sizeof...(Args) }; - typedef std::tuple<Args...> ArgsTupleT; -}; - -// Calling a function with arguments in a tuple. - -template <typename Fn, int... S> -void ApplyImpl(Fn F, const typename FnTraits<Fn>::ArgsTupleT &Params, - Seq<S...>) { - F(std::get<S>(Params)...); -} - -template <typename Fn> -void Apply(Fn F, const typename FnTraits<Fn>::ArgsTupleT &Params) { - // S is Seq<0, ..., Arity-1> - auto S = typename GenSeq<FnTraits<Fn>::Arity>::type(); - ApplyImpl(F, Params, S); -} - -// Unpacking data into arguments tuple of correct type and calling the function. -template <typename Fn> -bool UnpackAndApply(Fn F, const uint8_t *Data, size_t Size) { - typename FnTraits<Fn>::ArgsTupleT Tuple; - if (!Unpack(Data, Size, &Tuple)) - return false; - - Apply(F, Tuple); - return true; -} - -} // namespace impl - -template <typename Fn> bool Adapt(Fn F, const uint8_t *Data, size_t Size) { - return impl::UnpackAndApply(F, Data, Size); -} - -} // namespace fuzzer - -#endif
--- a/tools/fuzzing/libfuzzer/FuzzerIO.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerIO.cpp @@ -4,17 +4,16 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // IO functions. //===----------------------------------------------------------------------===// -#include "mozilla/Unused.h" #include "FuzzerIO.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include <algorithm> #include <cstdarg> #include <fstream> #include <iterator> #include <sys/stat.h> @@ -34,17 +33,19 @@ long GetEpoch(const std::string &Path) { Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) { std::ifstream T(Path); if (ExitOnError && !T) { Printf("No such directory: %s; exiting\n", Path.c_str()); exit(1); } T.seekg(0, T.end); - size_t FileLen = T.tellg(); + auto EndPos = T.tellg(); + if (EndPos < 0) return {}; + size_t FileLen = EndPos; if (MaxSize) FileLen = std::min(FileLen, MaxSize); T.seekg(0, T.beg); Unit Res(FileLen); T.read(reinterpret_cast<char *>(Res.data()), FileLen); return Res; } @@ -58,38 +59,47 @@ std::string FileToString(const std::stri void CopyFileToErr(const std::string &Path) { Printf("%s", FileToString(Path).c_str()); } void WriteToFile(const Unit &U, const std::string &Path) { // Use raw C interface because this function may be called from a sig handler. FILE *Out = fopen(Path.c_str(), "w"); if (!Out) return; - mozilla::Unused << fwrite(U.data(), sizeof(U[0]), U.size(), Out); + fwrite(U.data(), sizeof(U[0]), U.size(), Out); fclose(Out); } -void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, +void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch, size_t MaxSize, bool ExitOnError) { long E = Epoch ? *Epoch : 0; - std::vector<std::string> Files; + Vector<std::string> Files; ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true); size_t NumLoaded = 0; for (size_t i = 0; i < Files.size(); i++) { auto &X = Files[i]; if (Epoch && GetEpoch(X) < E) continue; NumLoaded++; if ((NumLoaded & (NumLoaded - 1)) == 0 && NumLoaded >= 1024) Printf("Loaded %zd/%zd files from %s\n", NumLoaded, Files.size(), Path); auto S = FileToVector(X, MaxSize, ExitOnError); if (!S.empty()) V->push_back(S); } } + +void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V) { + Vector<std::string> Files; + ListFilesInDirRecursive(Dir, 0, &Files, /*TopDir*/true); + for (auto &File : Files) + if (size_t Size = FileSize(File)) + V->push_back({File, Size}); +} + std::string DirPlusFile(const std::string &DirPath, const std::string &FileName) { return DirPath + GetSeparator() + FileName; } void DupAndCloseStderr() { int OutputFd = DuplicateFile(2); if (OutputFd > 0) {
--- a/tools/fuzzing/libfuzzer/FuzzerIO.h +++ b/tools/fuzzing/libfuzzer/FuzzerIO.h @@ -22,17 +22,17 @@ Unit FileToVector(const std::string &Pat bool ExitOnError = true); std::string FileToString(const std::string &Path); void CopyFileToErr(const std::string &Path); void WriteToFile(const Unit &U, const std::string &Path); -void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V, +void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V, long *Epoch, size_t MaxSize, bool ExitOnError); // Returns "Dir/FileName" or equivalent for the current OS. std::string DirPlusFile(const std::string &DirPath, const std::string &FileName); // Returns the name of the dir, similar to the 'dirname' utility. std::string DirName(const std::string &FileName); @@ -48,19 +48,28 @@ void CloseStdout(); void Printf(const char *Fmt, ...); // Print using raw syscalls, useful when printing at early init stages. void RawPrint(const char *Str); // Platform specific functions: bool IsFile(const std::string &Path); +size_t FileSize(const std::string &Path); void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - std::vector<std::string> *V, bool TopDir); + Vector<std::string> *V, bool TopDir); + +struct SizedFile { + std::string File; + size_t Size; + bool operator<(const SizedFile &B) const { return Size < B.Size; } +}; + +void GetSizedFilesFromDir(const std::string &Dir, Vector<SizedFile> *V); char GetSeparator(); FILE* OpenFile(int Fd, const char *Mode); int CloseFile(int Fd); int DuplicateFile(int Fd);
--- a/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerIOPosix.cpp @@ -4,19 +4,18 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // IO functions implementation using Posix API. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" -#if LIBFUZZER_POSIX +#if LIBFUZZER_POSIX || LIBFUZZER_FUCHSIA -#include "mozilla/Unused.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" #include <cstdarg> #include <cstdio> #include <dirent.h> #include <fstream> #include <iterator> #include <libgen.h> @@ -28,32 +27,49 @@ namespace fuzzer { bool IsFile(const std::string &Path) { struct stat St; if (stat(Path.c_str(), &St)) return false; return S_ISREG(St.st_mode); } +static bool IsDirectory(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return false; + return S_ISDIR(St.st_mode); +} + +size_t FileSize(const std::string &Path) { + struct stat St; + if (stat(Path.c_str(), &St)) + return 0; + return St.st_size; +} + void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - std::vector<std::string> *V, bool TopDir) { + Vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; DIR *D = opendir(Dir.c_str()); if (!D) { Printf("No such directory: %s; exiting\n", Dir.c_str()); exit(1); } while (auto E = readdir(D)) { std::string Path = DirPlusFile(Dir, E->d_name); - if (E->d_type == DT_REG || E->d_type == DT_LNK) + if (E->d_type == DT_REG || E->d_type == DT_LNK || + (E->d_type == DT_UNKNOWN && IsFile(Path))) V->push_back(Path); - else if (E->d_type == DT_DIR && *E->d_name != '.') + else if ((E->d_type == DT_DIR || + (E->d_type == DT_UNKNOWN && IsDirectory(Path))) && + *E->d_name != '.') ListFilesInDirRecursive(Path, Epoch, V, false); } closedir(D); if (Epoch && TopDir) *Epoch = E; } char GetSeparator() { @@ -111,14 +127,14 @@ bool IsInterestingCoverageFile(const std return false; if (FileName == "<null>") return false; return true; } void RawPrint(const char *Str) { - mozilla::Unused << write(2, Str, strlen(Str)); + write(2, Str, strlen(Str)); } } // namespace fuzzer #endif // LIBFUZZER_POSIX
--- a/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerIOWindows.cpp @@ -68,17 +68,17 @@ bool IsFile(const std::string &Path) { Path.c_str(), GetLastError()); return false; } return IsFile(Path, Att); } void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, - std::vector<std::string> *V, bool TopDir) { + Vector<std::string> *V, bool TopDir) { auto E = GetEpoch(Dir); if (Epoch) if (E && *Epoch >= E) return; std::string Path(Dir); assert(!Path.empty()); if (Path.back() != '\\') Path.push_back('\\'); @@ -177,32 +177,32 @@ static size_t ParseDrive(const std::stri static size_t ParseFileName(const std::string &FileName, const size_t Offset) { size_t Pos = Offset; const size_t End = FileName.size(); for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) ; return Pos - Offset; } -// Parse a directory ending in separator, like: SomeDir\ +// Parse a directory ending in separator, like: `SomeDir\` // Returns number of characters considered if successful. static size_t ParseDir(const std::string &FileName, const size_t Offset) { size_t Pos = Offset; const size_t End = FileName.size(); if (Pos >= End || IsSeparator(FileName[Pos])) return 0; for(; Pos < End && !IsSeparator(FileName[Pos]); ++Pos) ; if (Pos >= End) return 0; ++Pos; // Include separator. return Pos - Offset; } -// Parse a servername and share, like: SomeServer\SomeShare\ +// Parse a servername and share, like: `SomeServer\SomeShare\` // Returns number of characters considered if successful. static size_t ParseServerAndShare(const std::string &FileName, const size_t Offset) { size_t Pos = Offset, Res; if (!(Res = ParseDir(FileName, Pos))) return 0; Pos += Res; if (!(Res = ParseDir(FileName, Pos)))
--- a/tools/fuzzing/libfuzzer/FuzzerInterface.h +++ b/tools/fuzzing/libfuzzer/FuzzerInterface.h @@ -25,43 +25,47 @@ #ifdef __cplusplus extern "C" { #endif // __cplusplus // Mandatory user-provided target function. // Executes the code under test with [Data, Data+Size) as the input. // libFuzzer will invoke this function *many* times with different inputs. // Must return 0. -int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); +__attribute__((visibility("default"))) int +LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); // Optional user-provided initialization function. // If provided, this function will be called by libFuzzer once at startup. // It may read and modify argc/argv. // Must return 0. -int LLVMFuzzerInitialize(int *argc, char ***argv); +__attribute__((visibility("default"))) int LLVMFuzzerInitialize(int *argc, + char ***argv); // Optional user-provided custom mutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. // Given the same Seed produces the same mutation. -size_t LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, - unsigned int Seed); +__attribute__((visibility("default"))) size_t +LLVMFuzzerCustomMutator(uint8_t *Data, size_t Size, size_t MaxSize, + unsigned int Seed); // Optional user-provided custom cross-over function. // Combines pieces of Data1 & Data2 together into Out. // Returns the new size, which is not greater than MaxOutSize. // Should produce the same mutation given the same Seed. -size_t LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, - const uint8_t *Data2, size_t Size2, - uint8_t *Out, size_t MaxOutSize, - unsigned int Seed); +__attribute__((visibility("default"))) size_t +LLVMFuzzerCustomCrossOver(const uint8_t *Data1, size_t Size1, + const uint8_t *Data2, size_t Size2, uint8_t *Out, + size_t MaxOutSize, unsigned int Seed); // Experimental, may go away in future. // libFuzzer-provided function to be used inside LLVMFuzzerCustomMutator. // Mutates raw data in [Data, Data+Size) inplace. // Returns the new size, which is not greater than MaxSize. -size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); +__attribute__((visibility("default"))) size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize); #ifdef __cplusplus } // extern "C" #endif // __cplusplus #endif // LLVM_FUZZER_INTERFACE_H
--- a/tools/fuzzing/libfuzzer/FuzzerInternal.h +++ b/tools/fuzzing/libfuzzer/FuzzerInternal.h @@ -30,20 +30,19 @@ namespace fuzzer { using namespace std::chrono; class Fuzzer { public: Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options); ~Fuzzer(); - void Loop(); + void Loop(const Vector<std::string> &CorpusDirs); + void ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs); void MinimizeCrashLoop(const Unit &U); - void ShuffleAndMinimize(UnitVector *V); - void InitializeTraceState(); void RereadOutputCorpus(size_t MaxSize); size_t secondsSinceProcessStartUp() { return duration_cast<seconds>(system_clock::now() - ProcessStartTime) .count(); } bool TimedOut() { @@ -56,28 +55,32 @@ public: size_t Seconds = secondsSinceProcessStartUp(); return Seconds ? TotalNumberOfRuns / Seconds : 0; } size_t getTotalNumberOfRuns() { return TotalNumberOfRuns; } static void StaticAlarmCallback(); static void StaticCrashSignalCallback(); + static void StaticExitCallback(); static void StaticInterruptCallback(); static void StaticFileSizeExceedCallback(); + static void StaticGracefulExitCallback(); void ExecuteCallback(const uint8_t *Data, size_t Size); - size_t RunOne(const uint8_t *Data, size_t Size); + bool RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile = false, + InputInfo *II = nullptr, bool *FoundUniqFeatures = nullptr); // Merge Corpora[1:] into Corpora[0]. - void Merge(const std::vector<std::string> &Corpora); - void CrashResistantMerge(const std::vector<std::string> &Args, - const std::vector<std::string> &Corpora, + void Merge(const Vector<std::string> &Corpora); + void CrashResistantMerge(const Vector<std::string> &Args, + const Vector<std::string> &Corpora, const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull); + const char *CoverageSummaryOutputPathOrNull, + const char *MergeControlFilePathOrNull); void CrashResistantMergeInternalStep(const std::string &ControlFilePath); MutationDispatcher &GetMD() { return MD; } void PrintFinalStats(); void SetMaxInputLen(size_t MaxInputLen); void SetMaxMutationLen(size_t MaxMutationLen); void RssLimitCallback(); bool InFuzzingThread() const { return IsMyThread; } @@ -86,66 +89,67 @@ public: bool DuringInitialCorpusExecution); void HandleMalloc(size_t Size); void AnnounceOutput(const uint8_t *Data, size_t Size); private: void AlarmCallback(); void CrashCallback(); + void ExitCallback(); + void MaybeExitGracefully(); void CrashOnOverwrittenData(); void InterruptCallback(); void MutateAndTestOne(); + void PurgeAllocator(); void ReportNewCoverage(InputInfo *II, const Unit &U); - size_t RunOne(const Unit &U) { return RunOne(U.data(), U.size()); } + void PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size); void WriteToOutputCorpus(const Unit &U); void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix); void PrintStats(const char *Where, const char *End = "\n", size_t Units = 0); - void PrintStatusForNewUnit(const Unit &U); - void ShuffleCorpus(UnitVector *V); - void AddToCorpus(const Unit &U); + void PrintStatusForNewUnit(const Unit &U, const char *Text); void CheckExitOnSrcPosOrItem(); - // Trace-based fuzzing: we run a unit with some kind of tracing - // enabled and record potentially useful mutations. Then - // We apply these mutations one by one to the unit and run it again. - - // Start tracing; forget all previously proposed mutations. - void StartTraceRecording(); - // Stop tracing. - void StopTraceRecording(); - static void StaticDeathCallback(); void DumpCurrentUnit(const char *Prefix); void DeathCallback(); void AllocateCurrentUnitData(); uint8_t *CurrentUnitData = nullptr; std::atomic<size_t> CurrentUnitSize; uint8_t BaseSha1[kSHA1NumBytes]; // Checksum of the base unit. bool RunningCB = false; + bool GracefulExitRequested = false; + size_t TotalNumberOfRuns = 0; size_t NumberOfNewUnitsAdded = 0; + size_t LastCorpusUpdateRun = 0; + bool HasMoreMallocsThanFrees = false; size_t NumberOfLeakDetectionAttempts = 0; + system_clock::time_point LastAllocatorPurgeAttemptTime = system_clock::now(); + UserCallback CB; InputCorpus &Corpus; MutationDispatcher &MD; FuzzingOptions Options; system_clock::time_point ProcessStartTime = system_clock::now(); system_clock::time_point UnitStartTime, UnitStopTime; long TimeOfLongestUnitInSeconds = 0; long EpochOfLastReadOfOutputCorpus = 0; size_t MaxInputLen = 0; size_t MaxMutationLen = 0; + size_t TmpMaxMutationLen = 0; + + Vector<uint32_t> UniqFeatureSetTmp; // Need to know our own thread. static thread_local bool IsMyThread; }; } // namespace fuzzer #endif // LLVM_FUZZER_INTERNAL_H
--- a/tools/fuzzing/libfuzzer/FuzzerLoop.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerLoop.cpp @@ -5,31 +5,29 @@ // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Fuzzer's main loop. //===----------------------------------------------------------------------===// #include "FuzzerCorpus.h" +#include "FuzzerIO.h" #include "FuzzerInternal.h" -#include "FuzzerIO.h" #include "FuzzerMutate.h" #include "FuzzerRandom.h" #include "FuzzerShmem.h" #include "FuzzerTracePC.h" #include <algorithm> #include <cstring> #include <memory> +#include <mutex> #include <set> #if defined(__has_include) -#if __has_include(<sanitizer / coverage_interface.h>) -#include <sanitizer/coverage_interface.h> -#endif #if __has_include(<sanitizer / lsan_interface.h>) #include <sanitizer/lsan_interface.h> #endif #endif #define NO_SANITIZE_MEMORY #if defined(__has_feature) #if __has_feature(memory_sanitizer) @@ -67,44 +65,71 @@ struct MallocFreeTracer { Mallocs = 0; Frees = 0; TraceLevel = 0; return Result; } std::atomic<size_t> Mallocs; std::atomic<size_t> Frees; int TraceLevel = 0; + + std::recursive_mutex TraceMutex; + bool TraceDisabled = false; }; static MallocFreeTracer AllocTracer; +// Locks printing and avoids nested hooks triggered from mallocs/frees in +// sanitizer. +class TraceLock { +public: + TraceLock() : Lock(AllocTracer.TraceMutex) { + AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; + } + ~TraceLock() { AllocTracer.TraceDisabled = !AllocTracer.TraceDisabled; } + + bool IsDisabled() const { + // This is already inverted value. + return !AllocTracer.TraceDisabled; + } + +private: + std::lock_guard<std::recursive_mutex> Lock; +}; + ATTRIBUTE_NO_SANITIZE_MEMORY void MallocHook(const volatile void *ptr, size_t size) { size_t N = AllocTracer.Mallocs++; F->HandleMalloc(size); if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; Printf("MALLOC[%zd] %p %zd\n", N, ptr, size); if (TraceLevel >= 2 && EF) EF->__sanitizer_print_stack_trace(); } } ATTRIBUTE_NO_SANITIZE_MEMORY void FreeHook(const volatile void *ptr) { size_t N = AllocTracer.Frees++; if (int TraceLevel = AllocTracer.TraceLevel) { + TraceLock Lock; + if (Lock.IsDisabled()) + return; Printf("FREE[%zd] %p\n", N, ptr); if (TraceLevel >= 2 && EF) EF->__sanitizer_print_stack_trace(); } } // Crash on a single malloc that exceeds the rss limit. void Fuzzer::HandleMalloc(size_t Size) { - if (!Options.RssLimitMb || (Size >> 20) < (size_t)Options.RssLimitMb) + if (!Options.MallocLimitMb || (Size >> 20) < (size_t)Options.MallocLimitMb) return; Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(), Size); Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n"); if (EF->__sanitizer_print_stack_trace) EF->__sanitizer_print_stack_trace(); DumpCurrentUnit("oom-"); Printf("SUMMARY: libFuzzer: out-of-memory\n"); @@ -112,51 +137,53 @@ void Fuzzer::HandleMalloc(size_t Size) { _Exit(Options.ErrorExitCode); // Stop right now. } Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options) : CB(CB), Corpus(Corpus), MD(MD), Options(Options) { if (EF->__sanitizer_set_death_callback) EF->__sanitizer_set_death_callback(StaticDeathCallback); - InitializeTraceState(); assert(!F); F = this; TPC.ResetMaps(); IsMyThread = true; if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks) EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook); TPC.SetUseCounters(Options.UseCounters); TPC.SetUseValueProfile(Options.UseValueProfile); - TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetUseClangCoverage(Options.UseClangCoverage); if (Options.Verbosity) TPC.PrintModuleInfo(); if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec) EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus); MaxInputLen = MaxMutationLen = Options.MaxLen; + TmpMaxMutationLen = Max(size_t(4), Corpus.MaxInputSize()); AllocateCurrentUnitData(); CurrentUnitSize = 0; memset(BaseSha1, 0, sizeof(BaseSha1)); } -Fuzzer::~Fuzzer() { } +Fuzzer::~Fuzzer() {} void Fuzzer::AllocateCurrentUnitData() { - if (CurrentUnitData || MaxInputLen == 0) return; + if (CurrentUnitData || MaxInputLen == 0) + return; CurrentUnitData = new uint8_t[MaxInputLen]; } void Fuzzer::StaticDeathCallback() { assert(F); F->DeathCallback(); } void Fuzzer::DumpCurrentUnit(const char *Prefix) { - if (!CurrentUnitData) return; // Happens when running individual inputs. + if (!CurrentUnitData) + return; // Happens when running individual inputs. MD.PrintMutationSequence(); Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str()); size_t UnitSize = CurrentUnitSize; if (UnitSize <= kMaxUnitSizeToPrint) { PrintHexArray(CurrentUnitData, UnitSize, "\n"); PrintASCII(CurrentUnitData, UnitSize, "\n"); } WriteUnitToFileWithPrefix({CurrentUnitData, CurrentUnitData + UnitSize}, @@ -174,51 +201,82 @@ void Fuzzer::StaticAlarmCallback() { F->AlarmCallback(); } void Fuzzer::StaticCrashSignalCallback() { assert(F); F->CrashCallback(); } +void Fuzzer::StaticExitCallback() { + assert(F); + F->ExitCallback(); +} + void Fuzzer::StaticInterruptCallback() { assert(F); F->InterruptCallback(); } +void Fuzzer::StaticGracefulExitCallback() { + assert(F); + F->GracefulExitRequested = true; + Printf("INFO: signal received, trying to exit gracefully\n"); +} + void Fuzzer::StaticFileSizeExceedCallback() { Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid()); exit(1); } void Fuzzer::CrashCallback() { Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid()); if (EF->__sanitizer_print_stack_trace) EF->__sanitizer_print_stack_trace(); Printf("NOTE: libFuzzer has rudimentary signal handlers.\n" " Combine libFuzzer with AddressSanitizer or similar for better " "crash reports.\n"); Printf("SUMMARY: libFuzzer: deadly signal\n"); DumpCurrentUnit("crash-"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // Stop right now. + _Exit(Options.ErrorExitCode); // Stop right now. +} + +void Fuzzer::ExitCallback() { + if (!RunningCB) + return; // This exit did not come from the user callback + Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid()); + if (EF->__sanitizer_print_stack_trace) + EF->__sanitizer_print_stack_trace(); + Printf("SUMMARY: libFuzzer: fuzz target exited\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); +} + +void Fuzzer::MaybeExitGracefully() { + if (!GracefulExitRequested) return; + Printf("==%lu== INFO: libFuzzer: exiting as requested\n", GetPid()); + PrintFinalStats(); + _Exit(0); } void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); - _Exit(0); // Stop right now, don't perform any at-exit actions. + _Exit(0); // Stop right now, don't perform any at-exit actions. } NO_SANITIZE_MEMORY void Fuzzer::AlarmCallback() { assert(Options.UnitTimeoutSec > 0); // In Windows Alarm callback is executed by a different thread. #if !LIBFUZZER_WINDOWS - if (!InFuzzingThread()) return; + if (!InFuzzingThread()) + return; #endif if (!RunningCB) return; // We have not started running units yet. size_t Seconds = duration_cast<seconds>(system_clock::now() - UnitStartTime).count(); if (Seconds == 0) return; if (Options.Verbosity >= 2) @@ -254,171 +312,170 @@ void Fuzzer::RssLimitCallback() { void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) { size_t ExecPerSec = execPerSec(); if (!Options.Verbosity) return; Printf("#%zd\t%s", TotalNumberOfRuns, Where); if (size_t N = TPC.GetTotalPCCoverage()) Printf(" cov: %zd", N); if (size_t N = Corpus.NumFeatures()) - Printf( " ft: %zd", N); + Printf(" ft: %zd", N); if (!Corpus.empty()) { Printf(" corp: %zd", Corpus.NumActiveUnits()); if (size_t N = Corpus.SizeInBytes()) { - if (N < (1<<14)) + if (N < (1 << 14)) Printf("/%zdb", N); else if (N < (1 << 24)) Printf("/%zdKb", N >> 10); else Printf("/%zdMb", N >> 20); } } + if (TmpMaxMutationLen) + Printf(" lim: %zd", TmpMaxMutationLen); if (Units) Printf(" units: %zd", Units); Printf(" exec/s: %zd", ExecPerSec); Printf(" rss: %zdMb", GetPeakRSSMb()); Printf("%s", End); } void Fuzzer::PrintFinalStats() { if (Options.PrintCoverage) TPC.PrintCoverage(); if (Options.DumpCoverage) TPC.DumpCoverage(); if (Options.PrintCorpusStats) Corpus.PrintStats(); - if (!Options.PrintFinalStats) return; + if (!Options.PrintFinalStats) + return; size_t ExecPerSec = execPerSec(); Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns); Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec); Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded); Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds); Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb()); } void Fuzzer::SetMaxInputLen(size_t MaxInputLen) { assert(this->MaxInputLen == 0); // Can only reset MaxInputLen from 0 to non-0. assert(MaxInputLen); this->MaxInputLen = MaxInputLen; this->MaxMutationLen = MaxInputLen; AllocateCurrentUnitData(); - Printf("INFO: -max_len is not provided, using %zd\n", MaxInputLen); + Printf("INFO: -max_len is not provided; " + "libFuzzer will not generate inputs larger than %zd bytes\n", + MaxInputLen); } void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) { assert(MaxMutationLen && MaxMutationLen <= MaxInputLen); this->MaxMutationLen = MaxMutationLen; } void Fuzzer::CheckExitOnSrcPosOrItem() { if (!Options.ExitOnSrcPos.empty()) { - static auto *PCsSet = new std::set<uintptr_t>; - for (size_t i = 1, N = TPC.GetNumPCs(); i < N; i++) { - uintptr_t PC = TPC.GetPC(i); - if (!PC) continue; - if (!PCsSet->insert(PC).second) continue; - std::string Descr = DescribePC("%L", PC); + static auto *PCsSet = new Set<uintptr_t>; + auto HandlePC = [&](uintptr_t PC) { + if (!PCsSet->insert(PC).second) + return; + std::string Descr = DescribePC("%F %L", PC + 1); if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) { Printf("INFO: found line matching '%s', exiting.\n", Options.ExitOnSrcPos.c_str()); _Exit(0); } - } + }; + TPC.ForEachObservedPC(HandlePC); } if (!Options.ExitOnItem.empty()) { if (Corpus.HasUnit(Options.ExitOnItem)) { Printf("INFO: found item with checksum '%s', exiting.\n", Options.ExitOnItem.c_str()); _Exit(0); } } } void Fuzzer::RereadOutputCorpus(size_t MaxSize) { - if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return; - std::vector<Unit> AdditionalCorpus; + if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) + return; + Vector<Unit> AdditionalCorpus; ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus, &EpochOfLastReadOfOutputCorpus, MaxSize, /*ExitOnError*/ false); if (Options.Verbosity >= 2) Printf("Reload: read %zd new units.\n", AdditionalCorpus.size()); bool Reloaded = false; for (auto &U : AdditionalCorpus) { if (U.size() > MaxSize) U.resize(MaxSize); if (!Corpus.HasUnit(U)) { - if (size_t NumFeatures = RunOne(U)) { + if (RunOne(U.data(), U.size())) { CheckExitOnSrcPosOrItem(); - Corpus.AddToCorpus(U, NumFeatures); Reloaded = true; } } } if (Reloaded) PrintStats("RELOAD"); } -void Fuzzer::ShuffleCorpus(UnitVector *V) { - std::shuffle(V->begin(), V->end(), MD.GetRand()); - if (Options.PreferSmall) - std::stable_sort(V->begin(), V->end(), [](const Unit &A, const Unit &B) { - return A.size() < B.size(); - }); -} - -void Fuzzer::ShuffleAndMinimize(UnitVector *InitialCorpus) { - Printf("#0\tREAD units: %zd\n", InitialCorpus->size()); - if (Options.ShuffleAtStartUp) - ShuffleCorpus(InitialCorpus); - - // Test the callback with empty input and never try it again. - uint8_t dummy; - ExecuteCallback(&dummy, 0); - - for (const auto &U : *InitialCorpus) { - if (size_t NumFeatures = RunOne(U)) { - CheckExitOnSrcPosOrItem(); - Corpus.AddToCorpus(U, NumFeatures); - } - TryDetectingAMemoryLeak(U.data(), U.size(), - /*DuringInitialCorpusExecution*/ true); - } - PrintStats("INITED"); - if (Corpus.empty()) { - Printf("ERROR: no interesting inputs were found. " - "Is the code instrumented for coverage? Exiting.\n"); - exit(1); - } -} - -size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) { - if (!Size) return 0; - TotalNumberOfRuns++; - - ExecuteCallback(Data, Size); - - size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); - TPC.CollectFeatures([&](size_t Feature) { - Corpus.AddFeature(Feature, Size, Options.Shrink); - }); - size_t NumUpdatesAfter = Corpus.NumFeatureUpdates(); - +void Fuzzer::PrintPulseAndReportSlowInput(const uint8_t *Data, size_t Size) { auto TimeOfUnit = duration_cast<seconds>(UnitStopTime - UnitStartTime).count(); if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) && secondsSinceProcessStartUp() >= 2) PrintStats("pulse "); if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 && TimeOfUnit >= Options.ReportSlowUnits) { TimeOfLongestUnitInSeconds = TimeOfUnit; Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds); WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-"); } - return NumUpdatesAfter - NumUpdatesBefore; +} + +bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile, + InputInfo *II, bool *FoundUniqFeatures) { + if (!Size) + return false; + + ExecuteCallback(Data, Size); + + UniqFeatureSetTmp.clear(); + size_t FoundUniqFeaturesOfII = 0; + size_t NumUpdatesBefore = Corpus.NumFeatureUpdates(); + TPC.CollectFeatures([&](size_t Feature) { + if (Options.UseFeatureFrequency) + Corpus.UpdateFeatureFrequency(Feature); + if (Corpus.AddFeature(Feature, Size, Options.Shrink)) + UniqFeatureSetTmp.push_back(Feature); + if (Options.ReduceInputs && II) + if (std::binary_search(II->UniqFeatureSet.begin(), + II->UniqFeatureSet.end(), Feature)) + FoundUniqFeaturesOfII++; + }); + if (FoundUniqFeatures) + *FoundUniqFeatures = FoundUniqFeaturesOfII; + PrintPulseAndReportSlowInput(Data, Size); + size_t NumNewFeatures = Corpus.NumFeatureUpdates() - NumUpdatesBefore; + if (NumNewFeatures) { + TPC.UpdateObservedPCs(); + Corpus.AddToCorpus({Data, Data + Size}, NumNewFeatures, MayDeleteFile, + UniqFeatureSetTmp); + return true; + } + if (II && FoundUniqFeaturesOfII && + FoundUniqFeaturesOfII == II->UniqFeatureSet.size() && + II->U.size() > Size) { + Corpus.Replace(II, {Data, Data + Size}); + return true; + } + return false; } size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const { assert(InFuzzingThread()); *Data = CurrentUnitData; return CurrentUnitSize; } @@ -436,16 +493,18 @@ static bool LooseMemeq(const uint8_t *A, if (Size <= 64) return !memcmp(A, B, Size); // Compare first and last Limit/2 bytes. return !memcmp(A, B, Limit / 2) && !memcmp(A + Size - Limit / 2, B + Size - Limit / 2, Limit / 2); } void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) { + TPC.RecordInitialStack(); + TotalNumberOfRuns++; assert(InFuzzingThread()); if (SMR.IsClient()) SMR.WriteByteArray(Data, Size); // We copy the contents of Unit into a separate heap buffer // so that we reliably find buffer overflows in it. uint8_t *DataCopy = new uint8_t[Size]; memcpy(DataCopy, Data, Size); if (CurrentUnitData && CurrentUnitData != Data) @@ -470,67 +529,74 @@ void Fuzzer::ExecuteCallback(const uint8 void Fuzzer::WriteToOutputCorpus(const Unit &U) { if (Options.OnlyASCII) assert(IsASCII(U)); if (Options.OutputCorpus.empty()) return; std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U)); WriteToFile(U, Path); if (Options.Verbosity >= 2) - Printf("Written to %s\n", Path.c_str()); + Printf("Written %zd bytes to %s\n", U.size(), Path.c_str()); } void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) { if (!Options.SaveArtifacts) return; std::string Path = Options.ArtifactPrefix + Prefix + Hash(U); if (!Options.ExactArtifactPath.empty()) Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix. WriteToFile(U, Path); Printf("artifact_prefix='%s'; Test unit written to %s\n", Options.ArtifactPrefix.c_str(), Path.c_str()); if (U.size() <= kMaxUnitSizeToPrint) Printf("Base64: %s\n", Base64(U).c_str()); } -void Fuzzer::PrintStatusForNewUnit(const Unit &U) { +void Fuzzer::PrintStatusForNewUnit(const Unit &U, const char *Text) { if (!Options.PrintNEW) return; - PrintStats("NEW ", ""); + PrintStats(Text, ""); if (Options.Verbosity) { - Printf(" L: %zd ", U.size()); + Printf(" L: %zd/%zd ", U.size(), Corpus.MaxInputSize()); MD.PrintMutationSequence(); Printf("\n"); } } void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) { II->NumSuccessfullMutations++; MD.RecordSuccessfulMutationSequence(); - PrintStatusForNewUnit(U); + PrintStatusForNewUnit(U, II->Reduced ? "REDUCE" : "NEW "); WriteToOutputCorpus(U); NumberOfNewUnitsAdded++; - TPC.PrintNewPCs(); + CheckExitOnSrcPosOrItem(); // Check only after the unit is saved to corpus. + LastCorpusUpdateRun = TotalNumberOfRuns; } // Tries detecting a memory leak on the particular input that we have just // executed before calling this function. void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, bool DuringInitialCorpusExecution) { - if (!HasMoreMallocsThanFrees) return; // mallocs==frees, a leak is unlikely. - if (!Options.DetectLeaks) return; + if (!HasMoreMallocsThanFrees) + return; // mallocs==frees, a leak is unlikely. + if (!Options.DetectLeaks) + return; + if (!DuringInitialCorpusExecution && + TotalNumberOfRuns >= Options.MaxNumberOfRuns) + return; if (!&(EF->__lsan_enable) || !&(EF->__lsan_disable) || !(EF->__lsan_do_recoverable_leak_check)) - return; // No lsan. + return; // No lsan. // Run the target once again, but with lsan disabled so that if there is // a real leak we do not report it twice. EF->__lsan_disable(); ExecuteCallback(Data, Size); EF->__lsan_enable(); - if (!HasMoreMallocsThanFrees) return; // a leak is unlikely. + if (!HasMoreMallocsThanFrees) + return; // a leak is unlikely. if (NumberOfLeakDetectionAttempts++ > 1000) { Options.DetectLeaks = false; Printf("INFO: libFuzzer disabled leak detection after every mutation.\n" " Most likely the target function accumulates allocated\n" " memory in a global state w/o actually leaking it.\n" " You may try running this binary with -trace_malloc=[12]" " to get a trace of mallocs and frees.\n" " If LeakSanitizer is enabled in this process it will still\n" @@ -541,107 +607,194 @@ void Fuzzer::TryDetectingAMemoryLeak(con // we don't call it too often. if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it. if (DuringInitialCorpusExecution) Printf("\nINFO: a leak has been found in the initial corpus.\n\n"); Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n"); CurrentUnitSize = Size; DumpCurrentUnit("leak-"); PrintFinalStats(); - _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on. + _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on. } } -static size_t ComputeMutationLen(size_t MaxInputSize, size_t MaxMutationLen, - Random &Rand) { - assert(MaxInputSize <= MaxMutationLen); - if (MaxInputSize == MaxMutationLen) return MaxMutationLen; - size_t Result = MaxInputSize; - size_t R = Rand.Rand(); - if ((R % (1U << 7)) == 0) - Result++; - if ((R % (1U << 15)) == 0) - Result += 10 + Result / 2; - return Min(Result, MaxMutationLen); -} - void Fuzzer::MutateAndTestOne() { MD.StartMutationSequence(); auto &II = Corpus.ChooseUnitToMutate(MD.GetRand()); + if (Options.UseFeatureFrequency) + Corpus.UpdateFeatureFrequencyScore(&II); const auto &U = II.U; memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1)); assert(CurrentUnitData); size_t Size = U.size(); assert(Size <= MaxInputLen && "Oversized Unit"); memcpy(CurrentUnitData, U.data(), Size); assert(MaxMutationLen > 0); size_t CurrentMaxMutationLen = - Options.ExperimentalLenControl - ? ComputeMutationLen(Corpus.MaxInputSize(), MaxMutationLen, - MD.GetRand()) - : MaxMutationLen; + Min(MaxMutationLen, Max(U.size(), TmpMaxMutationLen)); + assert(CurrentMaxMutationLen > 0); for (int i = 0; i < Options.MutateDepth; i++) { if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) break; + MaybeExitGracefully(); size_t NewSize = 0; NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen); assert(NewSize > 0 && "Mutator returned empty unit"); - assert(NewSize <= CurrentMaxMutationLen && "Mutator return overisized unit"); + assert(NewSize <= CurrentMaxMutationLen && "Mutator return oversized unit"); Size = NewSize; - if (i == 0) - StartTraceRecording(); II.NumExecutedMutations++; - if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) { - Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures, - /*MayDeleteFile=*/true); - ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); - CheckExitOnSrcPosOrItem(); - } - StopTraceRecording(); + + bool FoundUniqFeatures = false; + bool NewCov = RunOne(CurrentUnitData, Size, /*MayDeleteFile=*/true, &II, + &FoundUniqFeatures); TryDetectingAMemoryLeak(CurrentUnitData, Size, /*DuringInitialCorpusExecution*/ false); + if (NewCov) { + ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size}); + break; // We will mutate this input more in the next rounds. + } + if (Options.ReduceDepth && !FoundUniqFeatures) + break; } } -void Fuzzer::Loop() { - TPC.InitializePrintNewPCs(); +void Fuzzer::PurgeAllocator() { + if (Options.PurgeAllocatorIntervalSec < 0 || !EF->__sanitizer_purge_allocator) + return; + if (duration_cast<seconds>(system_clock::now() - + LastAllocatorPurgeAttemptTime) + .count() < Options.PurgeAllocatorIntervalSec) + return; + + if (Options.RssLimitMb <= 0 || + GetPeakRSSMb() > static_cast<size_t>(Options.RssLimitMb) / 2) + EF->__sanitizer_purge_allocator(); + + LastAllocatorPurgeAttemptTime = system_clock::now(); +} + +void Fuzzer::ReadAndExecuteSeedCorpora(const Vector<std::string> &CorpusDirs) { + const size_t kMaxSaneLen = 1 << 20; + const size_t kMinDefaultLen = 4096; + Vector<SizedFile> SizedFiles; + size_t MaxSize = 0; + size_t MinSize = -1; + size_t TotalSize = 0; + size_t LastNumFiles = 0; + for (auto &Dir : CorpusDirs) { + GetSizedFilesFromDir(Dir, &SizedFiles); + Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, + Dir.c_str()); + LastNumFiles = SizedFiles.size(); + } + for (auto &File : SizedFiles) { + MaxSize = Max(File.Size, MaxSize); + MinSize = Min(File.Size, MinSize); + TotalSize += File.Size; + } + if (Options.MaxLen == 0) + SetMaxInputLen(std::min(std::max(kMinDefaultLen, MaxSize), kMaxSaneLen)); + assert(MaxInputLen > 0); + + // Test the callback with empty input and never try it again. + uint8_t dummy = 0; + ExecuteCallback(&dummy, 0); + + if (SizedFiles.empty()) { + Printf("INFO: A corpus is not provided, starting from an empty corpus\n"); + Unit U({'\n'}); // Valid ASCII input. + RunOne(U.data(), U.size()); + } else { + Printf("INFO: seed corpus: files: %zd min: %zdb max: %zdb total: %zdb" + " rss: %zdMb\n", + SizedFiles.size(), MinSize, MaxSize, TotalSize, GetPeakRSSMb()); + if (Options.ShuffleAtStartUp) + std::shuffle(SizedFiles.begin(), SizedFiles.end(), MD.GetRand()); + + if (Options.PreferSmall) { + std::stable_sort(SizedFiles.begin(), SizedFiles.end()); + assert(SizedFiles.front().Size <= SizedFiles.back().Size); + } + + // Load and execute inputs one by one. + for (auto &SF : SizedFiles) { + auto U = FileToVector(SF.File, MaxInputLen, /*ExitOnError=*/false); + assert(U.size() <= MaxInputLen); + RunOne(U.data(), U.size()); + CheckExitOnSrcPosOrItem(); + TryDetectingAMemoryLeak(U.data(), U.size(), + /*DuringInitialCorpusExecution*/ true); + } + } + + PrintStats("INITED"); + if (Corpus.empty()) { + Printf("ERROR: no interesting inputs were found. " + "Is the code instrumented for coverage? Exiting.\n"); + exit(1); + } +} + +void Fuzzer::Loop(const Vector<std::string> &CorpusDirs) { + ReadAndExecuteSeedCorpora(CorpusDirs); + TPC.SetPrintNewPCs(Options.PrintNewCovPcs); + TPC.SetPrintNewFuncs(Options.PrintNewCovFuncs); system_clock::time_point LastCorpusReload = system_clock::now(); if (Options.DoCrossOver) MD.SetCorpus(&Corpus); while (true) { auto Now = system_clock::now(); if (duration_cast<seconds>(Now - LastCorpusReload).count() >= Options.ReloadIntervalSec) { RereadOutputCorpus(MaxInputLen); LastCorpusReload = system_clock::now(); } if (TotalNumberOfRuns >= Options.MaxNumberOfRuns) break; - if (TimedOut()) break; + if (TimedOut()) + break; + + // Update TmpMaxMutationLen + if (Options.LenControl) { + if (TmpMaxMutationLen < MaxMutationLen && + TotalNumberOfRuns - LastCorpusUpdateRun > + Options.LenControl * Log(TmpMaxMutationLen)) { + TmpMaxMutationLen = + Min(MaxMutationLen, TmpMaxMutationLen + Log(TmpMaxMutationLen)); + LastCorpusUpdateRun = TotalNumberOfRuns; + } + } else { + TmpMaxMutationLen = MaxMutationLen; + } + // Perform several mutations and runs. MutateAndTestOne(); + + PurgeAllocator(); } PrintStats("DONE ", "\n"); MD.PrintRecommendedDictionary(); } void Fuzzer::MinimizeCrashLoop(const Unit &U) { - if (U.size() <= 1) return; + if (U.size() <= 1) + return; while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) { MD.StartMutationSequence(); memcpy(CurrentUnitData, U.data(), U.size()); for (int i = 0; i < Options.MutateDepth; i++) { size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen); assert(NewSize > 0 && NewSize <= MaxMutationLen); - RunOne(CurrentUnitData, NewSize); + ExecuteCallback(CurrentUnitData, NewSize); + PrintPulseAndReportSlowInput(CurrentUnitData, NewSize); TryDetectingAMemoryLeak(CurrentUnitData, NewSize, /*DuringInitialCorpusExecution*/ false); } } } void Fuzzer::AnnounceOutput(const uint8_t *Data, size_t Size) { if (SMR.IsServer()) { @@ -652,32 +805,35 @@ void Fuzzer::AnnounceOutput(const uint8_ size_t OtherSize = SMR.ReadByteArraySize(); uint8_t *OtherData = SMR.GetByteArray(); if (Size != OtherSize || memcmp(Data, OtherData, Size) != 0) { size_t i = 0; for (i = 0; i < Min(Size, OtherSize); i++) if (Data[i] != OtherData[i]) break; Printf("==%lu== ERROR: libFuzzer: equivalence-mismatch. Sizes: %zd %zd; " - "offset %zd\n", GetPid(), Size, OtherSize, i); + "offset %zd\n", + GetPid(), Size, OtherSize, i); DumpCurrentUnit("mismatch-"); Printf("SUMMARY: libFuzzer: equivalence-mismatch\n"); PrintFinalStats(); _Exit(Options.ErrorExitCode); } } } } // namespace fuzzer extern "C" { -size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { +__attribute__((visibility("default"))) size_t +LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { assert(fuzzer::F); return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); } // Experimental -void LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) { +__attribute__((visibility("default"))) void +LLVMFuzzerAnnounceOutput(const uint8_t *Data, size_t Size) { assert(fuzzer::F); fuzzer::F->AnnounceOutput(Data, Size); } -} // extern "C" +} // extern "C"
--- a/tools/fuzzing/libfuzzer/FuzzerMain.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerMain.cpp @@ -11,11 +11,11 @@ #include "FuzzerDefs.h" extern "C" { // This function should be defined by the user. int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); } // extern "C" -int main(int argc, char **argv) { +__attribute__((visibility("default"))) int main(int argc, char **argv) { return fuzzer::FuzzerDriver(&argc, &argv, LLVMFuzzerTestOneInput); }
--- a/tools/fuzzing/libfuzzer/FuzzerMerge.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.cpp @@ -4,19 +4,20 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Merging corpora. //===----------------------------------------------------------------------===// -#include "FuzzerInternal.h" +#include "FuzzerCommand.h" +#include "FuzzerMerge.h" #include "FuzzerIO.h" -#include "FuzzerMerge.h" +#include "FuzzerInternal.h" #include "FuzzerTracePC.h" #include "FuzzerUtil.h" #include <fstream> #include <iterator> #include <set> #include <sstream> @@ -69,17 +70,17 @@ bool Merger::Parse(std::istream &IS, boo for (size_t i = 0; i < NumFiles; i++) if (!std::getline(IS, Files[i].Name, '\n')) return false; // Parse STARTED and DONE lines. size_t ExpectedStartMarker = 0; const size_t kInvalidStartMarker = -1; size_t LastSeenStartMarker = kInvalidStartMarker; - std::vector<uint32_t> TmpFeatures; + Vector<uint32_t> TmpFeatures; while (std::getline(IS, Line, '\n')) { std::istringstream ISS1(Line); std::string Marker; size_t N; ISS1 >> Marker; ISS1 >> N; if (Marker == "STARTED") { // STARTED FILE_ID FILE_SIZE @@ -117,33 +118,33 @@ size_t Merger::ApproximateMemoryConsumpt size_t Res = 0; for (const auto &F: Files) Res += sizeof(F) + F.Features.size() * sizeof(F.Features[0]); return Res; } // Decides which files need to be merged (add thost to NewFiles). // Returns the number of new features added. -size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures, - std::vector<std::string> *NewFiles) { +size_t Merger::Merge(const Set<uint32_t> &InitialFeatures, + Vector<std::string> *NewFiles) { NewFiles->clear(); assert(NumFilesInFirstCorpus <= Files.size()); - std::set<uint32_t> AllFeatures(InitialFeatures); + Set<uint32_t> AllFeatures(InitialFeatures); // What features are in the initial corpus? for (size_t i = 0; i < NumFilesInFirstCorpus; i++) { auto &Cur = Files[i].Features; AllFeatures.insert(Cur.begin(), Cur.end()); } size_t InitialNumFeatures = AllFeatures.size(); // Remove all features that we already know from all other inputs. for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) { auto &Cur = Files[i].Features; - std::vector<uint32_t> Tmp; + Vector<uint32_t> Tmp; std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(), AllFeatures.end(), std::inserter(Tmp, Tmp.begin())); Cur.swap(Tmp); } // Sort. Give preference to // * smaller files // * files with more features. @@ -173,26 +174,26 @@ void Merger::PrintSummary(std::ostream & OS << std::hex; OS << File.Name << " size: " << File.Size << " features: "; for (auto Feature : File.Features) OS << " " << Feature; OS << "\n"; } } -std::set<uint32_t> Merger::AllFeatures() const { - std::set<uint32_t> S; +Set<uint32_t> Merger::AllFeatures() const { + Set<uint32_t> S; for (auto &File : Files) S.insert(File.Features.begin(), File.Features.end()); return S; } -std::set<uint32_t> Merger::ParseSummary(std::istream &IS) { +Set<uint32_t> Merger::ParseSummary(std::istream &IS) { std::string Line, Tmp; - std::set<uint32_t> Res; + Set<uint32_t> Res; while (std::getline(IS, Line, '\n')) { size_t N; std::istringstream ISS1(Line); ISS1 >> Tmp; // Name ISS1 >> Tmp; // size: assert(Tmp == "size:" && "Corrupt summary file"); ISS1 >> std::hex; ISS1 >> N; // File Size @@ -216,89 +217,139 @@ void Fuzzer::CrashResistantMergeInternal M.LastFailure.c_str()); Printf("MERGE-INNER: %zd total files;" " %zd processed earlier; will process %zd files now\n", M.Files.size(), M.FirstNotProcessedFile, M.Files.size() - M.FirstNotProcessedFile); std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app); + Set<size_t> AllFeatures; for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) { + MaybeExitGracefully(); auto U = FileToVector(M.Files[i].Name); if (U.size() > MaxInputLen) { U.resize(MaxInputLen); U.shrink_to_fit(); } std::ostringstream StartedLine; // Write the pre-run marker. OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; - OF.flush(); // Flush is important since ExecuteCommand may crash. + OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); ExecuteCallback(U.data(), U.size()); - // Collect coverage. - std::set<size_t> Features; - TPC.CollectFeatures([&](size_t Feature) -> bool { - Features.insert(Feature); - return true; + // Collect coverage. We are iterating over the files in this order: + // * First, files in the initial corpus ordered by size, smallest first. + // * Then, all other files, smallest first. + // So it makes no sense to record all features for all files, instead we + // only record features that were not seen before. + Set<size_t> UniqFeatures; + TPC.CollectFeatures([&](size_t Feature) { + if (AllFeatures.insert(Feature).second) + UniqFeatures.insert(Feature); }); // Show stats. - TotalNumberOfRuns++; if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1))) PrintStats("pulse "); // Write the post-run marker and the coverage. OF << "DONE " << i; - for (size_t F : Features) + for (size_t F : UniqFeatures) OF << " " << std::hex << F; OF << "\n"; + OF.flush(); } } -// Outer process. Does not call the target code and thus sohuld not fail. -void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args, - const std::vector<std::string> &Corpora, - const char *CoverageSummaryInputPathOrNull, - const char *CoverageSummaryOutputPathOrNull) { - if (Corpora.size() <= 1) { - Printf("Merge requires two or more corpus dirs\n"); - return; - } - std::vector<std::string> AllFiles; - ListFilesInDirRecursive(Corpora[0], nullptr, &AllFiles, /*TopDir*/true); - size_t NumFilesInFirstCorpus = AllFiles.size(); - for (size_t i = 1; i < Corpora.size(); i++) - ListFilesInDirRecursive(Corpora[i], nullptr, &AllFiles, /*TopDir*/true); - Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", - AllFiles.size(), NumFilesInFirstCorpus); - auto CFPath = DirPlusFile(TmpDir(), - "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - // Write the control file. +static void WriteNewControlFile(const std::string &CFPath, + const Vector<SizedFile> &AllFiles, + size_t NumFilesInFirstCorpus) { RemoveFile(CFPath); std::ofstream ControlFile(CFPath); ControlFile << AllFiles.size() << "\n"; ControlFile << NumFilesInFirstCorpus << "\n"; - for (auto &Path: AllFiles) - ControlFile << Path << "\n"; + for (auto &SF: AllFiles) + ControlFile << SF.File << "\n"; if (!ControlFile) { Printf("MERGE-OUTER: failed to write to the control file: %s\n", CFPath.c_str()); exit(1); } - ControlFile.close(); +} + +// Outer process. Does not call the target code and thus sohuld not fail. +void Fuzzer::CrashResistantMerge(const Vector<std::string> &Args, + const Vector<std::string> &Corpora, + const char *CoverageSummaryInputPathOrNull, + const char *CoverageSummaryOutputPathOrNull, + const char *MergeControlFilePathOrNull) { + if (Corpora.size() <= 1) { + Printf("Merge requires two or more corpus dirs\n"); + return; + } + auto CFPath = + MergeControlFilePathOrNull + ? MergeControlFilePathOrNull + : DirPlusFile(TmpDir(), + "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - // Execute the inner process untill it passes. + size_t NumAttempts = 0; + if (MergeControlFilePathOrNull && FileSize(MergeControlFilePathOrNull)) { + Printf("MERGE-OUTER: non-empty control file provided: '%s'\n", + MergeControlFilePathOrNull); + Merger M; + std::ifstream IF(MergeControlFilePathOrNull); + if (M.Parse(IF, /*ParseCoverage=*/false)) { + Printf("MERGE-OUTER: control file ok, %zd files total," + " first not processed file %zd\n", + M.Files.size(), M.FirstNotProcessedFile); + if (!M.LastFailure.empty()) + Printf("MERGE-OUTER: '%s' will be skipped as unlucky " + "(merge has stumbled on it the last time)\n", + M.LastFailure.c_str()); + if (M.FirstNotProcessedFile >= M.Files.size()) { + Printf("MERGE-OUTER: nothing to do, merge has been completed before\n"); + exit(0); + } + + NumAttempts = M.Files.size() - M.FirstNotProcessedFile; + } else { + Printf("MERGE-OUTER: bad control file, will overwrite it\n"); + } + } + + if (!NumAttempts) { + // The supplied control file is empty or bad, create a fresh one. + Vector<SizedFile> AllFiles; + GetSizedFilesFromDir(Corpora[0], &AllFiles); + size_t NumFilesInFirstCorpus = AllFiles.size(); + std::sort(AllFiles.begin(), AllFiles.end()); + for (size_t i = 1; i < Corpora.size(); i++) + GetSizedFilesFromDir(Corpora[i], &AllFiles); + std::sort(AllFiles.begin() + NumFilesInFirstCorpus, AllFiles.end()); + Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n", + AllFiles.size(), NumFilesInFirstCorpus); + WriteNewControlFile(CFPath, AllFiles, NumFilesInFirstCorpus); + NumAttempts = AllFiles.size(); + } + + // Execute the inner process until it passes. // Every inner process should execute at least one input. - std::string BaseCmd = CloneArgsWithoutX(Args, "keep-all-flags"); + Command BaseCmd(Args); + BaseCmd.removeFlag("merge"); bool Success = false; - for (size_t i = 1; i <= AllFiles.size(); i++) { - Printf("MERGE-OUTER: attempt %zd\n", i); - auto ExitCode = - ExecuteCommand(BaseCmd + " -merge_control_file=" + CFPath); + for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { + MaybeExitGracefully(); + Printf("MERGE-OUTER: attempt %zd\n", Attempt); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); + Cmd.addFlag("merge_inner", "1"); + auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { - Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", i); + Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); Success = true; break; } } if (!Success) { Printf("MERGE-OUTER: zero succesfull attempts, exiting\n"); exit(1); } @@ -313,26 +364,27 @@ void Fuzzer::CrashResistantMerge(const s Printf("MERGE-OUTER: consumed %zdMb (%zdMb rss) to parse the control file\n", M.ApproximateMemoryConsumption() >> 20, GetPeakRSSMb()); if (CoverageSummaryOutputPathOrNull) { Printf("MERGE-OUTER: writing coverage summary for %zd files to %s\n", M.Files.size(), CoverageSummaryOutputPathOrNull); std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull); M.PrintSummary(SummaryOut); } - std::vector<std::string> NewFiles; - std::set<uint32_t> InitialFeatures; + Vector<std::string> NewFiles; + Set<uint32_t> InitialFeatures; if (CoverageSummaryInputPathOrNull) { std::ifstream SummaryIn(CoverageSummaryInputPathOrNull); InitialFeatures = M.ParseSummary(SummaryIn); Printf("MERGE-OUTER: coverage summary loaded from %s, %zd features found\n", CoverageSummaryInputPathOrNull, InitialFeatures.size()); } size_t NumNewFeatures = M.Merge(InitialFeatures, &NewFiles); Printf("MERGE-OUTER: %zd new files with %zd new features added\n", NewFiles.size(), NumNewFeatures); for (auto &F: NewFiles) - WriteToOutputCorpus(FileToVector(F)); - // We are done, delete the control file. - RemoveFile(CFPath); + WriteToOutputCorpus(FileToVector(F, MaxInputLen)); + // We are done, delete the control file if it was a temporary one. + if (!MergeControlFilePathOrNull) + RemoveFile(CFPath); } } // namespace fuzzer
--- a/tools/fuzzing/libfuzzer/FuzzerMerge.h +++ b/tools/fuzzing/libfuzzer/FuzzerMerge.h @@ -47,34 +47,34 @@ #include <set> #include <vector> namespace fuzzer { struct MergeFileInfo { std::string Name; size_t Size = 0; - std::vector<uint32_t> Features; + Vector<uint32_t> Features; }; struct Merger { - std::vector<MergeFileInfo> Files; + Vector<MergeFileInfo> Files; size_t NumFilesInFirstCorpus = 0; size_t FirstNotProcessedFile = 0; std::string LastFailure; bool Parse(std::istream &IS, bool ParseCoverage); bool Parse(const std::string &Str, bool ParseCoverage); void ParseOrExit(std::istream &IS, bool ParseCoverage); void PrintSummary(std::ostream &OS); - std::set<uint32_t> ParseSummary(std::istream &IS); - size_t Merge(const std::set<uint32_t> &InitialFeatures, - std::vector<std::string> *NewFiles); - size_t Merge(std::vector<std::string> *NewFiles) { - return Merge(std::set<uint32_t>{}, NewFiles); + Set<uint32_t> ParseSummary(std::istream &IS); + size_t Merge(const Set<uint32_t> &InitialFeatures, + Vector<std::string> *NewFiles); + size_t Merge(Vector<std::string> *NewFiles) { + return Merge(Set<uint32_t>{}, NewFiles); } size_t ApproximateMemoryConsumption() const; - std::set<uint32_t> AllFeatures() const; + Set<uint32_t> AllFeatures() const; }; } // namespace fuzzer #endif // LLVM_FUZZER_MERGE_H
--- a/tools/fuzzing/libfuzzer/FuzzerMutate.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.cpp @@ -4,21 +4,21 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Mutate a test input. //===----------------------------------------------------------------------===// +#include "FuzzerMutate.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" -#include "FuzzerMutate.h" #include "FuzzerOptions.h" namespace fuzzer { const size_t Dictionary::kMaxDictSize; static void PrintASCII(const Word &W, const char *PrintAfter) { PrintASCII(W.data(), W.size(), PrintAfter); @@ -38,18 +38,16 @@ MutationDispatcher::MutationDispatcher(R {&MutationDispatcher::Mutate_ChangeBit, "ChangeBit"}, {&MutationDispatcher::Mutate_ShuffleBytes, "ShuffleBytes"}, {&MutationDispatcher::Mutate_ChangeASCIIInteger, "ChangeASCIIInt"}, {&MutationDispatcher::Mutate_ChangeBinaryInteger, "ChangeBinInt"}, {&MutationDispatcher::Mutate_CopyPart, "CopyPart"}, {&MutationDispatcher::Mutate_CrossOver, "CrossOver"}, {&MutationDispatcher::Mutate_AddWordFromManualDictionary, "ManualDict"}, - {&MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary, - "TempAutoDict"}, {&MutationDispatcher::Mutate_AddWordFromPersistentAutoDictionary, "PersAutoDict"}, }); if(Options.UseCmp) DefaultMutators.push_back( {&MutationDispatcher::Mutate_AddWordFromTORC, "CMP"}); if (EF->LLVMFuzzerCustomMutator) @@ -59,17 +57,17 @@ MutationDispatcher::MutationDispatcher(R if (EF->LLVMFuzzerCustomCrossOver) Mutators.push_back( {&MutationDispatcher::Mutate_CustomCrossOver, "CustomCrossOver"}); } static char RandCh(Random &Rand) { if (Rand.RandBool()) return Rand(256); - const char *Special = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; + const char Special[] = "!*'();:@&=+$,/?%#[]012Az-`~.\xff\x00"; return Special[Rand(sizeof(Special) - 1)]; } size_t MutationDispatcher::Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize) { return EF->LLVMFuzzerCustomMutator(Data, Size, MaxSize, Rand.Rand()); } @@ -160,21 +158,16 @@ size_t MutationDispatcher::Mutate_Change } size_t MutationDispatcher::Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, size_t MaxSize) { return AddWordFromDictionary(ManualDictionary, Data, Size, MaxSize); } -size_t MutationDispatcher::Mutate_AddWordFromTemporaryAutoDictionary( - uint8_t *Data, size_t Size, size_t MaxSize) { - return AddWordFromDictionary(TempAutoDictionary, Data, Size, MaxSize); -} - size_t MutationDispatcher::ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, DictionaryEntry &DE) { const Word &W = DE.GetW(); bool UsePositionHint = DE.HasPositionHint() && DE.GetPositionHint() + W.size() < Size && Rand.RandBool(); if (Rand.RandBool()) { // Insert W. @@ -246,32 +239,36 @@ DictionaryEntry MutationDispatcher::Make return MakeDictionaryEntryFromCMP(Arg1.data(), Arg2.data(), Arg1.data(), Arg2.data(), Arg1.size(), Data, Size); } size_t MutationDispatcher::Mutate_AddWordFromTORC( uint8_t *Data, size_t Size, size_t MaxSize) { Word W; DictionaryEntry DE; - switch (Rand(3)) { + switch (Rand(4)) { case 0: { auto X = TPC.TORC8.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 1: { auto X = TPC.TORC4.Get(Rand.Rand()); if ((X.A >> 16) == 0 && (X.B >> 16) == 0 && Rand.RandBool()) DE = MakeDictionaryEntryFromCMP((uint16_t)X.A, (uint16_t)X.B, Data, Size); else DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; case 2: { auto X = TPC.TORCW.Get(Rand.Rand()); DE = MakeDictionaryEntryFromCMP(X.A, X.B, Data, Size); } break; + case 3: if (Options.UseMemmem) { + auto X = TPC.MMT.Get(Rand.Rand()); + DE = DictionaryEntry(X); + } break; default: assert(0); } if (!DE.GetW().size()) return 0; Size = ApplyDictionaryEntry(Data, Size, MaxSize, DE); if (!Size) return 0; DictionaryEntry &DERef = CmpDictionaryEntriesDeque[CmpDictionaryEntriesDequeIdx++ % @@ -464,17 +461,17 @@ void MutationDispatcher::RecordSuccessfu assert(DE->GetW().size()); // Linear search is fine here as this happens seldom. if (!PersistentAutoDictionary.ContainsWord(DE->GetW())) PersistentAutoDictionary.push_back({DE->GetW(), 1}); } } void MutationDispatcher::PrintRecommendedDictionary() { - std::vector<DictionaryEntry> V; + Vector<DictionaryEntry> V; for (auto &DE : PersistentAutoDictionary) if (!ManualDictionary.ContainsWord(DE.GetW())) V.push_back(DE); if (V.empty()) return; Printf("###### Recommended dictionary. ######\n"); for (auto &DE: V) { assert(DE.GetW().size()); Printf("\""); @@ -504,17 +501,17 @@ size_t MutationDispatcher::Mutate(uint8_ size_t MutationDispatcher::DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize) { return MutateImpl(Data, Size, MaxSize, DefaultMutators); } // Mutates Data in place, returns new size. size_t MutationDispatcher::MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - const std::vector<Mutator> &Mutators) { + Vector<Mutator> &Mutators) { assert(MaxSize > 0); // Some mutations may fail (e.g. can't insert more bytes if Size == MaxSize), // in which case they will return 0. // Try several times before returning un-mutated data. for (int Iter = 0; Iter < 100; Iter++) { auto M = Mutators[Rand(Mutators.size())]; size_t NewSize = (this->*(M.Fn))(Data, Size, MaxSize); if (NewSize && NewSize <= MaxSize) { @@ -528,19 +525,9 @@ size_t MutationDispatcher::MutateImpl(ui return 1; // Fallback, should not happen frequently. } void MutationDispatcher::AddWordToManualDictionary(const Word &W) { ManualDictionary.push_back( {W, std::numeric_limits<size_t>::max()}); } -void MutationDispatcher::AddWordToAutoDictionary(DictionaryEntry DE) { - static const size_t kMaxAutoDictSize = 1 << 14; - if (TempAutoDictionary.size() >= kMaxAutoDictSize) return; - TempAutoDictionary.push_back(DE); -} - -void MutationDispatcher::ClearAutoDictionary() { - TempAutoDictionary.clear(); -} - } // namespace fuzzer
--- a/tools/fuzzing/libfuzzer/FuzzerMutate.h +++ b/tools/fuzzing/libfuzzer/FuzzerMutate.h @@ -22,17 +22,17 @@ namespace fuzzer { class MutationDispatcher { public: MutationDispatcher(Random &Rand, const FuzzingOptions &Options); ~MutationDispatcher() {} /// Indicate that we are about to start a new sequence of mutations. void StartMutationSequence(); /// Print the current sequence of mutations. void PrintMutationSequence(); - /// Indicate that the current sequence of mutations was successfull. + /// Indicate that the current sequence of mutations was successful. void RecordSuccessfulMutationSequence(); /// Mutates data by invoking user-provided mutator. size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by invoking user-provided crossover. size_t Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by shuffling bytes. size_t Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by erasing bytes. @@ -47,20 +47,16 @@ public: size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by copying/inserting a part of data into a different place. size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by adding a word from the manual dictionary. size_t Mutate_AddWordFromManualDictionary(uint8_t *Data, size_t Size, size_t MaxSize); - /// Mutates data by adding a word from the temporary automatic dictionary. - size_t Mutate_AddWordFromTemporaryAutoDictionary(uint8_t *Data, size_t Size, - size_t MaxSize); - /// Mutates data by adding a word from the TORC. size_t Mutate_AddWordFromTORC(uint8_t *Data, size_t Size, size_t MaxSize); /// Mutates data by adding a word from the persistent automatic dictionary. size_t Mutate_AddWordFromPersistentAutoDictionary(uint8_t *Data, size_t Size, size_t MaxSize); /// Tries to find an ASCII integer in Data, changes it to another ASCII int. @@ -79,35 +75,33 @@ public: size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize); /// Creates a cross-over of two pieces of Data, returns its size. size_t CrossOver(const uint8_t *Data1, size_t Size1, const uint8_t *Data2, size_t Size2, uint8_t *Out, size_t MaxOutSize); void AddWordToManualDictionary(const Word &W); - void AddWordToAutoDictionary(DictionaryEntry DE); - void ClearAutoDictionary(); void PrintRecommendedDictionary(); void SetCorpus(const InputCorpus *Corpus) { this->Corpus = Corpus; } Random &GetRand() { return Rand; } private: struct Mutator { size_t (MutationDispatcher::*Fn)(uint8_t *Data, size_t Size, size_t Max); const char *Name; }; size_t AddWordFromDictionary(Dictionary &D, uint8_t *Data, size_t Size, size_t MaxSize); size_t MutateImpl(uint8_t *Data, size_t Size, size_t MaxSize, - const std::vector<Mutator> &Mutators); + Vector<Mutator> &Mutators); size_t InsertPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize, size_t MaxToSize); size_t CopyPartOf(const uint8_t *From, size_t FromSize, uint8_t *To, size_t ToSize); size_t ApplyDictionaryEntry(uint8_t *Data, size_t Size, size_t MaxSize, DictionaryEntry &DE); @@ -126,31 +120,31 @@ private: const FuzzingOptions Options; // Dictionary provided by the user via -dict=DICT_FILE. Dictionary ManualDictionary; // Temporary dictionary modified by the fuzzer itself, // recreated periodically. Dictionary TempAutoDictionary; // Persistent dictionary modified by the fuzzer, consists of - // entries that led to successfull discoveries in the past mutations. + // entries that led to successful discoveries in the past mutations. Dictionary PersistentAutoDictionary; - std::vector<Mutator> CurrentMutatorSequence; - std::vector<DictionaryEntry *> CurrentDictionaryEntrySequence; + Vector<Mutator> CurrentMutatorSequence; + Vector<DictionaryEntry *> CurrentDictionaryEntrySequence; static const size_t kCmpDictionaryEntriesDequeSize = 16; DictionaryEntry CmpDictionaryEntriesDeque[kCmpDictionaryEntriesDequeSize]; size_t CmpDictionaryEntriesDequeIdx = 0; const InputCorpus *Corpus = nullptr; - std::vector<uint8_t> MutateInPlaceHere; + Vector<uint8_t> MutateInPlaceHere; // CustomCrossOver needs its own buffer as a custom implementation may call // LLVMFuzzerMutate, which in turn may resize MutateInPlaceHere. - std::vector<uint8_t> CustomCrossOverInPlaceHere; + Vector<uint8_t> CustomCrossOverInPlaceHere; - std::vector<Mutator> Mutators; - std::vector<Mutator> DefaultMutators; + Vector<Mutator> Mutators; + Vector<Mutator> DefaultMutators; }; } // namespace fuzzer #endif // LLVM_FUZZER_MUTATE_H
--- a/tools/fuzzing/libfuzzer/FuzzerOptions.h +++ b/tools/fuzzing/libfuzzer/FuzzerOptions.h @@ -13,55 +13,63 @@ #include "FuzzerDefs.h" namespace fuzzer { struct FuzzingOptions { int Verbosity = 1; size_t MaxLen = 0; - bool ExperimentalLenControl = false; + size_t LenControl = 1000; int UnitTimeoutSec = 300; int TimeoutExitCode = 77; int ErrorExitCode = 77; int MaxTotalTimeSec = 0; int RssLimitMb = 0; + int MallocLimitMb = 0; bool DoCrossOver = true; int MutateDepth = 5; + bool ReduceDepth = false; bool UseCounters = false; - bool UseIndirCalls = true; bool UseMemmem = true; bool UseCmp = false; bool UseValueProfile = false; bool Shrink = false; + bool ReduceInputs = false; int ReloadIntervalSec = 1; bool ShuffleAtStartUp = true; bool PreferSmall = true; size_t MaxNumberOfRuns = -1L; int ReportSlowUnits = 10; bool OnlyASCII = false; std::string OutputCorpus; std::string ArtifactPrefix = "./"; std::string ExactArtifactPath; std::string ExitOnSrcPos; std::string ExitOnItem; bool SaveArtifacts = true; bool PrintNEW = true; // Print a status line when new units are found; bool PrintNewCovPcs = false; + int PrintNewCovFuncs = 0; bool PrintFinalStats = false; bool PrintCorpusStats = false; bool PrintCoverage = false; bool DumpCoverage = false; + bool UseClangCoverage = false; bool DetectLeaks = true; + int PurgeAllocatorIntervalSec = 1; + int UseFeatureFrequency = false; int TraceMalloc = 0; bool HandleAbrt = false; bool HandleBus = false; bool HandleFpe = false; bool HandleIll = false; bool HandleInt = false; bool HandleSegv = false; bool HandleTerm = false; bool HandleXfsz = false; + bool HandleUsr1 = false; + bool HandleUsr2 = false; }; } // namespace fuzzer #endif // LLVM_FUZZER_OPTIONS_H
new file mode 100644 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerShmemFuchsia.cpp @@ -0,0 +1,38 @@ +//===- FuzzerShmemPosix.cpp - Posix shared memory ---------------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// SharedMemoryRegion. For Fuchsia, this is just stubs as equivalence servers +// are not currently supported. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" + +#if LIBFUZZER_FUCHSIA + +#include "FuzzerShmem.h" + +namespace fuzzer { + +bool SharedMemoryRegion::Create(const char *Name) { + return false; +} + +bool SharedMemoryRegion::Open(const char *Name) { + return false; +} + +bool SharedMemoryRegion::Destroy(const char *Name) { + return false; +} + +void SharedMemoryRegion::Post(int Idx) {} + +void SharedMemoryRegion::Wait(int Idx) {} + +} // namespace fuzzer + +#endif // LIBFUZZER_FUCHSIA
--- a/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerShmemPosix.cpp @@ -9,24 +9,24 @@ // SharedMemoryRegion //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_POSIX #include "FuzzerIO.h" #include "FuzzerShmem.h" -#include <sys/types.h> -#include <sys/stat.h> #include <errno.h> #include <fcntl.h> -#include <sys/mman.h> #include <semaphore.h> #include <stdio.h> #include <stdlib.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> #include <unistd.h> namespace fuzzer { std::string SharedMemoryRegion::Path(const char *Name) { return DirPlusFile(TmpDir(), Name); }
--- a/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerShmemWindows.cpp @@ -9,20 +9,20 @@ // SharedMemoryRegion //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_WINDOWS #include "FuzzerIO.h" #include "FuzzerShmem.h" -#include <sys/types.h> -#include <sys/stat.h> #include <fcntl.h> #include <stdio.h> +#include <sys/stat.h> +#include <sys/types.h> namespace fuzzer { std::string SharedMemoryRegion::Path(const char *Name) { return DirPlusFile(TmpDir(), Name); } std::string SharedMemoryRegion::SemName(const char *Name, int Idx) {
--- a/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.cpp @@ -7,57 +7,83 @@ // //===----------------------------------------------------------------------===// // Trace PCs. // This module implements __sanitizer_cov_trace_pc_guard[_init], // the callback required for -fsanitize-coverage=trace-pc-guard instrumentation. // //===----------------------------------------------------------------------===// +#include "FuzzerTracePC.h" #include "FuzzerCorpus.h" #include "FuzzerDefs.h" #include "FuzzerDictionary.h" #include "FuzzerExtFunctions.h" #include "FuzzerIO.h" -#include "FuzzerTracePC.h" #include "FuzzerUtil.h" #include "FuzzerValueBitMap.h" -#include <map> #include <set> -#include <sstream> // The coverage counters and PCs. // These are declared as global variables named "__sancov_*" to simplify // experiments with inlined instrumentation. alignas(64) ATTRIBUTE_INTERFACE uint8_t __sancov_trace_pc_guard_8bit_counters[fuzzer::TracePC::kNumPCs]; ATTRIBUTE_INTERFACE uintptr_t __sancov_trace_pc_pcs[fuzzer::TracePC::kNumPCs]; +// Used by -fsanitize-coverage=stack-depth to track stack depth +ATTRIBUTE_INTERFACE __attribute__((tls_model("initial-exec"))) +thread_local uintptr_t __sancov_lowest_stack; + namespace fuzzer { TracePC TPC; +int ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr; + uint8_t *TracePC::Counters() const { return __sancov_trace_pc_guard_8bit_counters; } uintptr_t *TracePC::PCs() const { return __sancov_trace_pc_pcs; } size_t TracePC::GetTotalPCCoverage() { + if (ObservedPCs.size()) + return ObservedPCs.size(); size_t Res = 0; for (size_t i = 1, N = GetNumPCs(); i < N; i++) if (PCs()[i]) Res++; return Res; } + +void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) { + if (Start == Stop) return; + if (NumModulesWithInline8bitCounters && + ModuleCounters[NumModulesWithInline8bitCounters-1].Start == Start) return; + assert(NumModulesWithInline8bitCounters < + sizeof(ModuleCounters) / sizeof(ModuleCounters[0])); + ModuleCounters[NumModulesWithInline8bitCounters++] = {Start, Stop}; + NumInline8bitCounters += Stop - Start; +} + +void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) { + const PCTableEntry *B = reinterpret_cast<const PCTableEntry *>(Start); + const PCTableEntry *E = reinterpret_cast<const PCTableEntry *>(Stop); + if (NumPCTables && ModulePCTable[NumPCTables - 1].Start == B) return; + assert(NumPCTables < sizeof(ModulePCTable) / sizeof(ModulePCTable[0])); + ModulePCTable[NumPCTables++] = {B, E}; + NumPCsInPCTables += E - B; +} + void TracePC::HandleInit(uint32_t *Start, uint32_t *Stop) { if (Start == Stop || *Start) return; assert(NumModules < sizeof(Modules) / sizeof(Modules[0])); for (uint32_t *P = Start; P < Stop; P++) { NumGuards++; if (NumGuards == kNumPCs) { RawPrint( "WARNING: The binary has too many instrumented PCs.\n" @@ -67,161 +93,197 @@ void TracePC::HandleInit(uint32_t *Start *P = NumGuards % kNumPCs; } Modules[NumModules].Start = Start; Modules[NumModules].Stop = Stop; NumModules++; } void TracePC::PrintModuleInfo() { - Printf("INFO: Loaded %zd modules (%zd guards): ", NumModules, NumGuards); - for (size_t i = 0; i < NumModules; i++) - Printf("[%p, %p), ", Modules[i].Start, Modules[i].Stop); - Printf("\n"); + if (NumGuards) { + Printf("INFO: Loaded %zd modules (%zd guards): ", NumModules, NumGuards); + for (size_t i = 0; i < NumModules; i++) + Printf("%zd [%p, %p), ", Modules[i].Stop - Modules[i].Start, + Modules[i].Start, Modules[i].Stop); + Printf("\n"); + } + if (NumModulesWithInline8bitCounters) { + Printf("INFO: Loaded %zd modules (%zd inline 8-bit counters): ", + NumModulesWithInline8bitCounters, NumInline8bitCounters); + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) + Printf("%zd [%p, %p), ", ModuleCounters[i].Stop - ModuleCounters[i].Start, + ModuleCounters[i].Start, ModuleCounters[i].Stop); + Printf("\n"); + } + if (NumPCTables) { + Printf("INFO: Loaded %zd PC tables (%zd PCs): ", NumPCTables, + NumPCsInPCTables); + for (size_t i = 0; i < NumPCTables; i++) { + Printf("%zd [%p,%p), ", ModulePCTable[i].Stop - ModulePCTable[i].Start, + ModulePCTable[i].Start, ModulePCTable[i].Stop); + } + Printf("\n"); + + if ((NumGuards && NumGuards != NumPCsInPCTables) || + (NumInline8bitCounters && NumInline8bitCounters != NumPCsInPCTables)) { + Printf("ERROR: The size of coverage PC tables does not match the\n" + "number of instrumented PCs. This might be a compiler bug,\n" + "please contact the libFuzzer developers.\n" + "Also check https://bugs.llvm.org/show_bug.cgi?id=34636\n" + "for possible workarounds (tl;dr: don't use the old GNU ld)\n"); + _Exit(1); + } + } + if (size_t NumClangCounters = ClangCountersEnd() - ClangCountersBegin()) + Printf("INFO: %zd Clang Coverage Counters\n", NumClangCounters); } ATTRIBUTE_NO_SANITIZE_ALL void TracePC::HandleCallerCallee(uintptr_t Caller, uintptr_t Callee) { const uintptr_t kBits = 12; const uintptr_t kMask = (1 << kBits) - 1; uintptr_t Idx = (Caller & kMask) | ((Callee & kMask) << kBits); ValueProfileMap.AddValueModPrime(Idx); } -void TracePC::InitializePrintNewPCs() { - if (!DoPrintNewPCs) return; - assert(!PrintedPCs); - PrintedPCs = new std::set<uintptr_t>; - for (size_t i = 1; i < GetNumPCs(); i++) - if (PCs()[i]) - PrintedPCs->insert(PCs()[i]); +void TracePC::UpdateObservedPCs() { + Vector<uintptr_t> CoveredFuncs; + auto ObservePC = [&](uintptr_t PC) { + if (ObservedPCs.insert(PC).second && DoPrintNewPCs) + PrintPC("\tNEW_PC: %p %F %L\n", "\tNEW_PC: %p\n", PC + 1); + }; + + auto Observe = [&](const PCTableEntry &TE) { + if (TE.PCFlags & 1) + if (ObservedFuncs.insert(TE.PC).second && NumPrintNewFuncs) + CoveredFuncs.push_back(TE.PC); + ObservePC(TE.PC); + }; + + if (NumPCsInPCTables) { + if (NumInline8bitCounters == NumPCsInPCTables) { + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { + uint8_t *Beg = ModuleCounters[i].Start; + size_t Size = ModuleCounters[i].Stop - Beg; + assert(Size == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t j = 0; j < Size; j++) + if (Beg[j]) + Observe(ModulePCTable[i].Start[j]); + } + } else if (NumGuards == NumPCsInPCTables) { + size_t GuardIdx = 1; + for (size_t i = 0; i < NumModules; i++) { + uint32_t *Beg = Modules[i].Start; + size_t Size = Modules[i].Stop - Beg; + assert(Size == + (size_t)(ModulePCTable[i].Stop - ModulePCTable[i].Start)); + for (size_t j = 0; j < Size; j++, GuardIdx++) + if (Counters()[GuardIdx]) + Observe(ModulePCTable[i].Start[j]); + } + } + } + if (size_t NumClangCounters = + ClangCountersEnd() - ClangCountersBegin()) { + auto P = ClangCountersBegin(); + for (size_t Idx = 0; Idx < NumClangCounters; Idx++) + if (P[Idx]) + ObservePC((uintptr_t)Idx); + } + + for (size_t i = 0, N = Min(CoveredFuncs.size(), NumPrintNewFuncs); i < N; i++) { + Printf("\tNEW_FUNC[%zd/%zd]: ", i, CoveredFuncs.size()); + PrintPC("%p %F %L\n", "%p\n", CoveredFuncs[i] + 1); + } } -void TracePC::PrintNewPCs() { - if (!DoPrintNewPCs) return; - assert(PrintedPCs); - for (size_t i = 1; i < GetNumPCs(); i++) - if (PCs()[i] && PrintedPCs->insert(PCs()[i]).second) - PrintPC("\tNEW_PC: %p %F %L\n", "\tNEW_PC: %p\n", PCs()[i]); +inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { + // TODO: this implementation is x86 only. + // see sanitizer_common GetPreviousInstructionPc for full implementation. + return PC - 1; +} + +inline ALWAYS_INLINE uintptr_t GetNextInstructionPc(uintptr_t PC) { + // TODO: this implementation is x86 only. + // see sanitizer_common GetPreviousInstructionPc for full implementation. + return PC + 1; +} + +static std::string GetModuleName(uintptr_t PC) { + char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? + void *OffsetRaw = nullptr; + if (!EF->__sanitizer_get_module_and_offset_for_pc( + reinterpret_cast<void *>(PC), ModulePathRaw, + sizeof(ModulePathRaw), &OffsetRaw)) + return ""; + return ModulePathRaw; } void TracePC::PrintCoverage() { if (!EF->__sanitizer_symbolize_pc || !EF->__sanitizer_get_module_and_offset_for_pc) { Printf("INFO: __sanitizer_symbolize_pc or " "__sanitizer_get_module_and_offset_for_pc is not available," " not printing coverage\n"); return; } - std::map<std::string, std::vector<uintptr_t>> CoveredPCsPerModule; - std::map<std::string, uintptr_t> ModuleOffsets; - std::set<std::string> CoveredDirs, CoveredFiles, CoveredFunctions, - CoveredLines; Printf("COVERAGE:\n"); - for (size_t i = 1; i < GetNumPCs(); i++) { - uintptr_t PC = PCs()[i]; - if (!PC) continue; - std::string FileStr = DescribePC("%s", PC); - if (!IsInterestingCoverageFile(FileStr)) continue; - std::string FixedPCStr = DescribePC("%p", PC); - std::string FunctionStr = DescribePC("%F", PC); - std::string LineStr = DescribePC("%l", PC); - char ModulePathRaw[4096] = ""; // What's PATH_MAX in portable C++? - void *OffsetRaw = nullptr; - if (!EF->__sanitizer_get_module_and_offset_for_pc( - reinterpret_cast<void *>(PC), ModulePathRaw, - sizeof(ModulePathRaw), &OffsetRaw)) - continue; - std::string Module = ModulePathRaw; - uintptr_t FixedPC = std::stoull(FixedPCStr, 0, 16); - uintptr_t PcOffset = reinterpret_cast<uintptr_t>(OffsetRaw); - ModuleOffsets[Module] = FixedPC - PcOffset; - CoveredPCsPerModule[Module].push_back(PcOffset); - CoveredFunctions.insert(FunctionStr); - CoveredFiles.insert(FileStr); - CoveredDirs.insert(DirName(FileStr)); - if (!CoveredLines.insert(FileStr + ":" + LineStr).second) - continue; - Printf("COVERED: %s %s:%s\n", FunctionStr.c_str(), - FileStr.c_str(), LineStr.c_str()); - } + std::string LastFunctionName = ""; + std::string LastFileStr = ""; + Set<size_t> UncoveredLines; + Set<size_t> CoveredLines; - std::string CoveredDirsStr; - for (auto &Dir : CoveredDirs) { - if (!CoveredDirsStr.empty()) - CoveredDirsStr += ","; - CoveredDirsStr += Dir; - } - Printf("COVERED_DIRS: %s\n", CoveredDirsStr.c_str()); + auto FunctionEndCallback = [&](const std::string &CurrentFunc, + const std::string &CurrentFile) { + if (LastFunctionName != CurrentFunc) { + if (CoveredLines.empty() && !UncoveredLines.empty()) { + Printf("UNCOVERED_FUNC: %s\n", LastFunctionName.c_str()); + } else { + for (auto Line : UncoveredLines) { + if (!CoveredLines.count(Line)) + Printf("UNCOVERED_LINE: %s %s:%zd\n", LastFunctionName.c_str(), + LastFileStr.c_str(), Line); + } + } - for (auto &M : CoveredPCsPerModule) { - std::set<std::string> UncoveredFiles, UncoveredFunctions; - std::map<std::string, std::set<int> > UncoveredLines; // Func+File => lines - auto &ModuleName = M.first; - auto &CoveredOffsets = M.second; - uintptr_t ModuleOffset = ModuleOffsets[ModuleName]; - std::sort(CoveredOffsets.begin(), CoveredOffsets.end()); - Printf("MODULE_WITH_COVERAGE: %s\n", ModuleName.c_str()); - // sancov does not yet fully support DSOs. - // std::string Cmd = "sancov -print-coverage-pcs " + ModuleName; - std::string Cmd = DisassembleCmd(ModuleName) + " | " + - SearchRegexCmd("call.*__sanitizer_cov_trace_pc_guard"); - std::string SanCovOutput; - if (!ExecuteCommandAndReadOutput(Cmd, &SanCovOutput)) { - Printf("INFO: Command failed: %s\n", Cmd.c_str()); - continue; + UncoveredLines.clear(); + CoveredLines.clear(); + LastFunctionName = CurrentFunc; + LastFileStr = CurrentFile; } - std::istringstream ISS(SanCovOutput); - std::string S; - while (std::getline(ISS, S, '\n')) { - size_t PcOffsetEnd = S.find(':'); - if (PcOffsetEnd == std::string::npos) - continue; - S.resize(PcOffsetEnd); - uintptr_t PcOffset = std::stoull(S, 0, 16); - if (!std::binary_search(CoveredOffsets.begin(), CoveredOffsets.end(), - PcOffset)) { - uintptr_t PC = ModuleOffset + PcOffset; - auto FileStr = DescribePC("%s", PC); - if (!IsInterestingCoverageFile(FileStr)) continue; - if (CoveredFiles.count(FileStr) == 0) { - UncoveredFiles.insert(FileStr); - continue; - } - auto FunctionStr = DescribePC("%F", PC); - if (CoveredFunctions.count(FunctionStr) == 0) { - UncoveredFunctions.insert(FunctionStr); - continue; - } - std::string LineStr = DescribePC("%l", PC); - uintptr_t Line = std::stoi(LineStr); - std::string FileLineStr = FileStr + ":" + LineStr; - if (CoveredLines.count(FileLineStr) == 0) - UncoveredLines[FunctionStr + " " + FileStr].insert(Line); - } + }; + + for (size_t i = 0; i < NumPCTables; i++) { + auto &M = ModulePCTable[i]; + assert(M.Start < M.Stop); + auto ModuleName = GetModuleName(M.Start->PC); + for (auto Ptr = M.Start; Ptr < M.Stop; Ptr++) { + auto PC = Ptr->PC; + auto VisualizePC = GetNextInstructionPc(PC); + bool IsObserved = ObservedPCs.count(PC); + std::string FileStr = DescribePC("%s", VisualizePC); + if (!IsInterestingCoverageFile(FileStr)) continue; + std::string FunctionStr = DescribePC("%F", VisualizePC); + FunctionEndCallback(FunctionStr, FileStr); + std::string LineStr = DescribePC("%l", VisualizePC); + size_t Line = std::stoul(LineStr); + if (IsObserved && CoveredLines.insert(Line).second) + Printf("COVERED: %s %s:%zd\n", FunctionStr.c_str(), FileStr.c_str(), + Line); + else + UncoveredLines.insert(Line); } - for (auto &FileLine: UncoveredLines) - for (int Line : FileLine.second) - Printf("UNCOVERED_LINE: %s:%d\n", FileLine.first.c_str(), Line); - for (auto &Func : UncoveredFunctions) - Printf("UNCOVERED_FUNC: %s\n", Func.c_str()); - for (auto &File : UncoveredFiles) - Printf("UNCOVERED_FILE: %s\n", File.c_str()); } -} - -inline ALWAYS_INLINE uintptr_t GetPreviousInstructionPc(uintptr_t PC) { - // TODO: this implementation is x86 only. - // see sanitizer_common GetPreviousInstructionPc for full implementation. - return PC - 1; + FunctionEndCallback("", ""); } void TracePC::DumpCoverage() { if (EF->__sanitizer_dump_coverage) { - std::vector<uintptr_t> PCsCopy(GetNumPCs()); + Vector<uintptr_t> PCsCopy(GetNumPCs()); for (size_t i = 0; i < GetNumPCs(); i++) PCsCopy[i] = PCs()[i] ? GetPreviousInstructionPc(PCs()[i]) : 0; EF->__sanitizer_dump_coverage(PCsCopy.data(), PCsCopy.size()); } } // Value profile. // We keep track of various values that affect control flow. @@ -270,16 +332,48 @@ void TracePC::HandleCmp(uintptr_t PC, T uintptr_t Idx = ((PC & 4095) + 1) * ArgDistance; if (sizeof(T) == 4) TORC4.Insert(ArgXor, Arg1, Arg2); else if (sizeof(T) == 8) TORC8.Insert(ArgXor, Arg1, Arg2); ValueProfileMap.AddValue(Idx); } +static size_t InternalStrnlen(const char *S, size_t MaxLen) { + size_t Len = 0; + for (; Len < MaxLen && S[Len]; Len++) {} + return Len; +} + +// Finds min of (strlen(S1), strlen(S2)). +// Needed bacause one of these strings may actually be non-zero terminated. +static size_t InternalStrnlen2(const char *S1, const char *S2) { + size_t Len = 0; + for (; S1[Len] && S2[Len]; Len++) {} + return Len; +} + +void TracePC::ClearInlineCounters() { + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { + uint8_t *Beg = ModuleCounters[i].Start; + size_t Size = ModuleCounters[i].Stop - Beg; + memset(Beg, 0, Size); + } +} + +ATTRIBUTE_NO_SANITIZE_ALL +void TracePC::RecordInitialStack() { + int stack; + __sancov_lowest_stack = InitialStack = reinterpret_cast<uintptr_t>(&stack); +} + +uintptr_t TracePC::GetMaxStackOffset() const { + return InitialStack - __sancov_lowest_stack; // Stack grows down +} + } // namespace fuzzer extern "C" { ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_guard(uint32_t *Guard) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); uint32_t Idx = *Guard; @@ -299,16 +393,27 @@ void __sanitizer_cov_trace_pc() { } ATTRIBUTE_INTERFACE void __sanitizer_cov_trace_pc_guard_init(uint32_t *Start, uint32_t *Stop) { fuzzer::TPC.HandleInit(Start, Stop); } ATTRIBUTE_INTERFACE +void __sanitizer_cov_8bit_counters_init(uint8_t *Start, uint8_t *Stop) { + fuzzer::TPC.HandleInline8bitCountersInit(Start, Stop); +} + +ATTRIBUTE_INTERFACE +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + fuzzer::TPC.HandlePCsInit(pcs_beg, pcs_end); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL void __sanitizer_cov_trace_pc_indir(uintptr_t Callee) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCallerCallee(PC, Callee); } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL @@ -316,40 +421,75 @@ ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp8(uint64_t Arg1, uint64_t Arg2) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT +// Now the __sanitizer_cov_trace_const_cmp[1248] callbacks just mimic +// the behaviour of __sanitizer_cov_trace_cmp[1248] ones. This, however, +// should be changed later to make full use of instrumentation. +void __sanitizer_cov_trace_const_cmp8(uint64_t Arg1, uint64_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp4(uint32_t Arg1, uint32_t Arg2) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp4(uint32_t Arg1, uint32_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp2(uint16_t Arg1, uint16_t Arg2) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp2(uint16_t Arg1, uint16_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_cmp1(uint8_t Arg1, uint8_t Arg2) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); } ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT +void __sanitizer_cov_trace_const_cmp1(uint8_t Arg1, uint8_t Arg2) { + uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); + fuzzer::TPC.HandleCmp(PC, Arg1, Arg2); +} + +ATTRIBUTE_INTERFACE +ATTRIBUTE_NO_SANITIZE_ALL +ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_switch(uint64_t Val, uint64_t *Cases) { uint64_t N = Cases[0]; uint64_t ValSizeInBits = Cases[1]; uint64_t *Vals = Cases + 2; // Skip the most common and the most boring case. if (Vals[N - 1] < 256 && Val < 256) return; uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); @@ -387,9 +527,76 @@ void __sanitizer_cov_trace_div8(uint64_t ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_ALL ATTRIBUTE_TARGET_POPCNT void __sanitizer_cov_trace_gep(uintptr_t Idx) { uintptr_t PC = reinterpret_cast<uintptr_t>(__builtin_return_address(0)); fuzzer::TPC.HandleCmp(PC, Idx, (uintptr_t)0); } + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, + const void *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, + const char *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + size_t Len1 = fuzzer::InternalStrnlen(s1, n); + size_t Len2 = fuzzer::InternalStrnlen(s2, n); + n = std::min(n, Len1); + n = std::min(n, Len2); + if (n <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, + const char *s2, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + if (result == 0) return; // No reason to mutate. + size_t N = fuzzer::InternalStrnlen2(s1, s2); + if (N <= 1) return; // Not interesting. + fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); +} + +ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY +void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, + const void *s2, size_t len2, void *result) { + if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; + fuzzer::TPC.MMT.Add(reinterpret_cast<const uint8_t *>(s2), len2); +} } // extern "C"
--- a/tools/fuzzing/libfuzzer/FuzzerTracePC.h +++ b/tools/fuzzing/libfuzzer/FuzzerTracePC.h @@ -40,123 +40,258 @@ struct TableOfRecentCompares { Table[Idx].B = Arg2; } Pair Get(size_t I) { return Table[I % kSize]; } Pair Table[kSize]; }; +template <size_t kSizeT> +struct MemMemTable { + static const size_t kSize = kSizeT; + Word MemMemWords[kSize]; + Word EmptyWord; + + void Add(const uint8_t *Data, size_t Size) { + if (Size <= 2) return; + Size = std::min(Size, Word::GetMaxSize()); + size_t Idx = SimpleFastHash(Data, Size) % kSize; + MemMemWords[Idx].Set(Data, Size); + } + const Word &Get(size_t Idx) { + for (size_t i = 0; i < kSize; i++) { + const Word &W = MemMemWords[(Idx + i) % kSize]; + if (W.size()) return W; + } + EmptyWord.Set(nullptr, 0); + return EmptyWord; + } +}; + class TracePC { public: static const size_t kNumPCs = 1 << 21; // How many bits of PC are used from __sanitizer_cov_trace_pc. static const size_t kTracePcBits = 18; - void HandleInit(uint32_t *start, uint32_t *stop); + void HandleInit(uint32_t *Start, uint32_t *Stop); + void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop); + void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop); void HandleCallerCallee(uintptr_t Caller, uintptr_t Callee); template <class T> void HandleCmp(uintptr_t PC, T Arg1, T Arg2); size_t GetTotalPCCoverage(); void SetUseCounters(bool UC) { UseCounters = UC; } + void SetUseClangCoverage(bool UCC) { UseClangCoverage = UCC; } void SetUseValueProfile(bool VP) { UseValueProfile = VP; } void SetPrintNewPCs(bool P) { DoPrintNewPCs = P; } + void SetPrintNewFuncs(size_t P) { NumPrintNewFuncs = P; } + void UpdateObservedPCs(); template <class Callback> void CollectFeatures(Callback CB) const; void ResetMaps() { ValueProfileMap.Reset(); - memset(Counters(), 0, GetNumPCs()); + if (NumModules) + memset(Counters(), 0, GetNumPCs()); ClearExtraCounters(); + ClearInlineCounters(); + if (UseClangCoverage) + ClearClangCounters(); } + void ClearInlineCounters(); + void UpdateFeatureSet(size_t CurrentElementIdx, size_t CurrentElementSize); void PrintFeatureSet(); void PrintModuleInfo(); void PrintCoverage(); void DumpCoverage(); void AddValueForMemcmp(void *caller_pc, const void *s1, const void *s2, size_t n, bool StopAtZero); TableOfRecentCompares<uint32_t, 32> TORC4; TableOfRecentCompares<uint64_t, 32> TORC8; TableOfRecentCompares<Word, 32> TORCW; + MemMemTable<1024> MMT; - void PrintNewPCs(); - void InitializePrintNewPCs(); size_t GetNumPCs() const { return NumGuards == 0 ? (1 << kTracePcBits) : Min(kNumPCs, NumGuards + 1); } uintptr_t GetPC(size_t Idx) { assert(Idx < GetNumPCs()); return PCs()[Idx]; } + void RecordInitialStack(); + uintptr_t GetMaxStackOffset() const; + + template<class CallBack> + void ForEachObservedPC(CallBack CB) { + for (auto PC : ObservedPCs) + CB(PC); + } + private: bool UseCounters = false; bool UseValueProfile = false; + bool UseClangCoverage = false; bool DoPrintNewPCs = false; + size_t NumPrintNewFuncs = 0; struct Module { uint32_t *Start, *Stop; }; Module Modules[4096]; size_t NumModules; // linker-initialized. size_t NumGuards; // linker-initialized. + struct { uint8_t *Start, *Stop; } ModuleCounters[4096]; + size_t NumModulesWithInline8bitCounters; // linker-initialized. + size_t NumInline8bitCounters; + + struct PCTableEntry { + uintptr_t PC, PCFlags; + }; + + struct { const PCTableEntry *Start, *Stop; } ModulePCTable[4096]; + size_t NumPCTables; + size_t NumPCsInPCTables; + uint8_t *Counters() const; uintptr_t *PCs() const; - std::set<uintptr_t> *PrintedPCs; + Set<uintptr_t> ObservedPCs; + Set<uintptr_t> ObservedFuncs; ValueBitMap ValueProfileMap; + uintptr_t InitialStack; }; -template <class Callback> // void Callback(size_t Idx, uint8_t Value); +template <class Callback> +// void Callback(size_t FirstFeature, size_t Idx, uint8_t Value); ATTRIBUTE_NO_SANITIZE_ALL void ForEachNonZeroByte(const uint8_t *Begin, const uint8_t *End, size_t FirstFeature, Callback Handle8bitCounter) { typedef uintptr_t LargeType; const size_t Step = sizeof(LargeType) / sizeof(uint8_t); - assert(!(reinterpret_cast<uintptr_t>(Begin) % 64)); - for (auto P = Begin; P < End; P += Step) + const size_t StepMask = Step - 1; + auto P = Begin; + // Iterate by 1 byte until either the alignment boundary or the end. + for (; reinterpret_cast<uintptr_t>(P) & StepMask && P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); + + // Iterate by Step bytes at a time. + for (; P < End; P += Step) if (LargeType Bundle = *reinterpret_cast<const LargeType *>(P)) for (size_t I = 0; I < Step; I++, Bundle >>= 8) if (uint8_t V = Bundle & 0xff) - Handle8bitCounter(FirstFeature + P - Begin + I, V); + Handle8bitCounter(FirstFeature, P - Begin + I, V); + + // Iterate by 1 byte until the end. + for (; P < End; P++) + if (uint8_t V = *P) + Handle8bitCounter(FirstFeature, P - Begin, V); } -template <class Callback> // bool Callback(size_t Feature) -ATTRIBUTE_NO_SANITIZE_ALL -__attribute__((noinline)) -void TracePC::CollectFeatures(Callback HandleFeature) const { - uint8_t *Counters = this->Counters(); - size_t N = GetNumPCs(); - auto Handle8bitCounter = [&](size_t Idx, uint8_t Counter) { +// Given a non-zero Counter returns a number in the range [0,7]. +template<class T> +unsigned CounterToFeature(T Counter) { + // Returns a feature number by placing Counters into buckets as illustrated + // below. + // + // Counter bucket: [1] [2] [3] [4-7] [8-15] [16-31] [32-127] [128+] + // Feature number: 0 1 2 3 4 5 6 7 + // + // This is a heuristic taken from AFL (see + // http://lcamtuf.coredump.cx/afl/technical_details.txt). + // + // This implementation may change in the future so clients should + // not rely on it. assert(Counter); unsigned Bit = 0; /**/ if (Counter >= 128) Bit = 7; else if (Counter >= 32) Bit = 6; else if (Counter >= 16) Bit = 5; else if (Counter >= 8) Bit = 4; else if (Counter >= 4) Bit = 3; else if (Counter >= 3) Bit = 2; else if (Counter >= 2) Bit = 1; - HandleFeature(Idx * 8 + Bit); + return Bit; +} + +template <class Callback> // void Callback(size_t Feature) +ATTRIBUTE_NO_SANITIZE_ADDRESS +__attribute__((noinline)) +void TracePC::CollectFeatures(Callback HandleFeature) const { + uint8_t *Counters = this->Counters(); + size_t N = GetNumPCs(); + auto Handle8bitCounter = [&](size_t FirstFeature, + size_t Idx, uint8_t Counter) { + if (UseCounters) + HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Counter)); + else + HandleFeature(FirstFeature + Idx); }; - ForEachNonZeroByte(Counters, Counters + N, 0, Handle8bitCounter); - ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), N * 8, - Handle8bitCounter); + size_t FirstFeature = 0; + + if (!NumInline8bitCounters) { + ForEachNonZeroByte(Counters, Counters + N, FirstFeature, Handle8bitCounter); + FirstFeature += N * 8; + } + + if (NumInline8bitCounters) { + for (size_t i = 0; i < NumModulesWithInline8bitCounters; i++) { + ForEachNonZeroByte(ModuleCounters[i].Start, ModuleCounters[i].Stop, + FirstFeature, Handle8bitCounter); + FirstFeature += 8 * (ModuleCounters[i].Stop - ModuleCounters[i].Start); + } + } - if (UseValueProfile) + if (size_t NumClangCounters = ClangCountersEnd() - ClangCountersBegin()) { + auto P = ClangCountersBegin(); + for (size_t Idx = 0; Idx < NumClangCounters; Idx++) + if (auto Cnt = P[Idx]) { + if (UseCounters) + HandleFeature(FirstFeature + Idx * 8 + CounterToFeature(Cnt)); + else + HandleFeature(FirstFeature + Idx); + } + FirstFeature += NumClangCounters; + } + + ForEachNonZeroByte(ExtraCountersBegin(), ExtraCountersEnd(), FirstFeature, + Handle8bitCounter); + FirstFeature += (ExtraCountersEnd() - ExtraCountersBegin()) * 8; + + if (UseValueProfile) { ValueProfileMap.ForEach([&](size_t Idx) { - HandleFeature(N * 8 + Idx); + HandleFeature(FirstFeature + Idx); }); + FirstFeature += ValueProfileMap.SizeInBits(); + } + + // Step function, grows similar to 8 * Log_2(A). + auto StackDepthStepFunction = [](uint32_t A) -> uint32_t { + if (!A) return A; + uint32_t Log2 = Log(A); + if (Log2 < 3) return A; + Log2 -= 3; + return (Log2 + 1) * 8 + ((A >> Log2) & 7); + }; + assert(StackDepthStepFunction(1024) == 64); + assert(StackDepthStepFunction(1024 * 4) == 80); + assert(StackDepthStepFunction(1024 * 1024) == 144); + + if (auto MaxStackOffset = GetMaxStackOffset()) + HandleFeature(FirstFeature + StackDepthStepFunction(MaxStackOffset / 8)); } extern TracePC TPC; } // namespace fuzzer #endif // LLVM_FUZZER_TRACE_PC
deleted file mode 100644 --- a/tools/fuzzing/libfuzzer/FuzzerTraceState.cpp +++ /dev/null @@ -1,181 +0,0 @@ -//===- FuzzerTraceState.cpp - Trace-based fuzzer mutator ------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// Data tracing. -//===----------------------------------------------------------------------===// - -#include "FuzzerDictionary.h" -#include "FuzzerInternal.h" -#include "FuzzerIO.h" -#include "FuzzerMutate.h" -#include "FuzzerTracePC.h" -#include <algorithm> -#include <cstring> -#include <map> -#include <set> -#include <thread> - -namespace fuzzer { - -// Declared as static globals for faster checks inside the hooks. -static bool RecordingMemmem = false; - -int ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr; - -class TraceState { -public: - TraceState(MutationDispatcher &MD, const FuzzingOptions &Options, - const Fuzzer *F) - : MD(MD), Options(Options), F(F) {} - - void StartTraceRecording() { - if (!Options.UseMemmem) - return; - RecordingMemmem = true; - InterestingWords.clear(); - MD.ClearAutoDictionary(); - } - - void StopTraceRecording() { - if (!RecordingMemmem) - return; - for (auto &W : InterestingWords) - MD.AddWordToAutoDictionary({W}); - } - - void AddInterestingWord(const uint8_t *Data, size_t Size) { - if (!RecordingMemmem || !F->InFuzzingThread()) return; - if (Size <= 1) return; - Size = std::min(Size, Word::GetMaxSize()); - Word W(Data, Size); - InterestingWords.insert(W); - } - - private: - - // TODO: std::set is too inefficient, need to have a custom DS here. - std::set<Word> InterestingWords; - MutationDispatcher &MD; - const FuzzingOptions Options; - const Fuzzer *F; -}; - -static TraceState *TS; - -void Fuzzer::StartTraceRecording() { - if (!TS) return; - TS->StartTraceRecording(); -} - -void Fuzzer::StopTraceRecording() { - if (!TS) return; - TS->StopTraceRecording(); -} - -void Fuzzer::InitializeTraceState() { - if (!Options.UseMemmem) return; - TS = new TraceState(MD, Options, this); -} - -static size_t InternalStrnlen(const char *S, size_t MaxLen) { - size_t Len = 0; - for (; Len < MaxLen && S[Len]; Len++) {} - return Len; -} - -// Finds min of (strlen(S1), strlen(S2)). -// Needed bacause one of these strings may actually be non-zero terminated. -static size_t InternalStrnlen2(const char *S1, const char *S2) { - size_t Len = 0; - for (; S1[Len] && S2[Len]; Len++) {} - return Len; -} - -} // namespace fuzzer - -using fuzzer::TS; - -extern "C" { - -// We may need to avoid defining weak hooks to stay compatible with older clang. -#ifndef LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS -# define LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS 1 -#endif - -#if LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_memcmp(void *caller_pc, const void *s1, - const void *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - if (n <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/false); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strncmp(void *caller_pc, const char *s1, - const char *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - size_t Len1 = fuzzer::InternalStrnlen(s1, n); - size_t Len2 = fuzzer::InternalStrnlen(s2, n); - n = std::min(n, Len1); - n = std::min(n, Len2); - if (n <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, n, /*StopAtZero*/true); -} - - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcmp(void *caller_pc, const char *s1, - const char *s2, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - if (result == 0) return; // No reason to mutate. - size_t N = fuzzer::InternalStrnlen2(s1, s2); - if (N <= 1) return; // Not interesting. - fuzzer::TPC.AddValueForMemcmp(caller_pc, s1, s2, N, /*StopAtZero*/true); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - return __sanitizer_weak_hook_strncmp(called_pc, s1, s2, n, result); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, - const char *s2, int result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - return __sanitizer_weak_hook_strcmp(called_pc, s1, s2, result); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, - const char *s2, char *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, - const char *s2, char *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), strlen(s2)); -} - -ATTRIBUTE_INTERFACE ATTRIBUTE_NO_SANITIZE_MEMORY -void __sanitizer_weak_hook_memmem(void *called_pc, const void *s1, size_t len1, - const void *s2, size_t len2, void *result) { - if (fuzzer::ScopedDoingMyOwnMemOrStr::DoingMyOwnMemOrStr) return; - TS->AddInterestingWord(reinterpret_cast<const uint8_t *>(s2), len2); -} - -#endif // LLVM_FUZZER_DEFINES_SANITIZER_WEAK_HOOOKS -} // extern "C"
--- a/tools/fuzzing/libfuzzer/FuzzerUtil.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.cpp @@ -119,17 +119,17 @@ bool ParseOneDictionaryEntry(const std:: } else { // Any other character. U->push_back(V); } } return true; } -bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) { +bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units) { if (Text.empty()) { Printf("ParseDictionaryFile: file does not exist or is empty\n"); return false; } std::istringstream ISS(Text); Units->clear(); Unit U; int LineNo = 0; @@ -176,17 +176,17 @@ std::string Base64(const Unit &U) { Res += Table[(x >> 6) & 63]; Res += "="; } return Res; } std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC) { if (!EF->__sanitizer_symbolize_pc) return "<can not symbolize>"; - char PcDescr[1024]; + char PcDescr[1024] = {}; EF->__sanitizer_symbolize_pc(reinterpret_cast<void*>(PC), SymbolizedFMT, PcDescr, sizeof(PcDescr)); PcDescr[sizeof(PcDescr) - 1] = 0; // Just in case. return PcDescr; } void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC) { if (EF->__sanitizer_symbolize_pc) @@ -200,19 +200,16 @@ unsigned NumberOfCpuCores() { if (!N) { Printf("WARNING: std::thread::hardware_concurrency not well defined for " "your platform. Assuming CPU count of 1.\n"); N = 1; } return N; } -bool ExecuteCommandAndReadOutput(const std::string &Command, std::string *Out) { - FILE *Pipe = OpenProcessPipe(Command.c_str(), "r"); - if (!Pipe) return false; - char Buff[1024]; - size_t N; - while ((N = fread(Buff, 1, sizeof(Buff), Pipe)) > 0) - Out->append(Buff, N); - return true; +size_t SimpleFastHash(const uint8_t *Data, size_t Size) { + size_t Res = 0; + for (size_t i = 0; i < Size; i++) + Res = Res * 11 + Data[i]; + return Res; } } // namespace fuzzer
--- a/tools/fuzzing/libfuzzer/FuzzerUtil.h +++ b/tools/fuzzing/libfuzzer/FuzzerUtil.h @@ -8,16 +8,17 @@ //===----------------------------------------------------------------------===// // Util functions. //===----------------------------------------------------------------------===// #ifndef LLVM_FUZZER_UTIL_H #define LLVM_FUZZER_UTIL_H #include "FuzzerDefs.h" +#include "FuzzerCommand.h" namespace fuzzer { void PrintHexArray(const Unit &U, const char *PrintAfter = ""); void PrintHexArray(const uint8_t *Data, size_t Size, const char *PrintAfter = ""); @@ -36,41 +37,51 @@ bool IsASCII(const uint8_t *Data, size_t std::string Base64(const Unit &U); void PrintPC(const char *SymbolizedFMT, const char *FallbackFMT, uintptr_t PC); std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC); unsigned NumberOfCpuCores(); -bool ExecuteCommandAndReadOutput(const std::string &Command, std::string *Out); - // Platform specific functions. void SetSignalHandler(const FuzzingOptions& Options); void SleepSeconds(int Seconds); unsigned long GetPid(); size_t GetPeakRSSMb(); -int ExecuteCommand(const std::string &Command); +int ExecuteCommand(const Command &Cmd); FILE *OpenProcessPipe(const char *Command, const char *Mode); const void *SearchMemory(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); -std::string CloneArgsWithoutX(const std::vector<std::string> &Args, +std::string CloneArgsWithoutX(const Vector<std::string> &Args, const char *X1, const char *X2); -inline std::string CloneArgsWithoutX(const std::vector<std::string> &Args, +inline std::string CloneArgsWithoutX(const Vector<std::string> &Args, const char *X) { return CloneArgsWithoutX(Args, X, X); } +inline std::pair<std::string, std::string> SplitBefore(std::string X, + std::string S) { + auto Pos = S.find(X); + if (Pos == std::string::npos) + return std::make_pair(S, ""); + return std::make_pair(S.substr(0, Pos), S.substr(Pos)); +} + std::string DisassembleCmd(const std::string &FileName); std::string SearchRegexCmd(const std::string &Regex); +size_t SimpleFastHash(const uint8_t *Data, size_t Size); + +inline uint32_t Log(uint32_t X) { return 32 - __builtin_clz(X) - 1; } + } // namespace fuzzer #endif // LLVM_FUZZER_UTIL_H
--- a/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerUtilDarwin.cpp @@ -5,21 +5,23 @@ // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Misc utils for Darwin. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_APPLE - +#include "FuzzerCommand.h" #include "FuzzerIO.h" #include <mutex> #include <signal.h> #include <spawn.h> +#include <stdlib.h> +#include <string.h> #include <sys/wait.h> // There is no header for this on macOS so declare here extern "C" char **environ; namespace fuzzer { static std::mutex SignalMutex; @@ -31,17 +33,18 @@ static struct sigaction OldSigQuitAction static sigset_t OldBlockedSignalsSet; // This is a reimplementation of Libc's `system()`. On Darwin the Libc // implementation contains a mutex which prevents it from being used // concurrently. This implementation **can** be used concurrently. It sets the // signal handlers when the first thread enters and restores them when the last // thread finishes execution of the function and ensures this is not racey by // using a mutex. -int ExecuteCommand(const std::string &Command) { +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); posix_spawnattr_t SpawnAttributes; if (posix_spawnattr_init(&SpawnAttributes)) return -1; // Block and ignore signals of the current process when the first thread // enters. { std::lock_guard<std::mutex> Lock(SignalMutex); if (ActiveThreadCount == 0) { @@ -91,22 +94,27 @@ int ExecuteCommand(const std::string &Co (void)posix_spawnattr_setsigdefault(&SpawnAttributes, &DefaultSigSet); // Make sure the child process doesn't block SIGCHLD (void)posix_spawnattr_setsigmask(&SpawnAttributes, &OldBlockedSignalsSet); short SpawnFlags = POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK; (void)posix_spawnattr_setflags(&SpawnAttributes, SpawnFlags); pid_t Pid; char **Environ = environ; // Read from global - const char *CommandCStr = Command.c_str(); - const char *Argv[] = {"sh", "-c", CommandCStr, NULL}; + const char *CommandCStr = CmdLine.c_str(); + char *const Argv[] = { + strdup("sh"), + strdup("-c"), + strdup(CommandCStr), + NULL + }; int ErrorCode = 0, ProcessStatus = 0; // FIXME: We probably shouldn't hardcode the shell path. ErrorCode = posix_spawn(&Pid, "/bin/sh", NULL, &SpawnAttributes, - (char *const *)Argv, Environ); + Argv, Environ); (void)posix_spawnattr_destroy(&SpawnAttributes); if (!ErrorCode) { pid_t SavedPid = Pid; do { // Repeat until call completes uninterrupted. Pid = waitpid(SavedPid, &ProcessStatus, /*options=*/0); } while (Pid == -1 && errno == EINTR); if (Pid == -1) { @@ -115,16 +123,18 @@ int ExecuteCommand(const std::string &Co } } else if (ErrorCode == ENOMEM || ErrorCode == EAGAIN) { // Fork failure. ProcessStatus = -1; } else { // Shell execution failure. ProcessStatus = W_EXITCODE(127, 0); } + for (unsigned i = 0, n = sizeof(Argv) / sizeof(Argv[0]); i < n; ++i) + free(Argv[i]); // Restore the signal handlers of the current process when the last thread // using this function finishes. { std::lock_guard<std::mutex> Lock(SignalMutex); --ActiveThreadCount; if (ActiveThreadCount == 0) { bool FailedRestore = false;
new file mode 100644 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerUtilFuchsia.cpp @@ -0,0 +1,240 @@ +//===- FuzzerUtilFuchsia.cpp - Misc utils for Fuchsia. --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Misc utils implementation using Fuchsia/Zircon APIs. +//===----------------------------------------------------------------------===// +#include "FuzzerDefs.h" + +#if LIBFUZZER_FUCHSIA + +#include "FuzzerInternal.h" +#include "FuzzerUtil.h" +#include <cerrno> +#include <cinttypes> +#include <cstdint> +#include <fcntl.h> +#include <launchpad/launchpad.h> +#include <string> +#include <thread> +#include <unistd.h> +#include <zircon/errors.h> +#include <zircon/process.h> +#include <zircon/status.h> +#include <zircon/syscalls.h> +#include <zircon/syscalls/port.h> +#include <zircon/types.h> + +namespace fuzzer { + +namespace { + +// A magic value for the Zircon exception port, chosen to spell 'FUZZING' +// when interpreted as a byte sequence on little-endian platforms. +const uint64_t kFuzzingCrash = 0x474e495a5a5546; + +void AlarmHandler(int Seconds) { + while (true) { + SleepSeconds(Seconds); + Fuzzer::StaticAlarmCallback(); + } +} + +void InterruptHandler() { + // Ctrl-C sends ETX in Zircon. + while (getchar() != 0x03); + Fuzzer::StaticInterruptCallback(); +} + +void CrashHandler(zx_handle_t *Port) { + std::unique_ptr<zx_handle_t> ExceptionPort(Port); + zx_port_packet_t Packet; + _zx_port_wait(*ExceptionPort, ZX_TIME_INFINITE, &Packet, 1); + // Unbind as soon as possible so we don't receive exceptions from this thread. + if (_zx_task_bind_exception_port(ZX_HANDLE_INVALID, ZX_HANDLE_INVALID, + kFuzzingCrash, 0) != ZX_OK) { + // Shouldn't happen; if it does the safest option is to just exit. + Printf("libFuzzer: unable to unbind exception port; aborting!\n"); + exit(1); + } + if (Packet.key != kFuzzingCrash) { + Printf("libFuzzer: invalid crash key: %" PRIx64 "; aborting!\n", + Packet.key); + exit(1); + } + // CrashCallback should not return from this call + Fuzzer::StaticCrashSignalCallback(); +} + +} // namespace + +// Platform specific functions. +void SetSignalHandler(const FuzzingOptions &Options) { + zx_status_t rc; + + // Set up alarm handler if needed. + if (Options.UnitTimeoutSec > 0) { + std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1); + T.detach(); + } + + // Set up interrupt handler if needed. + if (Options.HandleInt || Options.HandleTerm) { + std::thread T(InterruptHandler); + T.detach(); + } + + // Early exit if no crash handler needed. + if (!Options.HandleSegv && !Options.HandleBus && !Options.HandleIll && + !Options.HandleFpe && !Options.HandleAbrt) + return; + + // Create an exception port + zx_handle_t *ExceptionPort = new zx_handle_t; + if ((rc = _zx_port_create(0, ExceptionPort)) != ZX_OK) { + Printf("libFuzzer: zx_port_create failed: %s\n", _zx_status_get_string(rc)); + exit(1); + } + + // Bind the port to receive exceptions from our process + if ((rc = _zx_task_bind_exception_port(_zx_process_self(), *ExceptionPort, + kFuzzingCrash, 0)) != ZX_OK) { + Printf("libFuzzer: unable to bind exception port: %s\n", + zx_status_get_string(rc)); + exit(1); + } + + // Set up the crash handler. + std::thread T(CrashHandler, ExceptionPort); + T.detach(); +} + +void SleepSeconds(int Seconds) { + _zx_nanosleep(_zx_deadline_after(ZX_SEC(Seconds))); +} + +unsigned long GetPid() { + zx_status_t rc; + zx_info_handle_basic_t Info; + if ((rc = zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + zx_status_get_string(rc)); + exit(1); + } + return Info.koid; +} + +size_t GetPeakRSSMb() { + zx_status_t rc; + zx_info_task_stats_t Info; + if ((rc = _zx_object_get_info(_zx_process_self(), ZX_INFO_TASK_STATS, &Info, + sizeof(Info), NULL, NULL)) != ZX_OK) { + Printf("libFuzzer: unable to get info about self: %s\n", + _zx_status_get_string(rc)); + exit(1); + } + return (Info.mem_private_bytes + Info.mem_shared_bytes) >> 20; +} + +template <typename Fn> +class RunOnDestruction { + public: + explicit RunOnDestruction(Fn fn) : fn_(fn) {} + ~RunOnDestruction() { fn_(); } + + private: + Fn fn_; +}; + +template <typename Fn> +RunOnDestruction<Fn> at_scope_exit(Fn fn) { + return RunOnDestruction<Fn>(fn); +} + +int ExecuteCommand(const Command &Cmd) { + zx_status_t rc; + + // Convert arguments to C array + auto Args = Cmd.getArguments(); + size_t Argc = Args.size(); + assert(Argc != 0); + std::unique_ptr<const char *[]> Argv(new const char *[Argc]); + for (size_t i = 0; i < Argc; ++i) + Argv[i] = Args[i].c_str(); + + // Create the basic launchpad. Clone everything except stdio. + launchpad_t *lp; + launchpad_create(ZX_HANDLE_INVALID, Argv[0], &lp); + launchpad_load_from_file(lp, Argv[0]); + launchpad_set_args(lp, Argc, Argv.get()); + launchpad_clone(lp, LP_CLONE_ALL & (~LP_CLONE_FDIO_STDIO)); + + // Determine stdout + int FdOut = STDOUT_FILENO; + + if (Cmd.hasOutputFile()) { + auto Filename = Cmd.getOutputFile(); + FdOut = open(Filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0); + if (FdOut == -1) { + Printf("libFuzzer: failed to open %s: %s\n", Filename.c_str(), + strerror(errno)); + return ZX_ERR_IO; + } + } + auto CloseFdOut = at_scope_exit([&]() { close(FdOut); } ); + + // Determine stderr + int FdErr = STDERR_FILENO; + if (Cmd.isOutAndErrCombined()) + FdErr = FdOut; + + // Clone the file descriptors into the new process + if ((rc = launchpad_clone_fd(lp, STDIN_FILENO, STDIN_FILENO)) != ZX_OK || + (rc = launchpad_clone_fd(lp, FdOut, STDOUT_FILENO)) != ZX_OK || + (rc = launchpad_clone_fd(lp, FdErr, STDERR_FILENO)) != ZX_OK) { + Printf("libFuzzer: failed to clone FDIO: %s\n", _zx_status_get_string(rc)); + return rc; + } + + // Start the process + zx_handle_t ProcessHandle = ZX_HANDLE_INVALID; + const char *ErrorMsg = nullptr; + if ((rc = launchpad_go(lp, &ProcessHandle, &ErrorMsg)) != ZX_OK) { + Printf("libFuzzer: failed to launch '%s': %s, %s\n", Argv[0], ErrorMsg, + _zx_status_get_string(rc)); + return rc; + } + auto CloseHandle = at_scope_exit([&]() { _zx_handle_close(ProcessHandle); }); + + // Now join the process and return the exit status. + if ((rc = _zx_object_wait_one(ProcessHandle, ZX_PROCESS_TERMINATED, + ZX_TIME_INFINITE, nullptr)) != ZX_OK) { + Printf("libFuzzer: failed to join '%s': %s\n", Argv[0], + _zx_status_get_string(rc)); + return rc; + } + + zx_info_process_t Info; + if ((rc = _zx_object_get_info(ProcessHandle, ZX_INFO_PROCESS, &Info, + sizeof(Info), nullptr, nullptr)) != ZX_OK) { + Printf("libFuzzer: unable to get return code from '%s': %s\n", Argv[0], + zx_status_get_string(rc)); + return rc; + } + + return Info.return_code; +} + +const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, + size_t PattLen) { + return memmem(Data, DataLen, Patt, PattLen); +} + +} // namespace fuzzer + +#endif // LIBFUZZER_FUCHSIA
--- a/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerUtilLinux.cpp @@ -4,21 +4,23 @@ // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Misc utils for Linux. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" -#if LIBFUZZER_LINUX +#if LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD +#include "FuzzerCommand.h" #include <stdlib.h> namespace fuzzer { -int ExecuteCommand(const std::string &Command) { - return system(Command.c_str()); +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); } } // namespace fuzzer -#endif // LIBFUZZER_LINUX +#endif // LIBFUZZER_LINUX || LIBFUZZER_NETBSD || LIBFUZZER_FREEBSD
--- a/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerUtilPosix.cpp @@ -13,17 +13,16 @@ #include "FuzzerIO.h" #include "FuzzerInternal.h" #include <cassert> #include <chrono> #include <cstring> #include <errno.h> #include <iomanip> #include <signal.h> -#include <sstream> #include <stdio.h> #include <sys/resource.h> #include <sys/syscall.h> #include <sys/time.h> #include <sys/types.h> #include <thread> #include <unistd.h> @@ -36,24 +35,41 @@ static void AlarmHandler(int, siginfo_t static void CrashHandler(int, siginfo_t *, void *) { Fuzzer::StaticCrashSignalCallback(); } static void InterruptHandler(int, siginfo_t *, void *) { Fuzzer::StaticInterruptCallback(); } +static void GracefulExitHandler(int, siginfo_t *, void *) { + Fuzzer::StaticGracefulExitCallback(); +} + static void FileSizeExceedHandler(int, siginfo_t *, void *) { Fuzzer::StaticFileSizeExceedCallback(); } static void SetSigaction(int signum, void (*callback)(int, siginfo_t *, void *)) { - struct sigaction sigact; - memset(&sigact, 0, sizeof(sigact)); + struct sigaction sigact = {}; + if (sigaction(signum, nullptr, &sigact)) { + Printf("libFuzzer: sigaction failed with %d\n", errno); + exit(1); + } + if (sigact.sa_flags & SA_SIGINFO) { + if (sigact.sa_sigaction) + return; + } else { + if (sigact.sa_handler != SIG_DFL && sigact.sa_handler != SIG_IGN && + sigact.sa_handler != SIG_ERR) + return; + } + + sigact = {}; sigact.sa_sigaction = callback; if (sigaction(signum, &sigact, 0)) { Printf("libFuzzer: sigaction failed with %d\n", errno); exit(1); } } void SetTimer(int Seconds) { @@ -81,29 +97,33 @@ void SetSignalHandler(const FuzzingOptio if (Options.HandleAbrt) SetSigaction(SIGABRT, CrashHandler); if (Options.HandleIll) SetSigaction(SIGILL, CrashHandler); if (Options.HandleFpe) SetSigaction(SIGFPE, CrashHandler); if (Options.HandleXfsz) SetSigaction(SIGXFSZ, FileSizeExceedHandler); + if (Options.HandleUsr1) + SetSigaction(SIGUSR1, GracefulExitHandler); + if (Options.HandleUsr2) + SetSigaction(SIGUSR2, GracefulExitHandler); } void SleepSeconds(int Seconds) { sleep(Seconds); // Use C API to avoid coverage from instrumented libc++. } unsigned long GetPid() { return (unsigned long)getpid(); } size_t GetPeakRSSMb() { struct rusage usage; if (getrusage(RUSAGE_SELF, &usage)) return 0; - if (LIBFUZZER_LINUX) { + if (LIBFUZZER_LINUX || LIBFUZZER_FREEBSD || LIBFUZZER_NETBSD) { // ru_maxrss is in KiB return usage.ru_maxrss >> 10; } else if (LIBFUZZER_APPLE) { // ru_maxrss is in bytes return usage.ru_maxrss >> 20; } assert(0 && "GetPeakRSSMb() is not implemented for your platform"); return 0;
--- a/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp +++ b/tools/fuzzing/libfuzzer/FuzzerUtilWindows.cpp @@ -5,29 +5,31 @@ // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // Misc utils implementation for Windows. //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_WINDOWS +#include "FuzzerCommand.h" #include "FuzzerIO.h" #include "FuzzerInternal.h" #include <cassert> #include <chrono> #include <cstring> #include <errno.h> #include <iomanip> #include <signal.h> -#include <sstream> #include <stdio.h> #include <sys/types.h> #include <windows.h> -#include <psapi.h> + +// This must be included after windows.h. +#include <Psapi.h> namespace fuzzer { static const FuzzingOptions* HandlerOpt = nullptr; static LONG CALLBACK ExceptionHandler(PEXCEPTION_POINTERS ExceptionInfo) { switch (ExceptionInfo->ExceptionRecord->ExceptionCode) { case EXCEPTION_ACCESS_VIOLATION: @@ -145,18 +147,19 @@ size_t GetPeakRSSMb() { return 0; return info.PeakWorkingSetSize >> 20; } FILE *OpenProcessPipe(const char *Command, const char *Mode) { return _popen(Command, Mode); } -int ExecuteCommand(const std::string &Command) { - return system(Command.c_str()); +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); } const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, size_t PattLen) { // TODO: make this implementation more efficient. const char *Cdata = (const char *)Data; const char *Cpatt = (const char *)Patt;
--- a/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h +++ b/tools/fuzzing/libfuzzer/FuzzerValueBitMap.h @@ -47,48 +47,27 @@ struct ValueBitMap { inline bool Get(uintptr_t Idx) { assert(Idx < kMapSizeInBits); uintptr_t WordIdx = Idx / kBitsInWord; uintptr_t BitIdx = Idx % kBitsInWord; return Map[WordIdx] & (1UL << BitIdx); } - size_t GetNumBitsSinceLastMerge() const { return NumBits; } - - // Merges 'Other' into 'this', clears 'Other', updates NumBits, - // returns true if new bits were added. - ATTRIBUTE_TARGET_POPCNT - bool MergeFrom(ValueBitMap &Other) { - uintptr_t Res = 0; - size_t OldNumBits = NumBits; - for (size_t i = 0; i < kMapSizeInWords; i++) { - auto O = Other.Map[i]; - auto M = Map[i]; - if (O) { - Map[i] = (M |= O); - Other.Map[i] = 0; - } - if (M) - Res += __builtin_popcountll(M); - } - NumBits = Res; - return OldNumBits < NumBits; - } + size_t SizeInBits() const { return kMapSizeInBits; } template <class Callback> ATTRIBUTE_NO_SANITIZE_ALL void ForEach(Callback CB) const { for (size_t i = 0; i < kMapSizeInWords; i++) if (uintptr_t M = Map[i]) for (size_t j = 0; j < sizeof(M) * 8; j++) if (M & ((uintptr_t)1 << j)) CB(i * sizeof(M) * 8 + j); } private: - size_t NumBits = 0; uintptr_t Map[kMapSizeInWords] __attribute__((aligned(512))); }; } // namespace fuzzer #endif // LLVM_FUZZER_VALUE_BIT_MAP_H
--- a/tools/fuzzing/libfuzzer/moz.build +++ b/tools/fuzzing/libfuzzer/moz.build @@ -6,16 +6,17 @@ Library('fuzzer') EXPORTS += [ 'FuzzerDefs.h', ] SOURCES += [ + 'FuzzerClangCounters.cpp', 'FuzzerCrossOver.cpp', 'FuzzerDriver.cpp', 'FuzzerExtFunctionsDlsym.cpp', 'FuzzerExtFunctionsDlsymWin.cpp', 'FuzzerExtFunctionsWeak.cpp', 'FuzzerExtFunctionsWeakAlias.cpp', 'FuzzerExtraCounters.cpp', 'FuzzerIO.cpp', @@ -23,17 +24,16 @@ SOURCES += [ 'FuzzerIOWindows.cpp', 'FuzzerLoop.cpp', 'FuzzerMerge.cpp', 'FuzzerMutate.cpp', 'FuzzerSHA1.cpp', 'FuzzerShmemPosix.cpp', 'FuzzerShmemWindows.cpp', 'FuzzerTracePC.cpp', - 'FuzzerTraceState.cpp', 'FuzzerUtil.cpp', 'FuzzerUtilDarwin.cpp', 'FuzzerUtilLinux.cpp', 'FuzzerUtilPosix.cpp', 'FuzzerUtilWindows.cpp' ] if CONFIG['CC_TYPE'] == 'clang':