[libFuzzer] remove the deprecated 'tokens' feature

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@251069 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Fuzzer/FuzzerDriver.cpp

@@ -182,26 +182,6 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
   return HasErrors ? 1 : 0;
 }
 
-std::vector<std::string> ReadTokensFile(const char *TokensFilePath) {
-  if (!TokensFilePath) return {};
-  std::string TokensFileContents = FileToString(TokensFilePath);
-  std::istringstream ISS(TokensFileContents);
-  std::vector<std::string> Res = {std::istream_iterator<std::string>{ISS},
-                                  std::istream_iterator<std::string>{}};
-  Res.push_back(" ");
-  Res.push_back("\t");
-  Res.push_back("\n");
-  return Res;
-}
-
-int ApplyTokens(const Fuzzer &F, const char *InputFilePath) {
-  Unit U = FileToVector(InputFilePath);
-  auto T = F.SubstituteTokens(U);
-  T.push_back(0);
-  Printf("%s", T.data());
-  return 0;
-}
-
 int RunOneTest(Fuzzer *F, const char *InputFilePath) {
   Unit U = FileToVector(InputFilePath);
   F->ExecuteCallback(U);
@@ -258,7 +238,6 @@ int FuzzerDriver(const std::vector<std::string> &Args,
   Options.ShuffleAtStartUp = Flags.shuffle;
   Options.PreferSmallDuringInitialShuffle =
       Flags.prefer_small_during_initial_shuffle;
-  Options.Tokens = ReadTokensFile(Flags.deprecated_tokens);
   Options.Reload = Flags.reload;
   Options.OnlyASCII = Flags.only_ascii;
   Options.TBMDepth = Flags.tbm_depth;
@@ -282,9 +261,6 @@ int FuzzerDriver(const std::vector<std::string> &Args,
 
   Fuzzer F(USF, Options);
 
-  if (Flags.apply_tokens)
-    return ApplyTokens(F, Flags.apply_tokens);
-
   // Timer
   if (Flags.timeout > 0)
     SetTimer(Flags.timeout / 2 + 1);
@@ -300,13 +276,6 @@ int FuzzerDriver(const std::vector<std::string> &Args,
     Printf("Seed: %u\n", Seed);
   USF.GetRand().ResetSeed(Seed);
 
-  if (Flags.verbosity >= 2) {
-    Printf("Tokens: {");
-    for (auto &T : Options.Tokens)
-      Printf("%s,", T.c_str());
-    Printf("}\n");
-  }
-
   F.RereadOutputCorpus();
   for (auto &inp : *Inputs)
     if (inp != Options.OutputCorpus)

lib/Fuzzer/FuzzerFlags.def

@@ -47,11 +47,6 @@ FUZZER_FLAG_INT(workers, 0,
 FUZZER_FLAG_INT(reload, 1,
                 "Reload the main corpus periodically to get new units"
                 " discovered by other processes.")
-FUZZER_FLAG_STRING(deprecated_tokens,
-                   "Use the file with tokens (one token per line) to"
-                   " fuzz a token based input language.")
-FUZZER_FLAG_STRING(apply_tokens, "Read the given input file, substitute bytes "
-                                 " with tokens and write the result to stdout.")
 FUZZER_FLAG_STRING(sync_command, "Execute an external command "
                                  "\"<sync_command> <test_corpus>\" "
                                  "to synchronize the test corpus.")

lib/Fuzzer/FuzzerInternal.h

@@ -93,7 +93,6 @@ class Fuzzer {
     std::string OutputCorpus;
     std::string SyncCommand;
     std::string ArtifactPrefix = "./";
-    std::vector<std::string> Tokens;
     std::vector<Unit> Dictionary;
     bool SaveArtifacts = true;
   };
@@ -119,7 +118,6 @@ class Fuzzer {
 
   static void StaticAlarmCallback();
 
-  Unit SubstituteTokens(const Unit &U) const;
   void ExecuteCallback(const Unit &U);
 
  private:
@@ -133,7 +131,7 @@ class Fuzzer {
   void WriteToOutputCorpus(const Unit &U);
   void WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix);
   void PrintStats(const char *Where, size_t Cov, const char *End = "\n");
-  void PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter = "");
+  void PrintUnitInASCII(const Unit &U, const char *PrintAfter = "");
 
   void SyncCorpus();
 

lib/Fuzzer/FuzzerLoop.cpp

@@ -35,14 +35,8 @@ void Fuzzer::SetDeathCallback() {
   __sanitizer_set_death_callback(StaticDeathCallback);
 }
 
-void Fuzzer::PrintUnitInASCIIOrTokens(const Unit &U, const char *PrintAfter) {
-  if (Options.Tokens.empty()) {
-    PrintASCII(U, PrintAfter);
-  } else {
-    auto T = SubstituteTokens(U);
-    T.push_back(0);
-    Printf("%s%s", T.data(), PrintAfter);
-  }
+void Fuzzer::PrintUnitInASCII(const Unit &U, const char *PrintAfter) {
+  PrintASCII(U, PrintAfter);
 }
 
 void Fuzzer::StaticDeathCallback() {
@@ -54,7 +48,7 @@ void Fuzzer::DeathCallback() {
   Printf("DEATH:\n");
   if (CurrentUnit.size() <= kMaxUnitSizeToPrint) {
     Print(CurrentUnit, "\n");
-    PrintUnitInASCIIOrTokens(CurrentUnit, "\n");
+    PrintUnitInASCII(CurrentUnit, "\n");
   }
   WriteUnitToFileWithPrefix(CurrentUnit, "crash-");
 }
@@ -77,7 +71,7 @@ void Fuzzer::AlarmCallback() {
            Options.UnitTimeoutSec);
     if (CurrentUnit.size() <= kMaxUnitSizeToPrint) {
       Print(CurrentUnit, "\n");
-      PrintUnitInASCIIOrTokens(CurrentUnit, "\n");
+      PrintUnitInASCII(CurrentUnit, "\n");
     }
     WriteUnitToFileWithPrefix(CurrentUnit, "timeout-");
     Printf("==%d== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
@@ -191,28 +185,9 @@ void Fuzzer::RunOneAndUpdateCorpus(Unit &U) {
   ReportNewCoverage(RunOne(U), U);
 }
 
-Unit Fuzzer::SubstituteTokens(const Unit &U) const {
-  Unit Res;
-  for (auto Idx : U) {
-    if (Idx < Options.Tokens.size()) {
-      std::string Token = Options.Tokens[Idx];
-      Res.insert(Res.end(), Token.begin(), Token.end());
-    } else {
-      Res.push_back(' ');
-    }
-  }
-  // FIXME: Apply DFSan labels.
-  return Res;
-}
-
 void Fuzzer::ExecuteCallback(const Unit &U) {
-  int Res = 0;
-  if (Options.Tokens.empty()) {
-    Res = USF.TargetFunction(U.data(), U.size());
-  } else {
-    auto T = SubstituteTokens(U);
-    Res = USF.TargetFunction(T.data(), T.size());
-  }
+  int Res = USF.TargetFunction(U.data(), U.size());
+  (void)Res;
   assert(Res == 0);
 }
 
@@ -278,7 +253,7 @@ void Fuzzer::ReportNewCoverage(size_t NewCoverage, const Unit &U) {
     Printf(" L: %zd", U.size());
     if (U.size() < 30) {
       Printf(" ");
-      PrintUnitInASCIIOrTokens(U, "\t");
+      PrintUnitInASCII(U, "\t");
       Print(U);
     }
     Printf("\n");

lib/Fuzzer/test/CMakeLists.txt

@@ -14,7 +14,6 @@ set(DFSanTests
 
 set(Tests
   CounterTest
-  CxxTokensTest
   FourIndependentBranchesTest
   FullCoverageSetTest
   InfiniteTest

lib/Fuzzer/test/fuzzer.test

@@ -37,8 +37,6 @@ RUN: not LLVMFuzzer-CounterTest -use_counters=1 -max_len=6 -seed=1 -timeout=15 2
 
 RUN: not LLVMFuzzer-SimpleCmpTest -use_traces=1 -seed=1 -runs=1000000 -timeout=5 2>&1 | FileCheck %s
 
-RUN: not LLVMFuzzer-CxxTokensTest -seed=1 -timeout=15 -deprecated_tokens=%S/../cxx_fuzzer_tokens.txt 2>&1 | FileCheck %s
-
 RUN: not LLVMFuzzer-UserSuppliedFuzzerTest -seed=1 -timeout=15 2>&1 | FileCheck %s
 
 RUN: not LLVMFuzzer-MemcmpTest -use_traces=1 -seed=1 -runs=100000   2>&1 | FileCheck %s