Separate binary test data by endianness

lm/common/model_buffer_test.cc

@@ -8,12 +8,20 @@
 namespace lm { namespace {
 
 BOOST_AUTO_TEST_CASE(Query) {
-  std::string dir("test_data/");
+  std::string dir("test_data");
   if (boost::unit_test::framework::master_test_suite().argc == 2) {
     dir = boost::unit_test::framework::master_test_suite().argv[1];
   }
   ngram::Model ref((dir + "/toy0.arpa").c_str());
-  ModelBuffer test(dir + "/toy0");
+#if BYTE_ORDER == LITTLE_ENDIAN
+  std::string endian = "little";
+#elif BYTE_ORDER == BIG_ENDIAN
+  std::string endian = "big";
+#else
+#error "Unsupported byte order."
+#endif
+
+  ModelBuffer test(dir + "/" + endian + "endian/toy0");
   ngram::State ref_state, test_state;
   WordIndex a = ref.GetVocabulary().Index("a");
   BOOST_CHECK_CLOSE(

lm/common/test_data/generate.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
-../../../bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy0 --arpa toy0.arpa <<EOF
+../../../../build/bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy0 --arpa ../toy0.arpa <<EOF
 a a b a
 b a a b
 EOF
-../../../bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy1 --arpa toy1.arpa <<EOF
+../../../../build/bin/lmplz --discount_fallback -o 3 -S 100M --intermediate toy1 --arpa ../toy1.arpa <<EOF
 a a b b b b b b b
 c
 EOF

lm/common/test_data/littleendian/toy0.kenlm_intermediate

@@ -0,0 +1,3 @@
+KenLM intermediate binary file
+Counts 5 7 7
+Payload pb

lm/common/test_data/littleendian/toy1.kenlm_intermediate

@@ -0,0 +1,3 @@
+KenLM intermediate binary file
+Counts 6 7 6
+Payload pb

lm/common/test_data/toy0.arpa

@@ -15,7 +15,7 @@ ngram 3=7
 -0.37712017	a a	-0.30103
 -0.2984526	b a	-0.30103
 -0.58682007	a </s>	0
--0.52201796	b </s>	0
+-0.5220179	b </s>	0
 -0.41574955	<s> b	-0.30103
 -0.58682007	a b	-0.30103
 

lm/interpolate/CMakeLists.txt

@@ -49,7 +49,7 @@ if(EIGEN3_FOUND)
         # tune_instances_test needs an extra command line parameter
         KenLMAddTest(TEST tune_instances_test
                      LIBRARIES ${KENLM_INTERPOLATE_LIBS}
-                     TEST_ARGS -- ${CMAKE_CURRENT_SOURCE_DIR}/../common/test_data/toy0.1)
+                     TEST_ARGS -- ${CMAKE_CURRENT_SOURCE_DIR}/../common/test_data)
 
         foreach(test_file test1 test2 test3 test_bad_order test_no_unk)
           set(KENLM_MERGE_TESTS_PATH ${KENLM_MERGE_TESTS_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/merge_test/${test_file})

lm/interpolate/tune_instances_test.cc

@@ -20,17 +20,23 @@ BOOST_AUTO_TEST_CASE(Toy) {
   util::scoped_fd test_input(util::MakeTemp("temporary"));
   util::FileStream(test_input.get()) << "c\n";
 
-  StringPiece dir("../common/test_data/");
+  std::string dir("../common/test_data");
   if (boost::unit_test::framework::master_test_suite().argc == 2) {
-    StringPiece zero_file(boost::unit_test::framework::master_test_suite().argv[1]);
-    BOOST_REQUIRE(zero_file.size() > strlen("toy0.1"));
-    BOOST_REQUIRE_EQUAL("toy0.1", StringPiece(zero_file.data() + zero_file.size() - 6, 6));
-    dir = StringPiece(zero_file.data(), zero_file.size() - 6);
+    dir = boost::unit_test::framework::master_test_suite().argv[1];
   }
 
+#if BYTE_ORDER == LITTLE_ENDIAN
+  std::string endian = "little";
+#elif BYTE_ORDER == BIG_ENDIAN
+  std::string endian = "big";
+#else
+#error "Unsupported byte order."
+#endif
+  dir += "/" + endian + "endian/";
+
   std::vector<StringPiece> model_names;
-  std::string full0 = std::string(dir.data(), dir.size()) + "toy0";
-  std::string full1 = std::string(dir.data(), dir.size()) + "toy1";
+  std::string full0 = dir + "toy0";
+  std::string full1 = dir + "toy1";
   model_names.push_back(full0);
   model_names.push_back(full1);