Reland "iOS: Save perf results under Documents/perf_result.json"

This will require a manual roll to downstream projects, since the //test:perf_test target was introduced. This is a reland of 10a8e7a Original change's description: > iOS: Save perf results under Documents/perf_result.json > > [email protected] > > Bug: webrtc:7156 > Change-Id: Ib00992cce0007e0b5c9274340df1a892f810b0c5 > Reviewed-on: https://webrtc-review.googlesource.com/29202 > Commit-Queue: Edward Lemur <[email protected]> > Reviewed-by: Henrik Andreassson <[email protected]> > Reviewed-by: Patrik Höglund <[email protected]> > Cr-Commit-Position: refs/heads/master@{#21244} [email protected], [email protected] No-Try: true Bug: webrtc:7156 Change-Id: Iecdb108f605fd1c98acde4d50ab4f5a7b5f6bfaf Reviewed-on: https://webrtc-review.googlesource.com/32761 Reviewed-by: Edward Lemur <[email protected]> Commit-Queue: Edward Lemur <[email protected]> Cr-Commit-Position: refs/heads/master@{#21252}
czs100 · Dec 13, 2017 · 8b886bb · 8b886bb
1 parent d37709b
commit 8b886bb
Show file tree

Hide file tree

Showing 14 changed files with 91 additions and 22 deletions.
diff --git a/call/BUILD.gn b/call/BUILD.gn
@@ -255,6 +255,7 @@ if (rtc_include_tests) {
       "../test:direct_transport",
       "../test:fake_audio_device",
       "../test:field_trial",
+      "../test:perf_test",
       "../test:test_common",
       "../test:test_support",
       "../test:video_test_common",

diff --git a/modules/audio_coding/BUILD.gn b/modules/audio_coding/BUILD.gn
@@ -1318,6 +1318,7 @@ if (rtc_include_tests) {
       "../../rtc_base:rtc_base_approved",
       "../../system_wrappers",
       "../../system_wrappers:field_trial_api",
+      "../../test:perf_test",
       "../../test:test_support",
     ]
 
@@ -1806,6 +1807,7 @@ if (rtc_include_tests) {
 
     deps = [
       ":isac_fix",
+      "../../test:perf_test",
       "../../test:test_support",
     ]
 

diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn
@@ -713,6 +713,7 @@ if (rtc_include_tests) {
       "../../rtc_base:protobuf_utils",
       "../../rtc_base:rtc_base_approved",
       "../../system_wrappers",
+      "../../test:perf_test",
       "../../test:test_support",
     ]
 

diff --git a/modules/remote_bitrate_estimator/BUILD.gn b/modules/remote_bitrate_estimator/BUILD.gn
@@ -146,6 +146,7 @@ if (rtc_include_tests) {
       "../../rtc_base:rtc_numerics",
       "../../system_wrappers",
       "../../system_wrappers:field_trial_api",
+      "../../test:perf_test",
       "../../test:test_support",
       "../../voice_engine",
       "../bitrate_controller",

diff --git a/rtc_base/unittest_main.cc b/rtc_base/unittest_main.cc
@@ -109,7 +109,7 @@ int main(int argc, char* argv[]) {
   rtc::SSLStreamAdapter::enable_time_callback_for_testing();
 
 #if defined(WEBRTC_IOS)
-  rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+  rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv, false);
   rtc::test::RunTestsFromIOSApp();
 #endif
   const int res = RUN_ALL_TESTS();

diff --git a/test/BUILD.gn b/test/BUILD.gn
@@ -113,6 +113,19 @@ rtc_source_set("field_trial") {
   ]
 }
 
+rtc_source_set("perf_test") {
+  testonly = true
+  sources = [
+    "testsupport/perf_test.cc",
+    "testsupport/perf_test.h",
+  ]
+  deps = [
+    "..:webrtc_common",
+    "../api:array_view",
+    "../rtc_base:rtc_base_approved",
+  ]
+}
+
 if (is_ios) {
   rtc_source_set("test_support_objc") {
     testonly = true
@@ -122,6 +135,7 @@ if (is_ios) {
       "ios/test_support.mm",
     ]
     deps = [
+      ":perf_test",
       "../sdk:common_objc",
     ]
   }
@@ -135,8 +149,6 @@ rtc_source_set("test_support") {
     "gtest.h",
     "testsupport/packet_reader.cc",
     "testsupport/packet_reader.h",
-    "testsupport/perf_test.cc",
-    "testsupport/perf_test.h",
     "testsupport/unittest_utils.h",
   ]
 
@@ -188,6 +200,7 @@ if (rtc_include_tests) {
     ]
     deps = [
       ":field_trial",
+      ":perf_test",
       "../rtc_base:rtc_base_approved",
       "../system_wrappers:metrics_default",
       "//testing/gmock",
@@ -287,6 +300,7 @@ if (rtc_include_tests) {
   rtc_test("test_support_unittests") {
     deps = [
       ":fake_audio_device",
+      ":perf_test",
       ":rtp_test_utils",
       "../api:video_frame_api",
       "../api:video_frame_api_i420",

diff --git a/test/ios/test_support.h b/test/ios/test_support.h
@@ -17,7 +17,8 @@ namespace test {
 // This is necessary as iOS doesn't like processes without a gui
 // running for longer than a few seconds.
 void RunTestsFromIOSApp();
-void InitTestSuite(int (*test_suite)(void), int argc, char* argv[]);
+void InitTestSuite(int (*test_suite)(void), int argc, char* argv[],
+                   bool save_chartjson_result);
 
 }  // namespace test
 }  // namespace rtc

diff --git a/test/ios/test_support.mm b/test/ios/test_support.mm
@@ -11,8 +11,10 @@
 #import <UIKit/UIKit.h>
 
 #include "test/ios/test_support.h"
+#include "test/testsupport/perf_test.h"
 
 #import "sdk/objc/Framework/Classes/Common/RTCUIApplicationStatusObserver.h"
+#include "sdk/objc/Framework/Classes/Common/helpers.h"
 
 // Springboard will kill any iOS app that fails to check in after launch within
 // a given time. Starting a UIApplication before invoking TestSuite::Run
@@ -31,6 +33,7 @@
 static int (*g_test_suite)(void) = NULL;
 static int g_argc;
 static char **g_argv;
+static bool g_save_chartjson_result;
 
 @interface UIApplication (Testing)
 - (void)_terminateWithStatus:(int)status;
@@ -75,6 +78,20 @@ - (BOOL)application:(UIApplication *)application
 - (void)runTests {
   int exitStatus = g_test_suite();
 
+  if (g_save_chartjson_result) {
+    // Stores data into a json file under the app's document directory.
+    NSString* fileName = @"perf_result.json";
+    NSArray<NSString*>* outputDirectories = NSSearchPathForDirectoriesInDomains(
+        NSDocumentDirectory, NSUserDomainMask, YES);
+    if ([outputDirectories count] != 0) {
+      NSString* outputPath =
+          [outputDirectories[0] stringByAppendingPathComponent:fileName];
+
+      webrtc::test::WritePerfResults(
+          webrtc::ios::StdStringFromNSString(outputPath));
+    }
+  }
+
   // If a test app is too fast, it will exit before Instruments has has a
   // a chance to initialize and no test results will be seen.
   // TODO(crbug.com/137010): Figure out how much time is actually needed, and
@@ -93,10 +110,14 @@ - (void)runTests {
 namespace rtc {
 namespace test {
 
-void InitTestSuite(int (*test_suite)(void), int argc, char *argv[]) {
+// Note: This is not thread safe, and must be called from the same thread as
+// runTests above.
+void InitTestSuite(int (*test_suite)(void), int argc, char *argv[],
+                   bool save_chartjson_result) {
   g_test_suite = test_suite;
   g_argc = argc;
   g_argv = argv;
+  g_save_chartjson_result = save_chartjson_result;
 }
 
 void RunTestsFromIOSApp() {

diff --git a/test/test_main.cc b/test/test_main.cc
@@ -8,8 +8,6 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include <fstream>
-
 #include "rtc_base/flags.h"
 #include "rtc_base/logging.h"
 #include "system_wrappers/include/metrics_default.h"
@@ -26,6 +24,24 @@ DEFINE_string(NSTreatUnknownArgumentsAsOpen, "",
     "Intentionally ignored flag intended for iOS simulator.");
 DEFINE_string(ApplePersistenceIgnoreState, "",
     "Intentionally ignored flag intended for iOS simulator.");
+DEFINE_bool(
+    save_chartjson_result,
+    false,
+    "Store the perf results in Documents/perf_result.json in the format "
+    "described by "
+    "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+    "data-format.md.");
+
+#else
+
+DEFINE_string(
+    chartjson_result_file,
+    "",
+    "Path where the perf results should be stored in the JSON format described "
+    "by "
+    "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
+    "data-format.md.");
+
 #endif
 
 DEFINE_bool(logs, false, "print logs to stderr");
@@ -35,13 +51,6 @@ DEFINE_string(force_fieldtrials, "",
     "E.g. running with --force_fieldtrials=WebRTC-FooFeature/Enable/"
     " will assign the group Enable to field trial WebRTC-FooFeature.");
 
-DEFINE_string(
-    chartjson_result_file,
-    "",
-    "Path where the perf results should be stored it the JSON format described "
-    "by "
-    "https://github.com/catapult-project/catapult/blob/master/dashboard/docs/"
-    "data-format.md.");
 
 DEFINE_bool(help, false, "Print this message.");
 
@@ -66,21 +75,25 @@ int main(int argc, char* argv[]) {
   webrtc::test::InitFieldTrialsFromString(fieldtrials);
   webrtc::metrics::Enable();
 
+
   rtc::LogMessage::SetLogToStderr(FLAG_logs);
+
 #if defined(WEBRTC_IOS)
-  rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv);
+
+  rtc::test::InitTestSuite(RUN_ALL_TESTS, argc, argv,
+                           FLAG_save_chartjson_result);
   rtc::test::RunTestsFromIOSApp();
-#endif
+
+#else
 
   int exit_code = RUN_ALL_TESTS();
 
   std::string chartjson_result_file = FLAG_chartjson_result_file;
   if (chartjson_result_file != "") {
-    std::string json_results = webrtc::test::GetPerfResultsJSON();
-    std::fstream json_file(chartjson_result_file, std::fstream::out);
-    json_file << json_results;
-    json_file.close();
+    webrtc::test::WritePerfResults(chartjson_result_file);
   }
 
   return exit_code;
+
+#endif
 }
diff --git a/test/testsupport/perf_test.cc b/test/testsupport/perf_test.cc
@@ -12,6 +12,7 @@
 #include "rtc_base/criticalsection.h"
 
 #include <stdio.h>
+#include <fstream>
 #include <map>
 #include <sstream>
 #include <vector>
@@ -154,6 +155,13 @@ std::string GetPerfResultsJSON() {
   return GetPerfResultsLogger().ToJSON();
 }
 
+void WritePerfResults(const std::string& output_path) {
+  std::string json_results = GetPerfResultsJSON();
+  std::fstream json_file(output_path, std::fstream::out);
+  json_file << json_results;
+  json_file.close();
+}
+
 void PrintResult(const std::string& measurement,
                  const std::string& modifier,
                  const std::string& trace,

diff --git a/test/testsupport/perf_test.h b/test/testsupport/perf_test.h
@@ -61,8 +61,10 @@ void PrintResultList(const std::string& measurement,
                      const std::string& units,
                      bool important);
 
-// Get all perf results to date in a JSON format as described in
+// Write  all perf results to date to a JSON file formatted as described in
 // https://github.com/catapult-project/catapult/blob/master/dashboard/docs/data-format.md
+void WritePerfResults(const std::string& output_path);
+
 std::string GetPerfResultsJSON();
 
 // You shouldn't use this function. It's only used to test the functions above.

diff --git a/tools_webrtc/ios/tests/perf_tests.json b/tools_webrtc/ios/tests/perf_tests.json
@@ -3,7 +3,7 @@
     {
       "app": "webrtc_perf_tests",
       "test args": [
-        "--perf_results_json_path=Documents/perf_result.json"
+        "--save_chartjson_result"
       ],
       "bot_id": "build15-a7",
       "pool": "WebRTC"

diff --git a/tools_webrtc/ios/tests/perf_trybot_tests.json b/tools_webrtc/ios/tests/perf_trybot_tests.json
@@ -2,6 +2,10 @@
   "tests": [
     {
       "app": "webrtc_perf_tests",
+      "test args": [
+        "--gtest_filter=AudioEncoderOpusComplexityAdaptationTest.AdaptationOn",
+        "--save_chartjson_result"
+      ],
       "bot_id": "build16-a7",
       "pool": "WebRTC"
     }

diff --git a/video/BUILD.gn b/video/BUILD.gn
@@ -298,6 +298,7 @@ if (rtc_include_tests) {
       "../system_wrappers:metrics_default",
       "../test:direct_transport",
       "../test:field_trial",
+      "../test:perf_test",
       "../test:rtp_test_utils",
       "../test:test_common",
       "../test:test_support",