Raise an error if race is not found by id

Previously Rally was lenient when a user was searching for a race by id
but it was not found. This leads to follow-up errors that are hard to
understand. With this commit we raise an error instead if a race is not
found by its id.

Relates elastic#774

esrally/exceptions.py

@@ -76,3 +76,7 @@ class TrackConfigError(RallyError):
     that can't be set
     """
     pass
+
+
+class NotFound(RallyError):
+    pass

esrally/metrics.py

@@ -1417,7 +1417,7 @@ def find_by_trial_id(self, trial_id):
             races = self._to_races([race_file])
             if races:
                 return races[0]
-        return None
+        raise exceptions.NotFound("No race with trial id [{}]".format(trial_id))
 
     def _to_races(self, results):
         import json
@@ -1521,8 +1521,11 @@ def find_by_trial_id(self, trial_id):
             hits = hits["value"]
         if hits == 1:
             return Race.from_dict(result["hits"]["hits"][0]["_source"])
+        elif hits > 1:
+            raise exceptions.RallyAssertionError(
+                "Expected exactly one race to match trial id [{}] but there were [{}] matches.".format(trial_id, hits))
         else:
-            return None
+            raise exceptions.NotFound("No race with trial id [{}]".format(trial_id))
 
 
 class EsResultsStore:

tests/metrics_test.py

@@ -788,6 +788,52 @@ def setUp(self):
         # get hold of the mocked client...
         self.es_mock = self.race_store.client
 
+    def test_find_existing_race_by_trial_id(self):
+        self.es_mock.search.return_value = {
+            "hits": {
+                "total": {
+                    "value": 1,
+                    "relation": "eq"
+                },
+                "hits": [
+                    {
+                        "_source": {
+                            "rally-version": "0.4.4",
+                            "environment": "unittest",
+                            "trial-id": EsRaceStoreTests.TRIAL_ID,
+                            "trial-timestamp": "20160131T000000Z",
+                            "pipeline": "from-sources",
+                            "track": "unittest",
+                            "challenge": "index",
+                            "track-revision": "abc1",
+                            "car": "defaults",
+                            "results": {
+                                "young_gc_time": 100,
+                                "old_gc_time": 5,
+                            }
+                        }
+                    }
+                ]
+            }
+        }
+
+        race = self.race_store.find_by_trial_id(trial_id=EsRaceStoreTests.TRIAL_ID)
+        self.assertEqual(race.trial_id, EsRaceStoreTests.TRIAL_ID)
+
+    def test_does_not_find_missing_race_by_trial_id(self):
+        self.es_mock.search.return_value = {
+            "hits": {
+                "total": {
+                    "value": 0,
+                    "relation": "eq"
+                },
+                "hits": []
+            }
+        }
+
+        with self.assertRaisesRegex(exceptions.NotFound, r"No race with trial id \[.*\]"):
+            self.race_store.find_by_trial_id(trial_id="some invalid trial id")
+
     def test_store_race(self):
         schedule = [
             track.Task("index #1", track.Operation("index", track.OperationType.Bulk))
@@ -1284,6 +1330,11 @@ def setUp(self):
         self.cfg.add(config.Scope.application, "system", "trial.id", FileRaceStoreTests.TRIAL_ID)
         self.race_store = metrics.FileRaceStore(self.cfg)
 
+    def test_race_not_found(self):
+        with self.assertRaisesRegex(exceptions.NotFound, r"No race with trial id \[.*\]"):
+            # did not store anything yet
+            self.race_store.find_by_trial_id(FileRaceStoreTests.TRIAL_ID)
+
     def test_store_race(self):
         schedule = [
             track.Task("index #1", track.Operation("index", track.OperationType.Bulk))