Remove deprecated source pipelines (elastic#1174)

With this commit we remove the deprecated pipelines
`from-sources-skip-build` and `from-sources-complete` which are replaced
by the pipeline `from-sources`.

docs/migrate.rst

@@ -11,6 +11,11 @@ Previously, Rally has implicitly used the unit ``docs`` for bulk operations. Wit
 
     esrally.exceptions.DataError: Parameter source for operation 'bulk-index' did not provide the mandatory parameter 'unit'. Add it to your parameter source and try again.
 
+Pipelines from-sources-complete and from-sources-skip-build are removed
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The previously deprecated pipelines ``from-sources-complete`` and ``from-sources-skip-build`` have been removed. Specify ``--pipeline=from-sources`` instead.
+
 Migrating to Rally 2.0.4
 ------------------------
 

docs/pipelines.rst

@@ -12,8 +12,6 @@ You can get a list of all pipelines with ``esrally list pipelines``::
     Name                     Description
     -----------------------  ---------------------------------------------------------------------------------------------
     from-sources             Builds and provisions Elasticsearch, runs a benchmark and reports results.
-    from-sources-complete    Builds and provisions Elasticsearch, runs a benchmark and reports results [deprecated].
-    from-sources-skip-build  Provisions Elasticsearch (skips the build), runs a benchmark and reports results [deprecated].
     from-distribution        Downloads an Elasticsearch distribution, provisions it, runs a benchmark and reports results.
     benchmark-only           Assumes an already running Elasticsearch instance, runs a benchmark and reports results
 
@@ -67,13 +65,3 @@ Artifacts are cached for seven days by default in ``~/.rally/benchmarks/distribu
 
 * ``cache`` (default: ``True``): Set to ``False`` to disable artifact caching.
 * ``cache.days`` (default: ``7``): The maximum age in days of an artifact before it gets evicted from the artifact cache.
-
-from-sources-complete
-~~~~~~~~~~~~~~~~~~~~~
-
-This deprecated pipeline is an alias for ``from-sources`` and is only provided for backwards-compatibility. Use ``from-sources`` instead.
-
-from-sources-skip-build
-~~~~~~~~~~~~~~~~~~~~~~~
-
-This deprecated pipeline is similar to ``from-sources-complete`` except that it assumes you have built the binary once. Use ``from-sources`` instead which caches built artifacts automatically.

esrally/mechanic/supplier.py

@@ -176,7 +176,7 @@ def _supply_requirements(sources, distribution, build, plugins, revisions, distr
                 # this plugin always needs to built unless we explicitly disable it; we cannot solely rely on the Rally pipeline.
                 # We either have:
                 #
-                # * --pipeline=from-sources-skip-build --distribution-version=X.Y.Z where the plugin should not be built but ES should be
+                # * --pipeline=from-sources --distribution-version=X.Y.Z where the plugin should not be built but ES should be
                 #   a distributed version.
                 # * --distribution-version=X.Y.Z --revision="my-plugin:abcdef" where the plugin should be built from sources.
                 # pylint: disable=consider-using-ternary
@@ -394,7 +394,7 @@ def resolve_binary(self):
                                 self.template_renderer.render(self.car.mandatory_var("system.artifact_path_pattern")))
             return glob.glob(path)[0]
         except IndexError:
-            raise SystemSetupError("Couldn't find a tar.gz distribution. Please run Rally with the pipeline 'from-sources-complete'.")
+            raise SystemSetupError("Couldn't find a tar.gz distribution. Please run Rally with the pipeline 'from-sources'.")
 
 
 class PluginFileNameResolver:
@@ -463,7 +463,7 @@ def resolve_binary(self):
             name = glob.glob("%s/%s/*.zip" % (self.plugin_src_dir, artifact_path))[0]
             return "file://%s" % name
         except IndexError:
-            raise SystemSetupError("Couldn't find a plugin zip file for [%s]. Please run Rally with the pipeline 'from-sources-complete'." %
+            raise SystemSetupError("Couldn't find a plugin zip file for [%s]. Please run Rally with the pipeline 'from-sources'." %
                                    self.plugin.name)
 
 
@@ -494,7 +494,7 @@ def resolve_binary(self):
             name = glob.glob("%s/plugins/%s/build/distributions/*.zip" % (self.es_src_dir, self.plugin.name))[0]
             return "file://%s" % name
         except IndexError:
-            raise SystemSetupError("Couldn't find a plugin zip file for [%s]. Please run Rally with the pipeline 'from-sources-complete'." %
+            raise SystemSetupError("Couldn't find a plugin zip file for [%s]. Please run Rally with the pipeline 'from-sources'." %
                                    self.plugin.name)
 
 

esrally/racecontrol.py

@@ -279,19 +279,6 @@ def from_sources(cfg):
     return race(cfg, sources=True, build=True)
 
 
-def from_sources_complete(cfg):
-    console.warn("The pipeline from-sources-complete is deprecated. Use the pipeline \"from-sources\" instead.")
-    return from_sources(cfg)
-
-
-def from_sources_skip_build(cfg):
-    console.warn("The pipeline from-sources-skip-build is deprecated. Rally caches artifacts now automatically. "
-                 "Use the pipeline \"from-sources\" instead")
-    port = cfg.opts("provisioning", "node.http.port")
-    set_default_hosts(cfg, port=port)
-    return race(cfg, sources=True, build=False)
-
-
 def from_distribution(cfg):
     port = cfg.opts("provisioning", "node.http.port")
     set_default_hosts(cfg, port=port)
@@ -313,12 +300,6 @@ def docker(cfg):
 Pipeline("from-sources",
          "Builds and provisions Elasticsearch, runs a benchmark and reports results.", from_sources)
 
-Pipeline("from-sources-complete",
-         "Builds and provisions Elasticsearch, runs a benchmark and reports results [deprecated].", from_sources_complete)
-
-Pipeline("from-sources-skip-build",
-         "Provisions Elasticsearch (skips the build), runs a benchmark and reports results [deprecated].", from_sources_skip_build)
-
 Pipeline("from-distribution",
          "Downloads an Elasticsearch distribution, provisions it, runs a benchmark and reports results.", from_distribution)
 

it/sources_test.py

@@ -25,5 +25,5 @@ def test_sources(cfg):
                         "--challenge=append-no-conflicts --car=4gheap --elasticsearch-plugins=analysis-icu") == 0
 
     it.wait_until_port_is_free()
-    assert it.race(cfg, "--pipeline=from-sources-skip-build --track=geonames --test-mode "
+    assert it.race(cfg, "--pipeline=from-sources --track=geonames --test-mode "
                         "--challenge=append-no-conflicts-index-only --car=\"4gheap,ea\"") == 0

tests/mechanic/supplier_test.py

@@ -622,7 +622,7 @@ def test_derive_supply_requirements_es_source_build(self):
         self.assertDictEqual({"elasticsearch": ("source", "abc", True)}, requirements)
 
     def test_derive_supply_requirements_es_source_skip(self):
-        # corresponds to --pipeline=from-sources-skip-build
+        # corresponds to --pipeline=from-sources
         requirements = supplier._supply_requirements(
             sources=True, distribution=False, build=False, plugins=[], revisions={"elasticsearch": "current"}, distribution_version=None)
         self.assertDictEqual({"elasticsearch": ("source", "current", False)}, requirements)
@@ -665,7 +665,7 @@ def test_derive_supply_requirements_es_distribution_and_plugin_source_build(self
         }, requirements)
 
     def test_derive_supply_requirements_es_distribution_and_plugin_source_skip(self):
-        # corresponds to --from-sources-skip-build --revision="community-plugin:current" --distribution-version="6.0.0"
+        # corresponds to --from-sources --revision="community-plugin:current" --distribution-version="6.0.0"
         core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
         external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False)
 
@@ -713,7 +713,7 @@ def test_create_suppliers_for_es_distribution_plugin_source_skip(self):
         core_plugin = team.PluginDescriptor("analysis-icu", core_plugin=True)
         external_plugin = team.PluginDescriptor("community-plugin", core_plugin=False, variables={"enabled": True})
 
-        # --pipeline=from-sources-skip-build
+        # --pipeline=from-sources
         composite_supplier = supplier.create(cfg, sources=True, distribution=False, build=False, car=car, plugins=[
             core_plugin,
             external_plugin
@@ -744,7 +744,7 @@ def test_create_suppliers_for_es_missing_distribution_plugin_source_skip(self):
 
         car = team.Car("default", root_path=None, config_paths=[])
 
-        # --from-sources-skip-build --revision="community-plugin:current" (distribution version is missing!)
+        # --from-sources --revision="community-plugin:current" (distribution version is missing!)
         with self.assertRaises(exceptions.SystemSetupError) as ctx:
             supplier.create(cfg, sources=True, distribution=False, build=False, car=car, plugins=[
                 core_plugin,

tests/metrics_test.py

@@ -1513,7 +1513,7 @@ def test_calculate_global_stats(self):
         cfg.add(config.Scope.application, "mechanic", "car.params", {})
         cfg.add(config.Scope.application, "mechanic", "plugin.params", {})
         cfg.add(config.Scope.application, "race", "user.tag", "")
-        cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
+        cfg.add(config.Scope.application, "race", "pipeline", "from-sources")
         cfg.add(config.Scope.application, "track", "params", {})
 
         index1 = track.Task(name="index #1", operation=track.Operation(name="index", operation_type=track.OperationType.Bulk, params=None))
@@ -1596,7 +1596,7 @@ def test_calculate_system_stats(self):
         cfg.add(config.Scope.application, "mechanic", "car.params", {})
         cfg.add(config.Scope.application, "mechanic", "plugin.params", {})
         cfg.add(config.Scope.application, "race", "user.tag", "")
-        cfg.add(config.Scope.application, "race", "pipeline", "from-sources-skip-build")
+        cfg.add(config.Scope.application, "race", "pipeline", "from-sources")
         cfg.add(config.Scope.application, "track", "params", {})
 
         index = track.Task(name="index #1", operation=track.Operation(name="index", operation_type=track.OperationType.Bulk, params=None))

tests/racecontrol_test.py

@@ -52,10 +52,6 @@ def unittest_pipeline():
 def test_finds_available_pipelines():
     expected = [
         ["from-sources", "Builds and provisions Elasticsearch, runs a benchmark and reports results."],
-        ["from-sources-complete",
-         "Builds and provisions Elasticsearch, runs a benchmark and reports results [deprecated]."],
-        ["from-sources-skip-build",
-         "Provisions Elasticsearch (skips the build), runs a benchmark and reports results [deprecated]."],
         ["from-distribution",
          "Downloads an Elasticsearch distribution, provisions it, runs a benchmark and reports results."],
         ["benchmark-only", "Assumes an already running Elasticsearch instance, runs a benchmark and reports results"],