Fix hard coded constraint on pipeline name for metrics

Fixes elastic#11777

x-pack/lib/monitoring/inputs/metrics.rb

@@ -211,9 +211,18 @@ def fetch_global_stats
     end
 
     def extract_cluster_uuids(stats)
-      result = stats.extract_metrics([:stats, :pipelines, :main, :config], :cluster_uuids)
-      if result && !result[:cluster_uuids].empty?
-        cluster_uuids = result[:cluster_uuids]
+      cluster_uuids = []
+      agent.running_pipelines.each do |pipeline_id, _|
+        unless pipeline_id.to_sym == :".monitoring-logstash"
+          path = [:stats, :pipelines, pipeline_id.to_sym, :config]
+          found_cluster_uuids = stats.extract_metrics(path, :cluster_uuids)
+          if found_cluster_uuids && !found_cluster_uuids[:cluster_uuids].empty?
+            cluster_uuids |= found_cluster_uuids[:cluster_uuids]
+          end
+        end
+      end
+
+      unless cluster_uuids.empty?
         @logger.info("Found cluster_uuids from elasticsearch output plugins", :cluster_uuids => cluster_uuids)
         if LogStash::SETTINGS.set?("monitoring.cluster_uuid")
           @logger.warn("Found monitoring.cluster_uuid setting configured in logstash.yml while using the ones discovered from elasticsearch output plugins, ignoring setting monitoring.cluster_uuid")

x-pack/lib/monitoring/inputs/timer_task_logger.rb

@@ -14,7 +14,7 @@ def update(run_at, result, exception)
         if exception.is_a?(Concurrent::TimeoutError)
           logger.debug("metric shipper took too much time to complete", :exception => exception.class, :message => exception.message)
         else
-          logger.error("metric shipper exception", :exception => exception.class, :message => exception.message)
+          logger.error("metric shipper exception", :exception => exception.class, :message => exception.message, :backtrace => exception.backtrace)
         end
       end
     end

x-pack/qa/integration/monitoring/direct_shipping_spec.rb

@@ -3,6 +3,7 @@
 # you may not use this file except in compliance with the Elastic License.
 
 require_relative "../spec_helper"
+require 'securerandom'
 
 describe "Direct shipping" do
 
@@ -13,7 +14,7 @@
 
     config = "input { generator { count => 100 } tcp { port => 6000 } } output { null {} }"
 
-    @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1", {
+    @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1  --pipeline.id #{SecureRandom.hex(8)}", {
       :settings => {
         "monitoring.enabled" => true,
         "monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"],