configurable monitor storage (erda-project#2545)

* configurable monitor storage * configurable monitor storage * log storage v1 * remove cassandra from topology * update monitor config * add some test for log persist v1
mesosxzan · Oct 26, 2021 · 39714dc · 39714dc
1 parent fa21bf2
commit 39714dc
Show file tree

Hide file tree

Showing 29 changed files with 1,526 additions and 146 deletions.
diff --git a/cmd/monitor/streaming/main.go b/cmd/monitor/streaming/main.go
@@ -24,6 +24,7 @@ import (
 	_ "github.com/erda-project/erda/modules/core/monitor/event/persist"
 	_ "github.com/erda-project/erda/modules/core/monitor/event/storage/elasticsearch"
 	_ "github.com/erda-project/erda/modules/core/monitor/log/persist"
+	_ "github.com/erda-project/erda/modules/core/monitor/log/persist/v1"
 	_ "github.com/erda-project/erda/modules/core/monitor/log/storage/elasticsearch"
 	_ "github.com/erda-project/erda/modules/core/monitor/metric/persist"
 	_ "github.com/erda-project/erda/modules/core/monitor/metric/storage/elasticsearch"

diff --git a/conf/monitor/monitor/monitor.yaml b/conf/monitor/monitor/monitor.yaml
@@ -21,6 +21,7 @@ elasticsearch@logs:
   password: "${LOGS_ES_SECURITY_PASSWORD}"
 
 cassandra:
+  _enable: ${CASSANDRA_ENABLE:true}
   host: "${CASSANDRA_ADDR:localhost:9042}"
   security: ${CASSANDRA_SECURITY_ENABLE:false}
   username: ${CASSANDRA_SECURITY_USERNAME}
@@ -54,7 +55,14 @@ redis:
 permission:
 
 # log
+elasticsearch@log:
+  _enable: ${LOG_ELASTICSEARCH_ENABLE:false}
+  urls: "${LOG_ELASTICSEARCH_URL:http://localhost:9200}"
+  security: ${LOG_ELASTICSEARCH_SECURITY_ENABLE:false}
+  username: "${LOG_ELASTICSEARCH_SECURITY_USERNAME}"
+  password: "${LOG_ELASTICSEARCH_SECURITY_PASSWORD}"
 elasticsearch.index.loader@log:
+  _enable: ${QUERY_LOG_FROM_ES_ENABLE:true}
   load_mode: "LoadWithCache"
   index_reload_interval: "1m"
   query_index_time_range: true
@@ -66,13 +74,16 @@ elasticsearch.index.loader@log:
         - "<org>-{number}"
         - "<org>.<key>-{number}"
 storage-retention-strategy@log:
+  _enable: ${QUERY_LOG_FROM_ES_ENABLE:true}
   default_ttl: "${LOG_TTL:168h}"
   load_from_database: true
   ttl_reload_interval: "3m"
 elasticsearch.index.retention-strategy@log:
+  _enable: ${QUERY_LOG_FROM_ES_ENABLE:true}
   key_patterns: 
     - "erda-logs-<org>.<key>-{number}"
 elasticsearch.index.cleaner@log:
+  _enable: ${QUERY_LOG_FROM_ES_ENABLE:true}
   check_interval: "30m"
   print_onluy: true
   disk_clean:
@@ -89,24 +100,33 @@ elasticsearch.index.cleaner@log:
       - index: "erda-logs-<org>.<key>-{number}"
         alias: "erda-logs-<org>.<key>-rollover"
 log-storage-elasticsearch:
+  _enable: ${QUERY_LOG_FROM_ES_ENABLE:true}
   query_timeout: "1m"
   read_page_size: 200
 kubernetes-logs-storage:
+  _enable: ${QUERY_LOG_FROM_K8S_API_ENABLE:true}
   client_cache_size: 128
   client_cache_expiration: "10m"
   pod_info_cache_size: 256
   pod_info_cache_expiration: "3h"
   buffer_lines: 200
 log-storage-cassandra:
-  _enable: ${QUERY_LOG_IN_CASSANDRA_ENABLE:true}
+  _enable: ${QUERY_LOG_FROM_CASSANDRA_ENABLE:false}
   read_page_size: 201
   cassandra:
     keyspace:
       name: "spot_prod"
 erda.core.monitor.log.query:
 
 # event
+elasticsearch@event:
+  _enable: ${EVENT_ELASTICSEARCH_ENABLE:false}
+  urls: "${EVENT_ELASTICSEARCH_URL:http://localhost:9200}"
+  security: ${EVENT_ELASTICSEARCH_SECURITY_ENABLE:false}
+  username: "${EVENT_ELASTICSEARCH_SECURITY_USERNAME}"
+  password: "${EVENT_ELASTICSEARCH_SECURITY_PASSWORD}"
 elasticsearch.index.loader@event:
+  _enable: ${QUERY_EVENT_FROM_ES_ENABLE:true}
   load_mode: "LoadWithCache"
   index_reload_interval: "1m"
   query_index_time_range: true
@@ -117,13 +137,16 @@ elasticsearch.index.loader@event:
         - "<event>-<namespace>-{number}"
         - "<event>-<namespace>.<key>-{number}"
 storage-retention-strategy@event:
+  _enable: ${QUERY_EVENT_FROM_ES_ENABLE:true}
   default_ttl: "${EVENT_TTL:168h}"
-  load_from_database: true
+  load_from_database: false
   ttl_reload_interval: "3m"
 elasticsearch.index.retention-strategy@event:
+  _enable: ${QUERY_EVENT_FROM_ES_ENABLE:true}
   key_patterns:
     - "erda-events-<event>-<namespace>.<key>-{number}"
 elasticsearch.index.cleaner@event:
+  _enable: ${QUERY_EVENT_FROM_ES_ENABLE:true}
   check_interval: "30m"
   print_onluy: true
   disk_clean:
@@ -140,6 +163,7 @@ elasticsearch.index.cleaner@event:
       - index: "erda-events-<event>-<namespace>.<key>-{number}"
         alias: "erda-events-<event>-<namespace>.<key>-rollover"
 event-storage-elasticsearch:
+  _enable: ${QUERY_EVENT_FROM_ES_ENABLE:true}
   query_timeout: "1m"
   read_page_size: 50
 
@@ -218,12 +242,6 @@ node-topo:
 #apm providers
 apm-runtime:
 apm-topology:
-  cassandra:
-    keyspace:
-      name: "spot_prod"
-      replication:
-        class: ${CASSANDRA_KEYSPACE_REPLICATION_CLASS:SimpleStrategy}
-        factor: ${CASSANDRA_KEYSPACE_REPLICATION_FACTOR:2}
 apm-report:
 
 erda.core.monitor.alert:

diff --git a/conf/monitor/streaming/streaming.yaml b/conf/monitor/streaming/streaming.yaml
@@ -11,13 +11,6 @@ kafka:
     options:
       go.produce.channel.size: ${KAFKA_PRODUCE_SIZE:200000}
 
-cassandra:
-  host: "${CASSANDRA_ADDR:localhost:9042}"
-  security: ${CASSANDRA_SECURITY_ENABLE:false}
-  username: ${CASSANDRA_SECURITY_USERNAME}
-  password: ${CASSANDRA_SECURITY_PASSWORD}
-  timeout: "${CASSANDRA_TIMEOUT:3s}"
-
 mysql:
   host: "${MYSQL_HOST:localhost}"
   port: ${MYSQL_PORT:3306}
@@ -35,23 +28,22 @@ etcd:
 etcd-election@index:
   root_path: "/erda/monitor-index-rollover-election"
 
-elasticsearch:
-  urls: "${ES_URL:http://localhost:9200}"
-  security: ${ES_SECURITY_ENABLE:false}
-  username: "${ES_SECURITY_USERNAME}"
-  password: "${ES_SECURITY_PASSWORD}"
-
-elasticsearch.index.initializer:
+# elasticsearch for log
+elasticsearch@log:
+  _enable: ${LOG_ELASTICSEARCH_ENABLE:false}
+  urls: "${LOG_ELASTICSEARCH_URL:http://localhost:9200}"
+  security: ${LOG_ELASTICSEARCH_SECURITY_ENABLE:false}
+  username: "${LOG_ELASTICSEARCH_SECURITY_USERNAME}"
+  password: "${LOG_ELASTICSEARCH_SECURITY_PASSWORD}"
+
+elasticsearch.index.initializer@log:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   templates:
-    - name: "spot_metric_template"
-      path: "${CONFIG_PATH}/metric_index_template.json"
     - name: "erda-logs"
       path: "${CONFIG_PATH}/log_index_template.json"
-    - name: "erda-events"
-      path: "${CONFIG_PATH}/event_index_template.json"
 
-# elasticsearch for log
 elasticsearch.index.loader@log:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   load_mode: "LoadFromElasticSearchOnly"
   index_reload_interval: "1m"
   match:
@@ -61,6 +53,7 @@ elasticsearch.index.loader@log:
         - "<org>.<key>-{number}"
 
 elasticsearch.index.creator@log:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   patterns:
     - first_index: "erda-logs-<org>-000001"
       alias: "erda-logs-<org>-rollover"
@@ -69,6 +62,7 @@ elasticsearch.index.creator@log:
   remove_conflicting_indices: true
 
 elasticsearch.index.rollover@log:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   check_interval: "30s"
   body_file: "${CONFIG_PATH}/index_rollover.json"
   patterns:
@@ -80,12 +74,15 @@ elasticsearch.index.rollover@log:
 storage-retention-strategy@log:
   load_from_database: true
   ttl_reload_interval: "3m"
+  default_ttl: "${LOG_TTL:168h}"
 
 log-storage-elasticsearch:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   write_timeout: "1m"
   index_type: "logs"
 
 log-persist:
+  _enable: ${WRITE_LOG_TO_ES_ENABLE:true}
   input:
     topics: "${LOG_TOPICS:spot-container-log,spot-job-log}"
     group: "${LOG_GROUP_ID:erda-logs-dev}"
@@ -95,8 +92,64 @@ log-persist:
   parallelism: ${LOG_PERSIST_PARALLELISM:3}
   print_invalid_log: false
 
+cassandra:
+  _enable: ${CASSANDRA_ENABLE:true}
+  host: "${CASSANDRA_ADDR:localhost:9042}"
+  security: ${CASSANDRA_SECURITY_ENABLE:false}
+  username: ${CASSANDRA_SECURITY_USERNAME}
+  password: ${CASSANDRA_SECURITY_PASSWORD}
+  timeout: "${CASSANDRA_TIMEOUT:3s}"
+etcd-mutex:
+  _enable: ${WRITE_LOG_TO_CASSANDRA_ENABLE:false}
+  root_path: "/erda/streaming"
+log-persist-v1:
+  _enable: ${WRITE_LOG_TO_CASSANDRA_ENABLE:false}
+  input:
+    topics: "${LOG_TOPICS:spot-container-log,spot-job-log}"
+    group: "${LOG_GROUP_ID:spot-monitor-log-dev}"
+    parallelism: ${LOG_CONSUMERS:3}
+    options:
+      auto.offset.reset: "${KAFKA_AUTO_OFFSET_RESET:latest}"
+      auto.commit.interval.ms: "${KAFKA_AUTO_COMMIT_INTERVAL_MS:1000}"
+      queued.max.messages.kbytes: ${LOGS_STORE_INPUT_CONSUMER_QUEUE_SIZE_KB:102400} # 300MB = 100MB * parallelism
+  output:
+    id_keys: "${LOG_ID_KEYS:TERMINUS_DEFINE_TAG,terminus_define_tag,MESOS_TASK_ID,mesos_task_id}"
+    log_schema:
+      org_refresh_interval: "${LOG_SCHEMA_ORG_REFRESH_INTERVAL:3m}"
+    cassandra:
+      writer_config:
+        parallelism: ${CASSANDRA_PARALLELISM:3}
+        batch:
+          size: ${CASSANDRA_BATCH_SIZE:50}
+          timeout: ${CASSANDRA_BATCH_TIMEOUT:10s}
+        retry: -1  # block forever. kafka will handle the issue of stream block
+      session_config:
+        keyspace:
+          name: "spot_prod"
+          auto: ${LOG_STORE_CASSANDRA_SESSION_KEYSPACE_AUTO:true} # 自动创建 keyspace
+          replication:
+            class: ${CASSANDRA_KEYSPACE_REPLICATION_CLASS:SimpleStrategy}
+            factor: ${CASSANDRA_KEYSPACE_REPLICATION_FACTOR:2}
+        reconnection:
+          check_interval: ${LOG_STORE_CASSANDRA_RECONNECTION_CHECK_INTERVAL:3m}
+      gc_grace_seconds: 86400
+
 # elasticsearch for event
+elasticsearch@event:
+  _enable: ${EVENT_ELASTICSEARCH_ENABLE:false}
+  urls: "${EVENT_ELASTICSEARCH_URL:http://localhost:9200}"
+  security: ${EVENT_ELASTICSEARCH_SECURITY_ENABLE:false}
+  username: "${EVENT_ELASTICSEARCH_SECURITY_USERNAME}"
+  password: "${EVENT_ELASTICSEARCH_SECURITY_PASSWORD}"
+
+elasticsearch.index.initializer@event:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
+  templates:
+    - name: "erda-events"
+      path: "${CONFIG_PATH}/event_index_template.json"
+
 elasticsearch.index.loader@event:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
   load_mode: "LoadFromElasticSearchOnly"
   index_reload_interval: "1m"
   match:
@@ -106,6 +159,7 @@ elasticsearch.index.loader@event:
         - "<event>-<namespace>.<key>-{number}"
 
 elasticsearch.index.creator@event:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
   patterns:
     - first_index: "erda-events-<event>-<namespace>-000001"
       alias: "erda-events-<event>-<namespace>-rollover"
@@ -114,6 +168,7 @@ elasticsearch.index.creator@event:
   remove_conflicting_indices: true
 
 elasticsearch.index.rollover@event:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
   check_interval: "30s"
   body_file: "${CONFIG_PATH}/index_rollover.json"
   patterns:
@@ -123,14 +178,17 @@ elasticsearch.index.rollover@event:
       alias: "erda-events-<event>-<namespace>.<key>-rollover"
 
 storage-retention-strategy@event:
-  load_from_database: true
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
+  load_from_database: false
   ttl_reload_interval: "3m"
 
 event-storage-elasticsearch:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
   write_timeout: "1m"
   index_type: "events"
 
 event-persist:
+  _enable: ${WRITE_EVENT_TO_ES_ENABLE:true}
   input:
     topics: "${EVENT_TOPICS:spot-events}"
     group: "${EVENT_GROUP_ID:erda-events-dev}"
@@ -140,6 +198,17 @@ event-persist:
   print_invalid_event: false
 
 # elasticsearch for metric
+elasticsearch:
+  urls: "${ES_URL:http://localhost:9200}"
+  security: ${ES_SECURITY_ENABLE:false}
+  username: "${ES_SECURITY_USERNAME}"
+  password: "${ES_SECURITY_PASSWORD}"
+
+elasticsearch.index.initializer@metric:
+  templates:
+    - name: "spot_metric_template"
+      path: "${CONFIG_PATH}/metric_index_template.json"
+
 elasticsearch.index.loader@metric:
   load_mode: "LoadFromElasticSearchOnly"
   index_reload_interval: "1m"

diff --git a/conf/msp/msp.yaml b/conf/msp/msp.yaml
@@ -42,6 +42,7 @@ redis:
   sentinels_addr: "${REDIS_SENTINELS_ADDR}"
 
 cassandra:
+  _enable: ${CASSANDRA_ENABLE:true}
   host: "${CASSANDRA_ADDR:localhost:9042}"
   security: ${CASSANDRA_SECURITY_ENABLE:false}
   username: ${CASSANDRA_SECURITY_USERNAME}

diff --git a/go.mod b/go.mod
@@ -32,7 +32,7 @@ require (
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/elastic/cloud-on-k8s v0.0.0-20210205172912-5ce0eca90c60
-	github.com/erda-project/erda-infra v0.0.0-20211025111229-a1e623f59b33
+	github.com/erda-project/erda-infra v0.0.0-20211026062148-67c7a77cea7a
 	github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b
 	github.com/erda-project/erda-proto-go v0.0.0
 	github.com/erda-project/erda-sourcecov v0.1.0

diff --git a/go.sum b/go.sum
@@ -447,8 +447,8 @@ github.com/erda-project/elastic v0.0.1-ex h1:5ajfxQ5S5YjpzFqY9LzL9hiKWCn6q/JDT4n
 github.com/erda-project/elastic v0.0.1-ex/go.mod h1:iAVsas6fcmt9pxtge1+dErMhecv+RLSXlD4rnZRJVW0=
 github.com/erda-project/erda-infra v0.0.0-20210706133120-0a742437972c/go.mod h1:TUQYSZ60w9dk7m0q3U3AVg7U74APj/sdEVvRWR3wYv8=
 github.com/erda-project/erda-infra v0.0.0-20210929085513-aaf78ecd92ac/go.mod h1:Y+6YRve14bMNFVrG5wxwC70gxH8cwNiZ4ukhhhnq8EA=
-github.com/erda-project/erda-infra v0.0.0-20211025111229-a1e623f59b33 h1:gsoO6DqImUMuhfFidkeY8/qyTmIIN03vewzU1ZAzgA0=
-github.com/erda-project/erda-infra v0.0.0-20211025111229-a1e623f59b33/go.mod h1:YWl4gg86UMFLrr160Jba/4yhovqC/t7Vi6cpGqx1TrY=
+github.com/erda-project/erda-infra v0.0.0-20211026062148-67c7a77cea7a h1:bdGZqbMwKEoxRROen0EPV0yeMwvRKfjraaV91HcoHc0=
+github.com/erda-project/erda-infra v0.0.0-20211026062148-67c7a77cea7a/go.mod h1:YWl4gg86UMFLrr160Jba/4yhovqC/t7Vi6cpGqx1TrY=
 github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b h1:GWf2ChasZFerFwQoTokIvjJLWH57ligTSLD2hUb7UWk=
 github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b/go.mod h1:H/f81Thef2Tnz4nUeLt0r4VwHdOznthpyXBwT9vDWo0=
 github.com/erda-project/erda-proto-go v0.0.0-20210820073528-bcd612c2efb7/go.mod h1:rSETXX3nKxxIhgrVn7fKDM3mla1nNlWcPz4AkepixaU=