forked from erda-project/erda
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: implement of clickhouse creator,initializer,loader,writer,stora…
…ge (erda-project#4431) * feat: implement of clickhouse creator,initializer,loader * feat: implement of clickhouse writer, storage * feature: adjust clickhouse log configuration * feature: add clickhouse config sections * merge master
- Loading branch information
Showing
23 changed files
with
1,076 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| // 创建数据库 | ||
| CREATE DATABASE IF NOT EXISTS <database> ON CLUSTER '{cluster}'; |
38 changes: 38 additions & 0 deletions
38
conf/monitor/streaming/clickhouse/logs_ddl_create_default_tables.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| // 创建日志表 | ||
| CREATE TABLE IF NOT EXISTS <database>.logs ON CLUSTER '{cluster}' | ||
| ( | ||
| `_id` String, | ||
| `timestamp` DateTime64(9,'Asia/Shanghai'), | ||
| `source` String, | ||
| `id` String, | ||
| `org_name` String, | ||
| `stream` String, | ||
| `offset` Int64, | ||
| `content` String, | ||
| `tags` Map(String,String), | ||
| INDEX idx__id(_id) TYPE minmax GRANULARITY 1 | ||
| ) | ||
| ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}-{shard}/logs', '{replica}') | ||
| PARTITION BY toYYYYMMDD(timestamp) | ||
| ORDER BY (org_name, timestamp, id) | ||
| TTL toDateTime(timestamp) + INTERVAL 7 DAY; | ||
|
|
||
| // 将常用字段添加为物化列 | ||
| ALTER TABLE <database>.logs ON CLUSTER '{cluster}' | ||
| ADD COLUMN IF NOT EXISTS `tags.trace_id` String MATERIALIZED tags['trace_id'], | ||
| ADD COLUMN IF NOT EXISTS `tags.level` String MATERIALIZED tags['level'], | ||
| ADD COLUMN IF NOT EXISTS `tags.application_name` String MATERIALIZED tags['application_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.service_name` String MATERIALIZED tags['service_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.pod_name` String MATERIALIZED tags['pod_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.pod_ip` String MATERIALIZED tags['pod_ip'], | ||
| ADD COLUMN IF NOT EXISTS `tags.container_name` String MATERIALIZED tags['container_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.container_id` String MATERIALIZED tags['container_id']; | ||
|
|
||
| // 对常用字段添加索引 | ||
| ALTER TABLE <database>.logs ON CLUSTER '{cluster}' ADD INDEX IF NOT EXISTS idx_tace_id(tags.trace_id) TYPE bloom_filter GRANULARITY 1; | ||
|
|
||
| // 创建分布式表 | ||
| // 注意: 如果对logs表结构新增列, 需要同步修改logs_all | ||
| CREATE TABLE IF NOT EXISTS <database>.logs_all ON CLUSTER '{cluster}' | ||
| AS <database>.logs | ||
| ENGINE = Distributed('{cluster}', <database>, logs, rand()); |
43 changes: 43 additions & 0 deletions
43
conf/monitor/streaming/clickhouse/logs_ddl_create_tenant_tables.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // 创建日志表 | ||
| CREATE TABLE IF NOT EXISTS <database>.<table_name> ON CLUSTER '{cluster}' | ||
| ( | ||
| `_id` String, | ||
| `timestamp` DateTime64(9,'Asia/Shanghai'), | ||
| `source` String, | ||
| `id` String, | ||
| `org_name` String, | ||
| `stream` String, | ||
| `offset` Int64, | ||
| `content` String, | ||
| `tags` Map(String,String), | ||
| INDEX idx__id(_id) TYPE minmax GRANULARITY 1 | ||
| ) | ||
| ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}-{shard}/<table_name>', '{replica}') | ||
| PARTITION BY toYYYYMMDD(timestamp) | ||
| ORDER BY (org_name, timestamp, id) | ||
| TTL toDateTime(timestamp) + INTERVAL 7 DAY; | ||
|
|
||
| // 将常用字段添加为物化列 | ||
| ALTER TABLE <database>.<table_name> ON CLUSTER '{cluster}' | ||
| ADD COLUMN IF NOT EXISTS `tags.trace_id` String MATERIALIZED tags['trace_id'], | ||
| ADD COLUMN IF NOT EXISTS `tags.level` String MATERIALIZED tags['level'], | ||
| ADD COLUMN IF NOT EXISTS `tags.application_name` String MATERIALIZED tags['application_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.service_name` String MATERIALIZED tags['service_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.pod_name` String MATERIALIZED tags['pod_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.pod_ip` String MATERIALIZED tags['pod_ip'], | ||
| ADD COLUMN IF NOT EXISTS `tags.container_name` String MATERIALIZED tags['container_name'], | ||
| ADD COLUMN IF NOT EXISTS `tags.container_id` String MATERIALIZED tags['container_id']; | ||
|
|
||
| // 对常用字段添加索引 | ||
| ALTER TABLE <database>.<table_name> ON CLUSTER '{cluster}' ADD INDEX IF NOT EXISTS idx_tace_id(tags.trace_id) TYPE bloom_filter GRANULARITY 1; | ||
|
|
||
| // 创建分布式表 | ||
| // 注意: 如果对logs表结构新增列, 需要同步修改logs_all | ||
| CREATE TABLE IF NOT EXISTS <database>.<table_name>_all ON CLUSTER '{cluster}' | ||
| AS <database>.logs | ||
| ENGINE = Distributed('{cluster}', <database>, <table_name>, rand()); | ||
|
|
||
| // 创建Merge查询表 | ||
| CREATE TABLE IF NOT EXISTS <database>.<alias_table_name>_search ON CLUSTER '{cluster}' | ||
| AS <database>.logs | ||
| ENGINE = Merge(<database>, 'logs_all|<alias_table_name>.*_all$'); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| // Copyright (c) 2021 Terminus, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package clickhouse | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/erda-project/erda/modules/core/monitor/log/storage" | ||
| "github.com/erda-project/erda/modules/core/monitor/storekit" | ||
| ) | ||
|
|
||
| func (p *provider) Iterator(ctx context.Context, sel *storage.Selector) (storekit.Iterator, error) { | ||
| panic("implement me") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| // Copyright (c) 2021 Terminus, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package clickhouse | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/erda-project/erda-infra/base/logs" | ||
| "github.com/erda-project/erda-infra/base/servicehub" | ||
| "github.com/erda-project/erda-infra/providers/clickhouse" | ||
| "github.com/erda-project/erda/modules/core/monitor/log/storage" | ||
| "github.com/erda-project/erda/modules/core/monitor/settings/retention-strategy" | ||
| "github.com/erda-project/erda/modules/core/monitor/storekit/clickhouse/table/creator" | ||
| ) | ||
|
|
||
| type config struct { | ||
| } | ||
|
|
||
| type provider struct { | ||
| Cfg *config | ||
| Log logs.Logger | ||
| Creator creator.Interface `autowired:"clickhouse.table.creator@log"` | ||
| Retention retention.Interface `autowired:"storage-retention-strategy@log" optional:"true"` | ||
|
|
||
| clickhouse clickhouse.Interface | ||
| } | ||
|
|
||
| var _ storage.Storage = (*provider)(nil) | ||
|
|
||
| func (p *provider) Init(ctx servicehub.Context) error { | ||
| svc := ctx.Service("clickhouse@log") | ||
| if svc == nil { | ||
| svc = ctx.Service("clickhouse") | ||
| } | ||
| if svc == nil { | ||
| return fmt.Errorf("service clickhouse is required") | ||
| } | ||
| p.clickhouse = svc.(clickhouse.Interface) | ||
| return nil | ||
| } | ||
|
|
||
| func init() { | ||
| servicehub.Register("log-storage-clickhouse", &servicehub.Spec{ | ||
| Services: []string{"log-storage-clickhouse-reader", "log-storage-clickhouse-writer"}, | ||
| Dependencies: []string{"clickhouse", "clickhouse.table.creator"}, | ||
| ConfigFunc: func() interface{} { return &config{} }, | ||
| Creator: func() servicehub.Provider { return &provider{} }, | ||
| }) | ||
| } |
Oops, something went wrong.