diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbac0a5c75a..b12955dd75c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,6 +79,7 @@ Also use `docker-compose -f ./docker/dev/cassandra.yml down` to stop and clean u * Alternatively, use `./docker/dev/cassandra-esv7-kafka.yml` for Cassandra, ElasticSearch(v7) and Kafka/ZooKeeper dependencies * Alternatively, use `./docker/dev/mysql-esv7-kafka.yml` for MySQL, ElasticSearch(v7) and Kafka/ZooKeeper dependencies * Alternatively, use `./docker/dev/cassandra-opensearch-kafka.yml` for Cassandra, OpenSearch(compatible with ElasticSearch v7) and Kafka/ZooKeeper dependencies +* Alternatively, use `./docker/dev/mongo-esv7-kafka.yml` for MongoDB, ElasticSearch(v7) and Kafka/ZooKeeper dependencies ### 3. Schema installation Based on the above dependency setup, you also need to install the schemas. diff --git a/common/persistence/nosql/nosqlPersistenceTest.go b/common/persistence/nosql/nosqlPersistenceTest.go index 4b4c90bd476..6b13bc99dbd 100644 --- a/common/persistence/nosql/nosqlPersistenceTest.go +++ b/common/persistence/nosql/nosqlPersistenceTest.go @@ -78,10 +78,7 @@ func (s *testCluster) Config() config.Persistence { // SetupTestDatabase from PersistenceTestCluster interface func (s *testCluster) SetupTestDatabase() { - // the keyspace is not created yet, so use empty and let the NoSQL DB to decide how to connect - s.cfg.Keyspace = "" adminDB, err := NewNoSQLAdminDB(&s.cfg, loggerimpl.NewNopLogger()) - s.cfg.Keyspace = s.keyspace // change it back if err != nil { log.Fatal(err) diff --git a/common/persistence/nosql/nosqlplugin/cassandra/admin.go b/common/persistence/nosql/nosqlplugin/cassandra/admin.go index 9ff98bf2366..43a5ec0c1ab 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/admin.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/admin.go @@ -48,11 +48,11 @@ func (db *cdb) SetupTestDatabase(schemaBaseDir string) error { return err } if schemaBaseDir == "" { - cadencePackageDir, err := getCadencePackageDir() + var err error + schemaBaseDir, err = nosqlplugin.GetDefaultTestSchemaDir(testSchemaDir) if err != nil { - log.Fatal(err) + return err } - schemaBaseDir = cadencePackageDir + testSchemaDir } err = db.loadSchema([]string{"schema.cql"}, schemaBaseDir) @@ -88,16 +88,6 @@ func (db *cdb) loadVisibilitySchema(fileNames []string, schemaBaseDir string) er return nil } -func getCadencePackageDir() (string, error) { - cadencePackageDir, err := os.Getwd() - if err != nil { - panic(err) - } - cadenceIndex := strings.LastIndex(cadencePackageDir, "/cadence/") - cadencePackageDir = cadencePackageDir[:cadenceIndex+len("/cadence/")] - return cadencePackageDir, err -} - func (db *cdb) TeardownTestDatabase() error { err := dropCassandraKeyspace(db.session, db.cfg.Keyspace) if err != nil { diff --git a/common/persistence/nosql/nosqlplugin/cassandra/configStore.go b/common/persistence/nosql/nosqlplugin/cassandra/configStore.go index d4897f8d8ea..f8ce4851b76 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/configStore.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/configStore.go @@ -32,10 +32,10 @@ import ( ) const ( + // version is the clustering key(DESC order) so this query will always return the record with largest version templateSelectLatestConfig = `SELECT row_type, version, timestamp, values, encoding FROM cluster_config WHERE row_type = ? LIMIT 1;` templateInsertConfig = `INSERT INTO cluster_config (row_type, version, timestamp, values, encoding) VALUES (?, ?, ?, ?, ?) IF NOT EXISTS;` - //for version value, x + 1 where x is the cached copy version. ) func (db *cdb) InsertConfig(ctx context.Context, row *persistence.InternalConfigStoreEntry) error { diff --git a/common/persistence/nosql/nosqlplugin/cassandra/plugin.go b/common/persistence/nosql/nosqlplugin/cassandra/plugin.go index 4e12e277b6a..aeafb0b73d3 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/plugin.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/plugin.go @@ -52,6 +52,14 @@ func (p *plugin) CreateDB(cfg *config.NoSQL, logger log.Logger) (nosqlplugin.DB, // CreateAdminDB initialize the AdminDB object func (p *plugin) CreateAdminDB(cfg *config.NoSQL, logger log.Logger) (nosqlplugin.AdminDB, error) { + // the keyspace is not created yet, so use empty and let the Cassandra connect + keyspace := cfg.Keyspace + cfg.Keyspace = "" + // change it back + defer func() { + cfg.Keyspace = keyspace + }() + return p.doCreateDB(cfg, logger) } diff --git a/common/persistence/nosql/nosqlplugin/common.go b/common/persistence/nosql/nosqlplugin/common.go new file mode 100644 index 00000000000..e751c869985 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/common.go @@ -0,0 +1,45 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// Portions of the Software are attributed to Copyright (c) 2020 Temporal Technologies Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package nosqlplugin + +import ( + "os" + "strings" +) + +func getCadencePackageDir() (string, error) { + cadencePackageDir, err := os.Getwd() + if err != nil { + panic(err) + } + cadenceIndex := strings.LastIndex(cadencePackageDir, "/cadence/") + cadencePackageDir = cadencePackageDir[:cadenceIndex+len("/cadence/")] + return cadencePackageDir, err +} + +func GetDefaultTestSchemaDir(testSchemaRelativePath string) (string, error) { + cadencePackageDir, err := getCadencePackageDir() + if err != nil { + return "", err + } + return cadencePackageDir + testSchemaRelativePath, nil +} diff --git a/common/persistence/nosql/nosqlplugin/interfaces.go b/common/persistence/nosql/nosqlplugin/interfaces.go index fae8eaf5fa0..7a12538b105 100644 --- a/common/persistence/nosql/nosqlplugin/interfaces.go +++ b/common/persistence/nosql/nosqlplugin/interfaces.go @@ -216,7 +216,9 @@ type ( } /** - * VisibilityCRUD is for visibility storage + * VisibilityCRUD is for visibility using database. + * Database visibility usually is no longer recommended. AdvancedVisibility(with Kafka+ElasticSearch) is more powerful and scalable. + * Feel free to skip this interface for any NoSQL plugin(use TODO() in the implementation) * * Recommendation: use one table with multiple indexes * @@ -510,7 +512,7 @@ type ( } /*** - * configStoreCRUD is for storing dynamic configuration parameters + * ConfigStoreCRUD is for storing dynamic configuration parameters * * Recommendation: one table * @@ -518,7 +520,9 @@ type ( * domain: partition key(row_type), range key(version) */ ConfigStoreCRUD interface { + // InsertConfig insert a config entry with version. Return nosqlplugin.NewConditionFailure if the same version of the row_type is existing InsertConfig(ctx context.Context, row *persistence.InternalConfigStoreEntry) error + // SelectLatestConfig returns the config entry of the row_type with the largest(latest) version value SelectLatestConfig(ctx context.Context, rowType int) (*persistence.InternalConfigStoreEntry, error) } ) diff --git a/common/persistence/nosql/nosqlplugin/mongodb/admin.go b/common/persistence/nosql/nosqlplugin/mongodb/admin.go new file mode 100644 index 00000000000..6c0dd66f93a --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/admin.go @@ -0,0 +1,71 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// Portions of the Software are attributed to Copyright (c) 2020 Temporal Technologies Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + "io/ioutil" + + "go.mongodb.org/mongo-driver/bson" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +var _ nosqlplugin.AdminDB = (*mdb)(nil) + +const ( + testSchemaDir = "schema/mongodb/" +) + +func (db *mdb) SetupTestDatabase(schemaBaseDir string) error { + if schemaBaseDir == "" { + var err error + schemaBaseDir, err = nosqlplugin.GetDefaultTestSchemaDir(testSchemaDir) + if err != nil { + return err + } + } + + schemaFile := schemaBaseDir + "cadence/schema.json" + byteValues, err := ioutil.ReadFile(schemaFile) + if err != nil { + return err + } + var commands []interface{} + err = bson.UnmarshalExtJSON(byteValues, false, &commands) + if err != nil { + return err + } + for _, cmd := range commands { + result := db.dbConn.RunCommand(context.Background(), cmd) + if result.Err() != nil { + return err + } + } + return nil +} + +func (db *mdb) TeardownTestDatabase() error { + result := db.dbConn.RunCommand(context.Background(), bson.D{{"dropDatabase", 1}}) + err := result.Err() + return err +} \ No newline at end of file diff --git a/common/persistence/nosql/nosqlplugin/mongodb/configStore.go b/common/persistence/nosql/nosqlplugin/mongodb/configStore.go new file mode 100644 index 00000000000..9047acff80a --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/configStore.go @@ -0,0 +1,70 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/uber/cadence/common" + "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" + "github.com/uber/cadence/schema/mongodb/cadence" +) + +func (db *mdb) InsertConfig(ctx context.Context, row *persistence.InternalConfigStoreEntry) error { + collection := db.dbConn.Collection(cadence.ClusterConfigCollectionName) + doc := cadence.ClusterConfigCollectionEntry{ + RowType: row.RowType, + Version: row.Version, + UnixTimestampSeconds: row.Timestamp.Unix(), + Data: row.Values.Data, + DataEncoding: row.Values.GetEncodingString(), + } + _, err := collection.InsertOne(ctx, doc) + if mongo.IsDuplicateKeyError(err) { + return nosqlplugin.NewConditionFailure("InsertConfig operation failed because of version collision") + } + return err +} + +func (db *mdb) SelectLatestConfig(ctx context.Context, rowType int) (*persistence.InternalConfigStoreEntry, error) { + filter := bson.D{{"rowtype", rowType}} + queryOptions := options.FindOneOptions{} + queryOptions.SetSort(bson.D{{"version", -1}}) + + collection := db.dbConn.Collection(cadence.ClusterConfigCollectionName) + var result cadence.ClusterConfigCollectionEntry + err := collection.FindOne(ctx, filter, &queryOptions).Decode(&result) + if err != nil { + return nil, err + } + return &persistence.InternalConfigStoreEntry{ + RowType: rowType, + Version: result.Version, + Timestamp: time.Unix(result.UnixTimestampSeconds, 0), + Values: persistence.NewDataBlob(result.Data, common.EncodingType(result.DataEncoding)), + }, nil +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/db.go b/common/persistence/nosql/nosqlplugin/mongodb/db.go new file mode 100644 index 00000000000..ebb2fb8a524 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/db.go @@ -0,0 +1,49 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + "go.mongodb.org/mongo-driver/mongo" + + "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +// mdb represents a logical connection to MongoDB database +type mdb struct { + client *mongo.Client + dbConn *mongo.Database + cfg *config.NoSQL + logger log.Logger +} + +var _ nosqlplugin.DB = (*mdb)(nil) + +func (db *mdb) Close() { + db.client.Disconnect(context.Background()) +} + +func (db *mdb) PluginName() string { + return PluginName +} \ No newline at end of file diff --git a/common/persistence/nosql/nosqlplugin/mongodb/domain.go b/common/persistence/nosql/nosqlplugin/mongodb/domain.go new file mode 100644 index 00000000000..cc0822be937 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/domain.go @@ -0,0 +1,86 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin/cassandra/gocql" +) + +// Insert a new record to domain, return error if failed or already exists +// Return ConditionFailure if the condition doesn't meet +func (db *mdb) InsertDomain( + ctx context.Context, + row *nosqlplugin.DomainRow, +) error { + panic("TODO") +} + +func (db *mdb) updateMetadataBatch( + ctx context.Context, + batch gocql.Batch, + notificationVersion int64, +) { + panic("TODO") +} + +// Update domain +func (db *mdb) UpdateDomain( + ctx context.Context, + row *nosqlplugin.DomainRow, +) error { + panic("TODO") +} + +// Get one domain data, either by domainID or domainName +func (db *mdb) SelectDomain( + ctx context.Context, + domainID *string, + domainName *string, +) (*nosqlplugin.DomainRow, error) { + panic("TODO") +} + +// Get all domain data +func (db *mdb) SelectAllDomains( + ctx context.Context, + pageSize int, + pageToken []byte, +) ([]*nosqlplugin.DomainRow, []byte, error) { + panic("TODO") +} + +// Delete a domain, either by domainID or domainName +func (db *mdb) DeleteDomain( + ctx context.Context, + domainID *string, + domainName *string, +) error { + panic("TODO") +} + +func (db *mdb) SelectDomainMetadata( + ctx context.Context, +) (int64, error) { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/error.go b/common/persistence/nosql/nosqlplugin/mongodb/error.go new file mode 100644 index 00000000000..a033be5b089 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/error.go @@ -0,0 +1,37 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "go.mongodb.org/mongo-driver/mongo" +) + +func (db *mdb) IsNotFoundError(err error) bool { + return err == mongo.ErrNoDocuments +} + +func (db *mdb) IsTimeoutError(err error) bool { + return mongo.IsTimeout(err) || mongo.IsNetworkError(err) +} + +func (db *mdb) IsThrottlingError(err error) bool { + return false +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/events.go b/common/persistence/nosql/nosqlplugin/mongodb/events.go new file mode 100644 index 00000000000..45412b75913 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/events.go @@ -0,0 +1,52 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +// InsertIntoHistoryTreeAndNode inserts one or two rows: tree row and node row(at least one of them) +func (db *mdb) InsertIntoHistoryTreeAndNode(ctx context.Context, treeRow *nosqlplugin.HistoryTreeRow, nodeRow *nosqlplugin.HistoryNodeRow) error { + panic("TODO") +} + +// SelectFromHistoryNode read nodes based on a filter +func (db *mdb) SelectFromHistoryNode(ctx context.Context, filter *nosqlplugin.HistoryNodeFilter) ([]*nosqlplugin.HistoryNodeRow, []byte, error) { + panic("TODO") +} + +// DeleteFromHistoryTreeAndNode delete a branch record, and a list of ranges of nodes. +func (db *mdb) DeleteFromHistoryTreeAndNode(ctx context.Context, treeFilter *nosqlplugin.HistoryTreeFilter, nodeFilters []*nosqlplugin.HistoryNodeFilter) error { + panic("TODO") +} + +// SelectAllHistoryTrees will return all tree branches with pagination +func (db *mdb) SelectAllHistoryTrees(ctx context.Context, nextPageToken []byte, pageSize int) ([]*nosqlplugin.HistoryTreeRow, []byte, error) { + panic("TODO") +} + +// SelectFromHistoryTree read branch records for a tree +func (db *mdb) SelectFromHistoryTree(ctx context.Context, filter *nosqlplugin.HistoryTreeFilter) ([]*nosqlplugin.HistoryTreeRow, error) { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/plugin.go b/common/persistence/nosql/nosqlplugin/mongodb/plugin.go new file mode 100644 index 00000000000..d98c6ae3e80 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/plugin.go @@ -0,0 +1,79 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/log" + "github.com/uber/cadence/common/persistence/nosql" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +const ( + // PluginName is the name of the plugin + PluginName = "mongodb" +) + +type plugin struct{} + +var _ nosqlplugin.Plugin = (*plugin)(nil) + +func init() { + nosql.RegisterPlugin(PluginName, &plugin{}) +} + +// CreateDB initialize the db object +func (p *plugin) CreateDB(cfg *config.NoSQL, logger log.Logger) (nosqlplugin.DB, error) { + return p.doCreateDB(cfg, logger) +} + +// CreateAdminDB initialize the AdminDB object +func (p *plugin) CreateAdminDB(cfg *config.NoSQL, logger log.Logger) (nosqlplugin.AdminDB, error) { + return p.doCreateDB(cfg, logger) +} + +func (p *plugin) doCreateDB(cfg *config.NoSQL, logger log.Logger) (*mdb, error) { + uri := fmt.Sprintf("mongodb://%v:%v@%v:%v/", cfg.User, cfg.Password, cfg.Hosts, cfg.Port) + // TODO CreateDB/CreateAdminDB don't pass in context.Context so we are using background for now + // It's okay because this is being called during server startup or CLI. + client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(uri)) + if err != nil { + return nil, err + } + if cfg.Keyspace == "" { + return nil, fmt.Errorf("database name cannot be empty") + } + db := client.Database(cfg.Keyspace) + return &mdb{ + client: client, + dbConn: db, + cfg: cfg, + logger: logger, + }, err +} + + diff --git a/common/persistence/nosql/nosqlplugin/mongodb/queue.go b/common/persistence/nosql/nosqlplugin/mongodb/queue.go new file mode 100644 index 00000000000..7609f5a1b8e --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/queue.go @@ -0,0 +1,128 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + "fmt" + + "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +//Insert message into queue, return error if failed or already exists +// Return ConditionFailure if the condition doesn't meet +func (db *mdb) InsertIntoQueue( + ctx context.Context, + row *nosqlplugin.QueueMessageRow, +) error { + panic("TODO") +} + +// Get the ID of last message inserted into the queue +func (db *mdb) SelectLastEnqueuedMessageID( + ctx context.Context, + queueType persistence.QueueType, +) (int64, error) { + panic("TODO") +} + +// Read queue messages starting from the exclusiveBeginMessageID +func (db *mdb) SelectMessagesFrom( + ctx context.Context, + queueType persistence.QueueType, + exclusiveBeginMessageID int64, + maxRows int, +) ([]*nosqlplugin.QueueMessageRow, error) { + panic("TODO") +} + +// Read queue message starting from exclusiveBeginMessageID int64, inclusiveEndMessageID int64 +func (db *mdb) SelectMessagesBetween( + ctx context.Context, + request nosqlplugin.SelectMessagesBetweenRequest, +) (*nosqlplugin.SelectMessagesBetweenResponse, error) { + panic("TODO") +} + +// Delete all messages before exclusiveBeginMessageID +func (db *mdb) DeleteMessagesBefore( + ctx context.Context, + queueType persistence.QueueType, + exclusiveBeginMessageID int64, +) error { + panic("TODO") +} + +// Delete all messages in a range between exclusiveBeginMessageID and inclusiveEndMessageID +func (db *mdb) DeleteMessagesInRange( + ctx context.Context, + queueType persistence.QueueType, + exclusiveBeginMessageID int64, + inclusiveEndMessageID int64, +) error { + panic("TODO") +} + +// Delete one message +func (db *mdb) DeleteMessage( + ctx context.Context, + queueType persistence.QueueType, + messageID int64, +) error { + panic("TODO") +} + +// Insert an empty metadata row, starting from a version +func (db *mdb) InsertQueueMetadata( + ctx context.Context, + queueType persistence.QueueType, + version int64, +) error { + fmt.Println("not implemented, ignore the eror for testing") + return nil +} + +// **Conditionally** update a queue metadata row, if current version is matched(meaning current == row.Version - 1), +// then the current version will increase by one when updating the metadata row +// Return ConditionFailure if the condition doesn't meet +func (db *mdb) UpdateQueueMetadataCas( + ctx context.Context, + row nosqlplugin.QueueMetadataRow, +) error { + panic("TODO") +} + +// Read a QueueMetadata +func (db *mdb) SelectQueueMetadata( + ctx context.Context, + queueType persistence.QueueType, +) (*nosqlplugin.QueueMetadataRow, error) { + fmt.Println("not implemented, ignore the eror for testing") + return nil, nil +} + +func (db *mdb) GetQueueSize( + ctx context.Context, + queueType persistence.QueueType, +) (int64, error) { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/shard.go b/common/persistence/nosql/nosqlplugin/mongodb/shard.go new file mode 100644 index 00000000000..2710bb32403 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/shard.go @@ -0,0 +1,53 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + log "github.com/sirupsen/logrus" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +// InsertShard creates a new shard, return error is there is any. +// Return ShardOperationConditionFailure if the condition doesn't meet +func (db *mdb) InsertShard(ctx context.Context, row *nosqlplugin.ShardRow) error { + log.Warn("not implemented...ignore the error for testing...") + return nil +} + +// SelectShard gets a shard +func (db *mdb) SelectShard(ctx context.Context, shardID int, currentClusterName string) (int64, *nosqlplugin.ShardRow, error) { + panic("TODO") +} + +// UpdateRangeID updates the rangeID, return error is there is any +// Return ShardOperationConditionFailure if the condition doesn't meet +func (db *mdb) UpdateRangeID(ctx context.Context, shardID int, rangeID int64, previousRangeID int64) error { + panic("TODO") +} + +// UpdateShard updates a shard, return error is there is any. +// Return ShardOperationConditionFailure if the condition doesn't meet +func (db *mdb) UpdateShard(ctx context.Context, row *nosqlplugin.ShardRow, previousRangeID int64) error { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/task.go b/common/persistence/nosql/nosqlplugin/mongodb/task.go new file mode 100644 index 00000000000..eef16a22223 --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/task.go @@ -0,0 +1,97 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +// SelectTaskList returns a single tasklist row. +// Return IsNotFoundError if the row doesn't exist +func (db *mdb) SelectTaskList(ctx context.Context, filter *nosqlplugin.TaskListFilter) (*nosqlplugin.TaskListRow, error) { + panic("TODO") +} + +// InsertTaskList insert a single tasklist row +// Return IsConditionFailedError if the row already exists, and also the existing row +func (db *mdb) InsertTaskList(ctx context.Context, row *nosqlplugin.TaskListRow) error { + panic("TODO") +} + +// UpdateTaskList updates a single tasklist row +// Return TaskOperationConditionFailure if the condition doesn't meet +func (db *mdb) UpdateTaskList( + ctx context.Context, + row *nosqlplugin.TaskListRow, + previousRangeID int64, +) error { + panic("TODO") +} + +// UpdateTaskList updates a single tasklist row, and set an TTL on the record +// Return TaskOperationConditionFailure if the condition doesn't meet +// Ignore TTL if it's not supported, which becomes exactly the same as UpdateTaskList, but ListTaskList must be +// implemented for TaskListScavenger +func (db *mdb) UpdateTaskListWithTTL( + ctx context.Context, + ttlSeconds int64, + row *nosqlplugin.TaskListRow, + previousRangeID int64, +) error { + panic("TODO") +} + +// ListTaskList returns all tasklists. +// Noop if TTL is already implemented in other methods +func (db *mdb) ListTaskList(ctx context.Context, pageSize int, nextPageToken []byte) (*nosqlplugin.ListTaskListResult, error) { + panic("TODO") +} + +// DeleteTaskList deletes a single tasklist row +// Return TaskOperationConditionFailure if the condition doesn't meet +func (db *mdb) DeleteTaskList(ctx context.Context, filter *nosqlplugin.TaskListFilter, previousRangeID int64) error { + panic("TODO") +} + +// InsertTasks inserts a batch of tasks +// Return TaskOperationConditionFailure if the condition doesn't meet +func (db *mdb) InsertTasks( + ctx context.Context, + tasksToInsert []*nosqlplugin.TaskRowForInsert, + tasklistCondition *nosqlplugin.TaskListRow, +) error { + panic("TODO") +} + +// SelectTasks return tasks that associated to a tasklist +func (db *mdb) SelectTasks(ctx context.Context, filter *nosqlplugin.TasksFilter) ([]*nosqlplugin.TaskRow, error) { + panic("TODO") +} + +// DeleteTask delete a batch tasks that taskIDs less than the row +// If TTL is not implemented, then should also return the number of rows deleted, otherwise persistence.UnknownNumRowsAffected +// NOTE: This API ignores the `BatchSize` request parameter i.e. either all tasks leq the task_id will be deleted or an error will +// be returned to the caller, because rowsDeleted is not supported by Cassandra +func (db *mdb) RangeDeleteTasks(ctx context.Context, filter *nosqlplugin.TasksFilter) (rowsDeleted int, err error) { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/tests/mongodb_persistence_test.go b/common/persistence/nosql/nosqlplugin/mongodb/tests/mongodb_persistence_test.go new file mode 100644 index 00000000000..c1d01e6063b --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/tests/mongodb_persistence_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package tests + +import ( + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin/mongodb" + persistencetests "github.com/uber/cadence/common/persistence/persistence-tests" + "github.com/uber/cadence/environment" +) + +func TestConfigStorePersistence(t *testing.T) { + s := new(persistencetests.ConfigStorePersistenceSuite) + s.TestBase = NewTestBaseWithMongo() + s.TestBase.Setup() + suite.Run(t, s) +} + +// TODO uncomment the test once HistoryEventsCRUD is implemented +// func TestMongoDBHistoryPersistence(t *testing.T) { +// s := new(persistencetests.HistoryV2PersistenceSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once TaskCRUD is implemented +// func TestMongoDBMatchingPersistence(t *testing.T) { +// s := new(persistencetests.MatchingPersistenceSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once DomainCRUD is implemented +// func TestMongoDBDomainPersistence(t *testing.T) { +// s := new(persistencetests.MetadataPersistenceSuiteV2) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once MessageQueueCRUD is implemented +// func TestQueuePersistence(t *testing.T) { +// s := new(persistencetests.QueuePersistenceSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once ShardCRUD is implemented +// func TestCassandraShardPersistence(t *testing.T) { +// s := new(persistencetests.ShardPersistenceSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once VisibilityCRUD is implemented +// func TestCassandraVisibilityPersistence(t *testing.T) { +// s := new(persistencetests.DBVisibilityPersistenceSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once WorkflowCRUD is implemented +// func TestCassandraExecutionManager(t *testing.T) { +// s := new(persistencetests.ExecutionManagerSuite) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +// TODO uncomment the test once WorkflowCRUD is implemented +// func TestCassandraExecutionManagerWithEventsV2(t *testing.T) { +// s := new(persistencetests.ExecutionManagerSuiteForEventsV2) +// s.TestBase = NewTestBaseWithMongo() +// s.TestBase.Setup() +// suite.Run(t, s) +// } + +func NewTestBaseWithMongo() persistencetests.TestBase { + options := &persistencetests.TestBaseOptions{ + DBPluginName: mongodb.PluginName, + DBHost: getTestConfig().Hosts, + DBUsername: getTestConfig().User, + DBPassword: getTestConfig().Password, + DBPort: getTestConfig().Port, + } + return persistencetests.NewTestBaseWithNoSQL(options) +} + +func getTestConfig() *config.NoSQL { + return &config.NoSQL{ + PluginName: mongodb.PluginName, + User: "root", + Password: "cadence", + Hosts: environment.GetMongoAddress(), + Port: environment.GetMongoPort(), + } +} \ No newline at end of file diff --git a/common/persistence/nosql/nosqlplugin/mongodb/visibility.go b/common/persistence/nosql/nosqlplugin/mongodb/visibility.go new file mode 100644 index 00000000000..936ec4076ff --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/visibility.go @@ -0,0 +1,64 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +func (db *mdb) InsertVisibility( + ctx context.Context, + ttlSeconds int64, + row *nosqlplugin.VisibilityRowForInsert, +) error { + panic("TODO") +} + +func (db *mdb) UpdateVisibility( + ctx context.Context, + ttlSeconds int64, + row *nosqlplugin.VisibilityRowForUpdate, +) error { + panic("TODO") +} + +func (db *mdb) SelectVisibility( + ctx context.Context, + filter *nosqlplugin.VisibilityFilter, +) (*nosqlplugin.SelectVisibilityResponse, error) { + panic("TODO") +} + +func (db *mdb) DeleteVisibility( + ctx context.Context, + domainID, workflowID, runID string, +) error { + panic("TODO") +} + +func (db *mdb) SelectOneClosedWorkflow( + ctx context.Context, + domainID, workflowID, runID string, +) (*nosqlplugin.VisibilityRow, error) { + panic("TODO") +} diff --git a/common/persistence/nosql/nosqlplugin/mongodb/workflow.go b/common/persistence/nosql/nosqlplugin/mongodb/workflow.go new file mode 100644 index 00000000000..e960e1fd44e --- /dev/null +++ b/common/persistence/nosql/nosqlplugin/mongodb/workflow.go @@ -0,0 +1,160 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// Portions of the Software are attributed to Copyright (c) 2020 Temporal Technologies Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +import ( + "context" + "time" + + "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin" +) + +var _ nosqlplugin.WorkflowCRUD = (*mdb)(nil) + +func (db *mdb) InsertWorkflowExecutionWithTasks( + ctx context.Context, + currentWorkflowRequest *nosqlplugin.CurrentWorkflowWriteRequest, + execution *nosqlplugin.WorkflowExecutionRequest, + transferTasks []*nosqlplugin.TransferTask, + crossClusterTasks []*nosqlplugin.CrossClusterTask, + replicationTasks []*nosqlplugin.ReplicationTask, + timerTasks []*nosqlplugin.TimerTask, + shardCondition *nosqlplugin.ShardCondition, +) error { + panic("TODO") +} + +func (db *mdb) UpdateWorkflowExecutionWithTasks( + ctx context.Context, + currentWorkflowRequest *nosqlplugin.CurrentWorkflowWriteRequest, + mutatedExecution *nosqlplugin.WorkflowExecutionRequest, + insertedExecution *nosqlplugin.WorkflowExecutionRequest, + resetExecution *nosqlplugin.WorkflowExecutionRequest, + transferTasks []*nosqlplugin.TransferTask, + crossClusterTasks []*nosqlplugin.CrossClusterTask, + replicationTasks []*nosqlplugin.ReplicationTask, + timerTasks []*nosqlplugin.TimerTask, + shardCondition *nosqlplugin.ShardCondition, +) error { + panic("TODO") +} + +func (db *mdb) SelectCurrentWorkflow(ctx context.Context, shardID int, domainID, workflowID string) (*nosqlplugin.CurrentWorkflowRow, error) { + panic("TODO") +} + +func (db *mdb) SelectWorkflowExecution(ctx context.Context, shardID int, domainID, workflowID, runID string) (*nosqlplugin.WorkflowExecution, error) { + panic("TODO") +} + +func (db *mdb) DeleteCurrentWorkflow(ctx context.Context, shardID int, domainID, workflowID, currentRunIDCondition string) error { + panic("TODO") +} + +func (db *mdb) DeleteWorkflowExecution(ctx context.Context, shardID int, domainID, workflowID, runID string) error { + panic("TODO") +} + +func (db *mdb) SelectAllCurrentWorkflows(ctx context.Context, shardID int, pageToken []byte, pageSize int) ([]*persistence.CurrentWorkflowExecution, []byte, error) { + panic("TODO") +} + +func (db *mdb) SelectAllWorkflowExecutions(ctx context.Context, shardID int, pageToken []byte, pageSize int) ([]*persistence.InternalListConcreteExecutionsEntity, []byte, error) { + panic("TODO") +} + +func (db *mdb) IsWorkflowExecutionExists(ctx context.Context, shardID int, domainID, workflowID, runID string) (bool, error) { + panic("TODO") +} + +func (db *mdb) SelectTransferTasksOrderByTaskID(ctx context.Context, shardID, pageSize int, pageToken []byte, exclusiveMinTaskID, inclusiveMaxTaskID int64) ([]*nosqlplugin.TransferTask, []byte, error) { + panic("TODO") +} + +func (db *mdb) DeleteTransferTask(ctx context.Context, shardID int, taskID int64) error { + panic("TODO") +} + +func (db *mdb) RangeDeleteTransferTasks(ctx context.Context, shardID int, exclusiveBeginTaskID, inclusiveEndTaskID int64) error { + panic("TODO") +} + +func (db *mdb) SelectTimerTasksOrderByVisibilityTime(ctx context.Context, shardID, pageSize int, pageToken []byte, inclusiveMinTime, exclusiveMaxTime time.Time) ([]*nosqlplugin.TimerTask, []byte, error) { + panic("TODO") +} + +func (db *mdb) DeleteTimerTask(ctx context.Context, shardID int, taskID int64, visibilityTimestamp time.Time) error { + panic("TODO") +} + +func (db *mdb) RangeDeleteTimerTasks(ctx context.Context, shardID int, inclusiveMinTime, exclusiveMaxTime time.Time) error { + panic("TODO") +} + +func (db *mdb) SelectReplicationTasksOrderByTaskID(ctx context.Context, shardID, pageSize int, pageToken []byte, exclusiveMinTaskID, inclusiveMaxTaskID int64) ([]*nosqlplugin.ReplicationTask, []byte, error) { + panic("TODO") +} + +func (db *mdb) DeleteReplicationTask(ctx context.Context, shardID int, taskID int64) error { + panic("TODO") +} + +func (db *mdb) RangeDeleteReplicationTasks(ctx context.Context, shardID int, inclusiveEndTaskID int64) error { + panic("TODO") +} + +func (db *mdb) InsertReplicationTask(ctx context.Context, tasks []*nosqlplugin.ReplicationTask, condition nosqlplugin.ShardCondition) error { + panic("TODO") +} + +func (db *mdb) SelectCrossClusterTasksOrderByTaskID(ctx context.Context, shardID, pageSize int, pageToken []byte, targetCluster string, exclusiveMinTaskID, inclusiveMaxTaskID int64) ([]*nosqlplugin.CrossClusterTask, []byte, error) { + panic("TODO") +} + +func (db *mdb) DeleteCrossClusterTask(ctx context.Context, shardID int, targetCluster string, taskID int64) error { + panic("TODO") +} + +func (db *mdb) RangeDeleteCrossClusterTasks(ctx context.Context, shardID int, targetCluster string, exclusiveBeginTaskID, inclusiveEndTaskID int64) error { + panic("TODO") +} + +func (db *mdb) InsertReplicationDLQTask(ctx context.Context, shardID int, sourceCluster string, task nosqlplugin.ReplicationTask) error { + panic("TODO") +} + +func (db *mdb) SelectReplicationDLQTasksOrderByTaskID(ctx context.Context, shardID int, sourceCluster string, pageSize int, pageToken []byte, exclusiveMinTaskID, inclusiveMaxTaskID int64) ([]*nosqlplugin.ReplicationTask, []byte, error) { + panic("TODO") +} + +func (db *mdb) SelectReplicationDLQTasksCount(ctx context.Context, shardID int, sourceCluster string) (int64, error) { + panic("TODO") +} + +func (db *mdb) DeleteReplicationDLQTask(ctx context.Context, shardID int, sourceCluster string, taskID int64) error { + panic("TODO") +} + +func (db *mdb) RangeDeleteReplicationDLQTasks(ctx context.Context, shardID int, sourceCluster string, exclusiveBeginTaskID, inclusiveEndTaskID int64) error { + panic("TODO") +} diff --git a/common/persistence/persistence-tests/configStorePersistenceTest.go b/common/persistence/persistence-tests/configStorePersistenceTest.go index 1ab471a1f5a..909e9532cf3 100644 --- a/common/persistence/persistence-tests/configStorePersistenceTest.go +++ b/common/persistence/persistence-tests/configStorePersistenceTest.go @@ -32,11 +32,18 @@ import ( "github.com/uber/cadence/common/config" p "github.com/uber/cadence/common/persistence" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin/cassandra" + "github.com/uber/cadence/common/persistence/nosql/nosqlplugin/mongodb" "github.com/uber/cadence/common/types" ) -//Currently you cannot clear or remove any entries in cluster_config table -//Therefore, Teardown and Setup of Test DB is required before every test. +var supportedPlugins = map[string]bool{ + cassandra.PluginName: true, + mongodb.PluginName: true, +} + +// Currently you cannot clear or remove any entries in cluster_config table +// Therefore, Teardown and Setup of Test DB is required before every test. type ( // ConfigStorePersistenceSuite contains config store persistence tests @@ -66,7 +73,7 @@ func (s *ConfigStorePersistenceSuite) TearDownSuite() { s.TearDownWorkflowStore() } -//Tests if error is returned when trying to fetch dc values from empty table +// Tests if error is returned when trying to fetch dc values from empty table func (s *ConfigStorePersistenceSuite) TestFetchFromEmptyTable() { if !validDatabaseCheck(s.Config()) { s.T().Skip() @@ -197,8 +204,8 @@ func generateRandomSnapshot(version int64) *p.DynamicConfigSnapshot { func validDatabaseCheck(cfg config.Persistence) bool { if datastore, ok := cfg.DataStores[cfg.DefaultStore]; ok { - if datastore.NoSQL != nil && datastore.NoSQL.PluginName == config.StoreTypeCassandra { - return true + if datastore.NoSQL != nil { + return supportedPlugins[datastore.NoSQL.PluginName] } } return false diff --git a/common/persistence/persistence-tests/persistenceTestBase.go b/common/persistence/persistence-tests/persistenceTestBase.go index 3c8da82fe51..465beb37849 100644 --- a/common/persistence/persistence-tests/persistenceTestBase.go +++ b/common/persistence/persistence-tests/persistenceTestBase.go @@ -1949,7 +1949,8 @@ func GenerateRandomDBName(n int) string { for i := range b { b[i] = letterRunes[rand.Intn(len(letterRunes))] } - return string(b) + ts := time.Now().Unix() + return fmt.Sprintf("%v_%v", ts, string(b)) } func pickRandomEncoding() common.EncodingType { diff --git a/docker/buildkite/docker-compose-local.yml b/docker/buildkite/docker-compose-local.yml index 24736119956..f4c8fb3cce9 100644 --- a/docker/buildkite/docker-compose-local.yml +++ b/docker/buildkite/docker-compose-local.yml @@ -71,6 +71,17 @@ services: environment: - discovery.type=single-node + mongo: + image: mongo:5 + restart: always + networks: + services-network: + aliases: + - mongo + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: cadence + unit-test: build: context: ../../ @@ -86,6 +97,7 @@ services: - cassandra - mysql - postgres + - mongo volumes: - ../../:/cadence networks: diff --git a/docker/buildkite/docker-compose.yml b/docker/buildkite/docker-compose.yml index 35218e4d6b3..73904e6fca7 100644 --- a/docker/buildkite/docker-compose.yml +++ b/docker/buildkite/docker-compose.yml @@ -57,6 +57,17 @@ services: environment: - discovery.type=single-node + mongo: + image: mongo:5 + restart: always + networks: + services-network: + aliases: + - mongo + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: cadence + unit-test: build: context: ../../ @@ -65,6 +76,7 @@ services: - "CASSANDRA_SEEDS=cassandra" - "MYSQL_SEEDS=mysql" - "POSTGRES_SEEDS=postgres" + - "MONGO_SEEDS=mongo" - BUILDKITE_AGENT_ACCESS_TOKEN - BUILDKITE_JOB_ID - BUILDKITE_BUILD_ID @@ -73,6 +85,7 @@ services: - cassandra - mysql - postgres + - mongo volumes: - ../../:/cadence - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent diff --git a/docker/dev/mongo-esv7-kafka.yml b/docker/dev/mongo-esv7-kafka.yml new file mode 100644 index 00000000000..0eb6be347d8 --- /dev/null +++ b/docker/dev/mongo-esv7-kafka.yml @@ -0,0 +1,40 @@ +version: '3' +services: + mongo: + image: mongo:5 + restart: always + ports: + - 27017:27017 + environment: + MONGO_INITDB_ROOT_USERNAME: root + MONGO_INITDB_ROOT_PASSWORD: cadence + + mongo-express: + image: mongo-express + restart: always + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: root + ME_CONFIG_MONGODB_ADMINPASSWORD: cadence + ME_CONFIG_MONGODB_URL: mongodb://root:cadence@mongo:27017/ + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.9.3 + ports: + - "9200:9200" + environment: + - discovery.type=single-node + kafka: + image: wurstmeister/kafka:2.12-2.1.1 + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + zookeeper: + image: wurstmeister/zookeeper:3.4.6 + ports: + - "2181:2181" \ No newline at end of file diff --git a/environment/env.go b/environment/env.go index de96662d970..e9226e3782d 100644 --- a/environment/env.go +++ b/environment/env.go @@ -58,6 +58,13 @@ const ( // MySQLDefaultPassword is default password MySQLDefaultPassword = "cadence" + // MongoSeeds env + MongoSeeds = "MONGO_SEEDS" + // MongoPort env + MongoPort = "MONGO_PORT" + // MongoDefaultPort is Mongo default port + MongoDefaultPort = "27017" + // KafkaSeeds env KafkaSeeds = "KAFKA_SEEDS" // KafkaPort env @@ -269,3 +276,25 @@ func GetESVersion() string { } return version } + +// GetMongoAddress return the MySQL address +func GetMongoAddress() string { + addr := os.Getenv(MongoSeeds) + if addr == "" { + addr = Localhost + } + return addr +} + +// GetMongoPort return the MySQL port +func GetMongoPort() int { + port := os.Getenv(MongoPort) + if port == "" { + port = MongoDefaultPort + } + p, err := strconv.Atoi(port) + if err != nil { + panic(fmt.Sprintf("error getting env %v", MongoPort)) + } + return p +} \ No newline at end of file diff --git a/go.mod b/go.mod index 7311307b7ce..3ed79bc9a68 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/pierrec/lz4 v0.0.0-20190701081048-057d66e894a4 // indirect github.com/robfig/cron v1.2.0 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 github.com/uber-go/tally v3.3.15+incompatible github.com/uber/ringpop-go v0.8.5 github.com/uber/tchannel-go v1.22.0 @@ -58,6 +58,7 @@ require ( github.com/valyala/fastjson v1.4.1 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 + go.mongodb.org/mongo-driver v1.7.3 go.opencensus.io v0.22.5 // indirect go.uber.org/atomic v1.7.0 go.uber.org/cadence v0.17.1-0.20210820042115-b09692f6838f @@ -71,7 +72,6 @@ require ( golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211004093028-2c5d950f24ef // indirect - golang.org/x/text v0.3.4 // indirect golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/tools v0.1.5 gonum.org/v1/gonum v0.7.0 diff --git a/go.sum b/go.sum index ba7f4167be9..be94d27f689 100644 --- a/go.sum +++ b/go.sum @@ -129,7 +129,32 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gocql/gocql v0.0.0-20191126110522-1982a06ad6b9 h1:90F3aCM6RNWVC05yJUGbgxgNoDdXyy4mQ8S893sSEm8= github.com/gocql/gocql v0.0.0-20191126110522-1982a06ad6b9/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -211,6 +236,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 h1:ECW73yc9MY7935nNYXUkK7Dz17YuSUI9yqRqYS8aBww= github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= @@ -220,6 +246,7 @@ github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmoiron/sqlx v1.2.1-0.20200615141059-0794cb1f47ee h1:59lyMGvZusByi7Rvctn8cxdVAjhiOnqCv3G5DrYApYQ= github.com/jmoiron/sqlx v1.2.1-0.20200615141059-0794cb1f47ee/go.mod h1:ClpsPFzLpSBl7MvJ+BhV0JHz4vmKRBarpvZ9644v9Oo= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -232,10 +259,14 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -259,6 +290,8 @@ github.com/m3db/prometheus_procfs v0.8.1 h1:LsxWzVELhDU9sLsZTaFLCeAwCn7bC7qecZcK github.com/m3db/prometheus_procfs v0.8.1/go.mod h1:N8lv8fLh3U3koZx1Bnisj60GYUMDpWb09x1R+dmMOJo= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -285,6 +318,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= @@ -311,6 +345,7 @@ github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9w github.com/pborman/uuid v0.0.0-20160209185913-a97ce2ca70fa/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE= github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v0.0.0-20190701081048-057d66e894a4 h1:Z1aQt0R99dvsefB+Z6ACJBPwVIBSCjtNzE0TTTbNrEw= @@ -349,6 +384,8 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhD github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -358,11 +395,15 @@ github.com/schollz/progressbar/v2 v2.12.1/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJO github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A= github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -372,8 +413,11 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/uber-common/bark v1.2.1 h1:cREJ9b7CpTjwZr0/5wV82fXlitoCIEHHnt9WkQ4lIk0= github.com/uber-common/bark v1.2.1/go.mod h1:g0ZuPcD7XiExKHynr93Q742G/sbrdVQkghrqLGOoFuY= github.com/uber-go/mapdecode v1.0.0 h1:euUEFM9KnuCa1OBixz1xM+FIXmpixyay5DLymceOVrU= @@ -396,16 +440,26 @@ github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/fastjson v1.4.1 h1:hrltpHpIpkaxll8QltMU8c3QZ5+qIiCL8yKqPFJI/yE= github.com/valyala/fastjson v1.4.1/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs= +go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -452,9 +506,11 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -529,6 +585,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -543,9 +600,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -577,8 +636,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -593,9 +652,13 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -749,6 +812,8 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/schema/cassandra/README.md b/schema/cassandra/README.md index 8c76afc14f9..115da343d0c 100644 --- a/schema/cassandra/README.md +++ b/schema/cassandra/README.md @@ -4,8 +4,8 @@ This directory contains the cassandra schema for every keyspace that cadence own ``` ./schema - - keyspace1/ - - keyspace2/ + - visibility/ -- Contains schema for visibility data models + - cadence/ -- Contains schema for default data models - keyspace.cql -- Contains the keyspace definition - schema.cql -- Contains the latest & greatest snapshot of the schema for the keyspace - versioned diff --git a/schema/mongodb/README.md b/schema/mongodb/README.md new file mode 100644 index 00000000000..130be547668 --- /dev/null +++ b/schema/mongodb/README.md @@ -0,0 +1,71 @@ +What +---- +This directory contains the mongodb schema for every database that cadence owns. The directory structure is as follows + + +``` +./schema + - cadence/ -- Contains schema for default data models + - schema.json -- Contains the latest & greatest snapshot of the schema for the keyspace + - schema.go -- Contains the collection schema in Golang structs -- because MongoDB collection is shemaless. + - versioned + - v0.1/ + - v0.2/ -- One directory per schema version change + - v1.0/ + - manifest.json -- json file describing the change + - changes.json -- changes in this version, only [create collection/index/documents] commands are allowed +``` + +## MongoDB JSON schema format +Below is an example of a schema JSON file containing two commands, for collection/index/documents creation. +```json +[ + { + "create": "collection_name" + }, + { + "createIndexes": "collection_name", + "indexes": [ + { + "key": { + "fieldnamea": 1, + "fieldnameb": -1 + }, + "name": "fieldnamea_fieldnameb" + } + ], + "writeConcern": { "w": "majority" } + }, + { + "insert": "collection_name", + "documents": [ + { + "fieldnamea": 1, + "fieldnameb": 0, + "fieldnamec": "1234" + }, + { + "fieldnamea": 2, + "fieldnameb": 1, + "fieldnamec": "12344" + }, + { + "fieldnamea": 2, + "fieldnameb": 2, + "fieldnamec": "12345" + } + ], + "ordered": false + } +] +``` + + +How +--- + +Q: How do I update existing schema ? +* Add your changes to schema.json for snapshot +* Create a new schema version directory under ./schema/<>/versioned/vx.x + * Add a manifest.json + * Add your changes in a json file \ No newline at end of file diff --git a/schema/mongodb/cadence/collectionSchema.go b/schema/mongodb/cadence/collectionSchema.go new file mode 100644 index 00000000000..8c7fb13fe91 --- /dev/null +++ b/schema/mongodb/cadence/collectionSchema.go @@ -0,0 +1,41 @@ +// Copyright (c) 2021 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package cadence + +// below are the names of all mongoDB collections +const ( + ClusterConfigCollectionName = "cluster_config" +) + +// NOTE1: MongoDB collection is schemaless -- there is no schema file for collection. We use Go lang structs to define the collection fields. + +// NOTE2: MongoDB doesn't allow using camel case or underscore in the field names + +// ClusterConfigCollectionEntry is the schema of configStore +// IMPORTANT: making change to this struct is changing the MongoDB collection schema. Please make sure it's backward compatible(e.g., don't delete the field, or change the annotation value). +type ClusterConfigCollectionEntry struct { + ID int `json:"_id,omitempty"` + RowType int `json:"rowtype"` + Version int64 `json:"version"` + Data []byte `json:"data"` + DataEncoding string `json:"dataencoding"` + UnixTimestampSeconds int64 `json:"unixtimestampseconds"` +} diff --git a/schema/mongodb/cadence/schema.json b/schema/mongodb/cadence/schema.json new file mode 100644 index 00000000000..b4f524c2bb3 --- /dev/null +++ b/schema/mongodb/cadence/schema.json @@ -0,0 +1,21 @@ +[ + { + "create": "cluster_config" + }, + { + "createIndexes": "cluster_config", + "indexes": [ + { + "key": { + "rowtype": 1, + "version": -1 + }, + "name": "rowtype_version", + "unique": true + } + ], + "writeConcern": { + "w": "majority" + } + } +] \ No newline at end of file diff --git a/schema/mongodb/cadence/versioned/v0.1/base.json b/schema/mongodb/cadence/versioned/v0.1/base.json new file mode 100644 index 00000000000..b4f524c2bb3 --- /dev/null +++ b/schema/mongodb/cadence/versioned/v0.1/base.json @@ -0,0 +1,21 @@ +[ + { + "create": "cluster_config" + }, + { + "createIndexes": "cluster_config", + "indexes": [ + { + "key": { + "rowtype": 1, + "version": -1 + }, + "name": "rowtype_version", + "unique": true + } + ], + "writeConcern": { + "w": "majority" + } + } +] \ No newline at end of file diff --git a/schema/mongodb/cadence/versioned/v0.1/manifest.json b/schema/mongodb/cadence/versioned/v0.1/manifest.json new file mode 100644 index 00000000000..a3f7c162e73 --- /dev/null +++ b/schema/mongodb/cadence/versioned/v0.1/manifest.json @@ -0,0 +1,8 @@ +{ + "CurrVersion": "0.1", + "MinCompatibleVersion": "0.1", + "Description": "base version of schema", + "SchemaUpdateCqlFiles": [ + "base.json" + ] +} diff --git a/schema/mongodb/version.go b/schema/mongodb/version.go new file mode 100644 index 00000000000..a46e4f8cee1 --- /dev/null +++ b/schema/mongodb/version.go @@ -0,0 +1,26 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package mongodb + +// NOTE: whenever there is a new data base schema update, plz update the following versions + +// Version is the MongoDB database schema release version +const Version = "0.1"