diff --git a/common/persistence/nosql/nosqlplugin/cassandra/queue.go b/common/persistence/nosql/nosqlplugin/cassandra/queue.go index 68a7d6387e7..da59ae12ba6 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/queue.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/queue.go @@ -188,7 +188,11 @@ func (db *cdb) InsertQueueMetadata( ) error { clusterAckLevels := map[string]int64{} query := db.session.Query(templateInsertQueueMetadataQuery, queueType, clusterAckLevels, version).WithContext(ctx) - _, err := query.ScanCAS() + + // NOTE: Must pass nils to be compatible with ScyllaDB's LWT behavior + // "Scylla always returns the old version of the row, regardless of whether the condition is true or not." + // See also https://docs.scylladb.com/kb/lwt-differences/ + _, err := query.ScanCAS(nil, nil, nil) if err != nil { return err } @@ -209,7 +213,11 @@ func (db *cdb) UpdateQueueMetadataCas( row.QueueType, row.Version-1, ).WithContext(ctx) - applied, err := query.ScanCAS() + + // NOTE: Must pass nils to be compatible with ScyllaDB's LWT behavior + // "Scylla always returns the old version of the row, regardless of whether the condition is true or not." + // See also https://docs.scylladb.com/kb/lwt-differences/ + applied, err := query.ScanCAS(nil, nil, nil, nil) if err != nil { return err } diff --git a/config/development_scylla.yaml b/config/development_scylla.yaml new file mode 100644 index 00000000000..73212d9bcd4 --- /dev/null +++ b/config/development_scylla.yaml @@ -0,0 +1,118 @@ +persistence: + defaultStore: scylla-default + visibilityStore: scylla-visibility + numHistoryShards: 4 + datastores: + scylla-default: + cassandra: + hosts: "127.0.0.1" + keyspace: "cadence" + scylla-visibility: + cassandra: + hosts: "127.0.0.1" + keyspace: "cadence_visibility" + +ringpop: + name: cadence + bootstrapMode: hosts + bootstrapHosts: [ "127.0.0.1:7933", "127.0.0.1:7934", "127.0.0.1:7935" ] + maxJoinDuration: 30s + +services: + frontend: + rpc: + port: 7933 + bindOnLocalHost: true + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7936 + + matching: + rpc: + port: 7935 + bindOnLocalHost: true + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7938 + + history: + rpc: + port: 7934 + bindOnLocalHost: true + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7937 + + worker: + rpc: + port: 7939 + bindOnLocalHost: true + metrics: + statsd: + hostPort: "127.0.0.1:8125" + prefix: "cadence" + pprof: + port: 7940 + +clusterMetadata: + enableGlobalDomain: false + failoverVersionIncrement: 10 + masterClusterName: "active" + currentClusterName: "active" + clusterInformation: + active: + enabled: true + initialFailoverVersion: 0 + rpcName: "cadence-frontend" + rpcAddress: "localhost:7933" + +dcRedirectionPolicy: + policy: "noop" + toDC: "" + +archival: + history: + status: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + gstorage: + credentialsPath: "/tmp/gcloud/keyfile.json" + visibility: + status: "enabled" + enableRead: true + provider: + filestore: + fileMode: "0666" + dirMode: "0766" + +domainDefaults: + archival: + history: + status: "enabled" + URI: "file:///tmp/cadence_archival/development" + visibility: + status: "enabled" + URI: "file:///tmp/cadence_vis_archival/development" + +publicClient: + hostPort: "localhost:7933" + +dynamicConfigClient: + filepath: "config/dynamicconfig/development.yaml" + pollInterval: "10s" + +blobstore: + filestore: + outputDirectory: "/tmp/blobstore" diff --git a/docker/config_template.yaml b/docker/config_template.yaml index f11452bb4c5..dce0340c5df 100644 --- a/docker/config_template.yaml +++ b/docker/config_template.yaml @@ -12,7 +12,7 @@ persistence: {{- end }} datastores: {{- $db := default .Env.DB "cassandra" | lower -}} - {{- if eq $db "cassandra" }} + {{- if or (eq $db "cassandra") (eq $db "scylla") }} default: cassandra: hosts: {{ default .Env.CASSANDRA_SEEDS "" }} diff --git a/docker/docker-compose-scylla.yml b/docker/docker-compose-scylla.yml new file mode 100644 index 00000000000..ae678185bf6 --- /dev/null +++ b/docker/docker-compose-scylla.yml @@ -0,0 +1,39 @@ +version: '3' +services: + scylla: + image: scylladb/scylla:4.3.1 + command: + - "--smp" + - "1" + ports: + - "9042:9042" + statsd: + image: graphiteapp/graphite-statsd + ports: + - "8080:80" + - "2003:2003" + - "8125:8125" + - "8126:8126" + cadence: + image: ubercadence/server:master-auto-setup + ports: + - "7933:7933" + - "7934:7934" + - "7935:7935" + - "7939:7939" + environment: + - "DB=scylla" + - "CASSANDRA_SEEDS=scylla" + - "STATSD_ENDPOINT=statsd:8125" + - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml" + depends_on: + - scylla + - statsd + cadence-web: + image: ubercadence/web:latest + environment: + - "CADENCE_TCHANNEL_PEERS=cadence:7933" + ports: + - "8088:8088" + depends_on: + - cadence diff --git a/docker/start.sh b/docker/start.sh index 0b9bb5a503a..7e0f02d4ef2 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -91,6 +91,15 @@ wait_for_cassandra() { echo 'cassandra started' } +wait_for_scylla() { + server=`echo $CASSANDRA_SEEDS | awk -F ',' '{print $1}'` + until cqlsh -u $CASSANDRA_USER -p $CASSANDRA_PASSWORD --cqlversion=3.3.1 $server < /dev/null; do + echo 'waiting for scylla to start up' + sleep 1 + done + echo 'scylla started' +} + wait_for_mysql() { server=`echo $MYSQL_SEEDS | awk -F ',' '{print $1}'` nc -z $server $DB_PORT < /dev/null @@ -131,6 +140,8 @@ wait_for_db() { wait_for_mysql elif [ "$DB" == "postgres" ]; then wait_for_postgres + elif [ "$DB" == "scylla" ]; then + wait_for_scylla else wait_for_cassandra fi