Add support for ScyllaDB (cadence-workflow#4022)

common/persistence/nosql/nosqlplugin/cassandra/queue.go

@@ -188,7 +188,11 @@ func (db *cdb) InsertQueueMetadata(
 ) error {
 	clusterAckLevels := map[string]int64{}
 	query := db.session.Query(templateInsertQueueMetadataQuery, queueType, clusterAckLevels, version).WithContext(ctx)
-	_, err := query.ScanCAS()
+
+	// NOTE: Must pass nils to be compatible with ScyllaDB's LWT behavior
+	// "Scylla always returns the old version of the row, regardless of whether the condition is true or not."
+	// See also https://docs.scylladb.com/kb/lwt-differences/	
+	_, err := query.ScanCAS(nil, nil, nil)
 	if err != nil {
 		return err
 	}
@@ -209,7 +213,11 @@ func (db *cdb) UpdateQueueMetadataCas(
 		row.QueueType,
 		row.Version-1,
 	).WithContext(ctx)
-	applied, err := query.ScanCAS()
+
+	// NOTE: Must pass nils to be compatible with ScyllaDB's LWT behavior
+	// "Scylla always returns the old version of the row, regardless of whether the condition is true or not."
+	// See also https://docs.scylladb.com/kb/lwt-differences/
+	applied, err := query.ScanCAS(nil, nil, nil, nil)
 	if err != nil {
 		return err
 	}

config/development_scylla.yaml

@@ -0,0 +1,118 @@
+persistence:
+  defaultStore: scylla-default
+  visibilityStore: scylla-visibility
+  numHistoryShards: 4
+  datastores:
+    scylla-default:
+      cassandra:
+        hosts: "127.0.0.1"
+        keyspace: "cadence"
+    scylla-visibility:
+      cassandra:
+        hosts: "127.0.0.1"
+        keyspace: "cadence_visibility"
+
+ringpop:
+  name: cadence
+  bootstrapMode: hosts
+  bootstrapHosts: [ "127.0.0.1:7933", "127.0.0.1:7934", "127.0.0.1:7935" ]
+  maxJoinDuration: 30s
+
+services:
+  frontend:
+    rpc:
+      port: 7933
+      bindOnLocalHost: true
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7936
+
+  matching:
+    rpc:
+      port: 7935
+      bindOnLocalHost: true
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7938
+
+  history:
+    rpc:
+      port: 7934
+      bindOnLocalHost: true
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7937
+
+  worker:
+    rpc:
+      port: 7939
+      bindOnLocalHost: true
+    metrics:
+      statsd:
+        hostPort: "127.0.0.1:8125"
+        prefix: "cadence"
+    pprof:
+      port: 7940
+
+clusterMetadata:
+  enableGlobalDomain: false
+  failoverVersionIncrement: 10
+  masterClusterName: "active"
+  currentClusterName: "active"
+  clusterInformation:
+    active:
+      enabled: true
+      initialFailoverVersion: 0
+      rpcName: "cadence-frontend"
+      rpcAddress: "localhost:7933"
+
+dcRedirectionPolicy:
+  policy: "noop"
+  toDC: ""
+
+archival:
+  history:
+    status: "enabled"
+    enableRead: true
+    provider:
+      filestore:
+        fileMode: "0666"
+        dirMode: "0766"
+      gstorage:
+        credentialsPath: "/tmp/gcloud/keyfile.json"
+  visibility:
+    status: "enabled"
+    enableRead: true
+    provider:
+      filestore:
+        fileMode: "0666"
+        dirMode: "0766"
+
+domainDefaults:
+  archival:
+    history:
+      status: "enabled"
+      URI: "file:///tmp/cadence_archival/development"
+    visibility:
+      status: "enabled"
+      URI: "file:///tmp/cadence_vis_archival/development"
+
+publicClient:
+  hostPort: "localhost:7933"
+
+dynamicConfigClient:
+  filepath: "config/dynamicconfig/development.yaml"
+  pollInterval: "10s"
+
+blobstore:
+  filestore:
+    outputDirectory: "/tmp/blobstore"

docker/config_template.yaml

@@ -12,7 +12,7 @@ persistence:
     {{- end }}
     datastores:
         {{- $db := default .Env.DB "cassandra" | lower -}}
-        {{- if eq $db "cassandra" }}
+        {{- if or (eq $db "cassandra") (eq $db "scylla") }}
         default:
             cassandra:
                 hosts: {{ default .Env.CASSANDRA_SEEDS "" }}

docker/docker-compose-scylla.yml

@@ -0,0 +1,39 @@
+version: '3'
+services:
+  scylla:
+    image: scylladb/scylla:4.3.1
+    command:
+      - "--smp"
+      - "1"
+    ports:
+      - "9042:9042"
+  statsd:
+    image: graphiteapp/graphite-statsd
+    ports:
+      - "8080:80"
+      - "2003:2003"
+      - "8125:8125"
+      - "8126:8126"
+  cadence:
+    image: ubercadence/server:master-auto-setup
+    ports:
+      - "7933:7933"
+      - "7934:7934"
+      - "7935:7935"
+      - "7939:7939"
+    environment:
+      - "DB=scylla"
+      - "CASSANDRA_SEEDS=scylla"
+      - "STATSD_ENDPOINT=statsd:8125"
+      - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml"
+    depends_on:
+      - scylla
+      - statsd
+  cadence-web:
+    image: ubercadence/web:latest
+    environment:
+      - "CADENCE_TCHANNEL_PEERS=cadence:7933"
+    ports:
+      - "8088:8088"
+    depends_on:
+      - cadence

docker/start.sh

@@ -91,6 +91,15 @@ wait_for_cassandra() {
     echo 'cassandra started'
 }
 
+wait_for_scylla() {
+    server=`echo $CASSANDRA_SEEDS | awk -F ',' '{print $1}'`
+    until cqlsh -u $CASSANDRA_USER -p $CASSANDRA_PASSWORD --cqlversion=3.3.1 $server < /dev/null; do
+        echo 'waiting for scylla to start up'
+        sleep 1
+    done
+    echo 'scylla started'
+}
+
 wait_for_mysql() {
     server=`echo $MYSQL_SEEDS | awk -F ',' '{print $1}'`
     nc -z $server $DB_PORT < /dev/null
@@ -131,6 +140,8 @@ wait_for_db() {
         wait_for_mysql
     elif [ "$DB" == "postgres" ]; then
         wait_for_postgres
+    elif [ "$DB" == "scylla" ]; then
+        wait_for_scylla
     else
         wait_for_cassandra
     fi