Add support for sqlite as a datastore and eventstore (warrant-dev#64)

* Add support for sqlite as a datastore and eventstore * Rebase with main * Cleanup * Address code review comments * Change event id column to text w/ UUID generated in code * Insert code generated uuid as id column in event repository * Fix repeated id typo * Add InMemory config option for sqlite that specifies whether or not to run the database in memory (defaults to false) * Re-use the existing database connection when migrating sqlite so in memory databases can be successfully migrated * Remove duplicate import --------- Co-authored-by: Karan Kajla <[email protected]>
difftim · Apr 8, 2023 · 0502682 · 0502682
1 parent 1289cf6
commit 0502682
Show file tree

Hide file tree

Showing 48 changed files with 3,656 additions and 115 deletions.
diff --git a/cmd/warrant/main.go b/cmd/warrant/main.go
@@ -29,6 +29,8 @@ const (
 	MySQLEventstoreMigrationVersion    = 000001
 	PostgresDatastoreMigrationVersion  = 000002
 	PostgresEventstoreMigrationVersion = 000001
+	SQLiteDatastoreMigrationVersion    = 000002
+	SQLiteEventstoreMigrationVersion   = 000001
 )
 
 type ServiceEnv struct {
@@ -80,6 +82,22 @@ func (env *ServiceEnv) InitDB(config config.Config) error {
 		return nil
 	}
 
+	if config.Datastore.SQLite.Database != "" {
+		db := database.NewSQLite(*config.Datastore.SQLite)
+		err := db.Connect(ctx)
+		if err != nil {
+			return err
+		}
+
+		err = db.Migrate(ctx, SQLiteDatastoreMigrationVersion)
+		if err != nil {
+			return err
+		}
+
+		env.Datastore = db
+		return nil
+	}
+
 	return fmt.Errorf("invalid database configuration provided")
 }
 
@@ -119,6 +137,22 @@ func (env *ServiceEnv) InitEventDB(config config.Config) error {
 		return nil
 	}
 
+	if config.Eventstore.SQLite.Database != "" {
+		db := database.NewSQLite(*config.Eventstore.SQLite)
+		err := db.Connect(ctx)
+		if err != nil {
+			return err
+		}
+
+		err = db.Migrate(ctx, SQLiteEventstoreMigrationVersion)
+		if err != nil {
+			return err
+		}
+
+		env.Eventstore = db
+		return nil
+	}
+
 	return fmt.Errorf("invalid database configuration provided")
 }
 

diff --git a/go.mod b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.7
+	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.29.0
 	github.com/spf13/viper v1.15.0

diff --git a/go.sum b/go.sum
@@ -829,8 +829,9 @@ github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
-github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
 github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=

diff --git a/migrations/datastore/sqlite/000001_init.down.sql b/migrations/datastore/sqlite/000001_init.down.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS pricingTier;
+DROP TABLE IF EXISTS feature;
+DROP TABLE IF EXISTS user;
+DROP TABLE IF EXISTS tenant;
+DROP TABLE IF EXISTS role;
+DROP TABLE IF EXISTS permission;
+DROP TABLE IF EXISTS object;
+DROP TABLE IF EXISTS context;
+DROP TABLE IF EXISTS warrant;
+DROP TABLE IF EXISTS objectType;
diff --git a/migrations/datastore/sqlite/000001_init.up.sql b/migrations/datastore/sqlite/000001_init.up.sql
@@ -0,0 +1,208 @@
+CREATE TABLE IF NOT EXISTS objectType (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  typeId TEXT NOT NULL,
+  definition TEXT NOT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS object_type_uk_type_id
+    ON objectType (typeId);
+
+CREATE TABLE IF NOT EXISTS warrant (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectType TEXT NOT NULL,
+  objectId TEXT NOT NULL,
+  relation TEXT NOT NULL,
+  subjectType TEXT NOT NULL,
+  subjectId TEXT NOT NULL,
+  subjectRelation TEXT DEFAULT NULL,
+  contextHash TEXT NOT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS warrant_uk_obj_rel_sub_ctx_hash
+    ON warrant (objectType, objectId, relation, subjectType, subjectId, subjectRelation, contextHash);
+
+CREATE INDEX IF NOT EXISTS warrant_uk_sub_type_sub_id_sub_rel
+    ON warrant (subjectType, subjectId, subjectRelation);
+
+CREATE INDEX IF NOT EXISTS warrant_uk_obj_type_obj_id_rel
+    ON warrant (objectType, objectId, relation);
+
+CREATE TABLE IF NOT EXISTS context (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  warrantId INTEGER NOT NULL,
+  name TEXT NOT NULL,
+  value TEXT NOT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (warrantId) REFERENCES warrant (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS context_uk_warrant_id_name
+    ON context (warrantId, name);
+
+CREATE TABLE IF NOT EXISTS object (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectType TEXT NOT NULL,
+  objectId TEXT NOT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS object_uk_obj_type_obj_id
+    ON object (objectType, objectId);
+
+CREATE INDEX IF NOT EXISTS object_uk_created_at_object_id
+    ON object (createdAt, objectId);
+
+CREATE INDEX IF NOT EXISTS object_uk_object_type_object_id
+    ON object (objectType, objectId);
+
+CREATE TABLE IF NOT EXISTS permission (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectId INTEGER NOT NULL,
+  permissionId TEXT NOT NULL,
+  name TEXT DEFAULT NULL,
+  description TEXT DEFAULT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS permission_uk_perm_id
+    ON permission (permissionId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON permission (objectId);
+
+CREATE INDEX IF NOT EXISTS permission_uk_created_at_permission_id
+    ON permission (createdAt, permissionId);
+
+CREATE INDEX IF NOT EXISTS permission_uk_name_permission_id
+    ON permission (name, permissionId);
+
+CREATE TABLE IF NOT EXISTS role (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectId INTEGER NOT NULL,
+  roleId TEXT NOT NULL,
+  name TEXT DEFAULT NULL,
+  description TEXT DEFAULT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS role_uk_role_id
+    ON role (roleId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON role (objectId);
+
+CREATE INDEX IF NOT EXISTS role_uk_created_at_role_id
+    ON role (createdAt, roleId);
+
+CREATE INDEX IF NOT EXISTS role_uk_name_role_id
+    ON role (name, roleId);
+
+CREATE TABLE IF NOT EXISTS tenant (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectId INTEGER NOT NULL,
+  tenantId TEXT NOT NULL,
+  name TEXT DEFAULT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS tenant_uk_tenant_id
+    ON tenant (tenantId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON tenant (objectId);
+
+CREATE INDEX IF NOT EXISTS tenant_uk_created_at_tenant_id
+    ON tenant (createdAt, tenantId);
+
+CREATE INDEX IF NOT EXISTS tenant_uk_name_tenant_id
+    ON tenant (name, tenantId);
+
+CREATE TABLE IF NOT EXISTS user (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  userId TEXT NOT NULL,
+  email TEXT DEFAULT NULL,
+  objectId INTEGER NOT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS user_uk_user_id
+    ON user (userId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON user (objectId);
+
+CREATE INDEX IF NOT EXISTS user_uk_created_at_user_id
+    ON user (createdAt, userId);
+
+CREATE INDEX IF NOT EXISTS user_uk_email_user_id
+    ON user (email, userId);
+
+CREATE TABLE IF NOT EXISTS feature (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectId INTEGER NOT NULL,
+  featureId TEXT NOT NULL,
+  name TEXT DEFAULT NULL,
+  description TEXT DEFAULT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS feature_uk_feature_id
+    ON feature (featureId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON feature (objectId);
+
+CREATE INDEX IF NOT EXISTS feature_uk_created_at_feature_id
+    ON feature (createdAt, featureId);
+
+CREATE INDEX IF NOT EXISTS feature_uk_name_feature_id
+    ON feature (name, featureId);
+
+CREATE TABLE IF NOT EXISTS pricingTier (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  objectId INTEGER NOT NULL,
+  pricingTierId TEXT NOT NULL,
+  name TEXT DEFAULT NULL,
+  description TEXT DEFAULT NULL,
+  createdAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  updatedAt DATETIME DEFAULT CURRENT_TIMESTAMP,
+  deletedAt DATETIME DEFAULT NULL,
+  FOREIGN KEY (objectId) REFERENCES object (id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS pricing_tier_uk_pricing_tier_id
+    ON pricingTier (pricingTierId);
+
+CREATE INDEX IF NOT EXISTS objectId
+    ON pricingTier (objectId);
+
+CREATE INDEX IF NOT EXISTS pricing_tier_uk_created_at_pricing_tier_id
+    ON pricingTier (createdAt, pricingTierId);
+
+CREATE INDEX IF NOT EXISTS pricing_tier_uk_name_pricing_tier_id
+    ON pricingTier (name, pricingTierId);
diff --git a/migrations/datastore/sqlite/000002_create_default_obj_types.down.sql b/migrations/datastore/sqlite/000002_create_default_obj_types.down.sql
@@ -0,0 +1,2 @@
+DELETE FROM objectType
+WHERE typeId IN ('role', 'permission', 'tenant', 'user', 'pricing-tier', 'feature');
diff --git a/migrations/datastore/sqlite/000002_create_default_obj_types.up.sql b/migrations/datastore/sqlite/000002_create_default_obj_types.up.sql
@@ -0,0 +1,11 @@
+INSERT INTO objectType (typeId, definition)
+VALUES
+    ('role', '{"type": "role", "relations": {"member": {"inheritIf": "member", "ofType": "role", "withRelation": "member"}}}'),
+    ('permission', '{"type": "permission", "relations": {"member": {"inheritIf": "anyOf", "rules": [{"inheritIf": "member", "ofType": "permission", "withRelation": "member"}, {"inheritIf": "member", "ofType": "role", "withRelation": "member"}]}}}'),
+    ('tenant', '{"type": "tenant", "relations": {"admin": {}, "member": {"inheritIf": "manager"}, "manager": {"inheritIf": "admin"}}}'),
+    ('user', '{"type": "user", "relations": {"parent": {"inheritIf": "parent", "ofType": "user", "withRelation": "parent"}}}'),
+    ('pricing-tier', '{"type": "pricing-tier", "relations": {"member": {"ofType": "pricing-tier", "inheritIf": "member", "withRelation": "member"}}}'),
+    ('feature', '{"type": "feature", "relations": {"member": {"inheritIf": "anyOf", "rules": [{"inheritIf": "member", "ofType": "feature", "withRelation": "member"}, {"ofType": "pricing-tier", "inheritIf": "member", "withRelation": "member"}]}}}')
+ON CONFLICT (typeId) DO UPDATE SET
+    definition=excluded.definition,
+    deletedAt=NULL;
diff --git a/migrations/datastore/sqlite/README.md b/migrations/datastore/sqlite/README.md
diff --git a/migrations/eventstore/sqlite/000001_init.down.sql b/migrations/eventstore/sqlite/000001_init.down.sql
@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS resourceEvent;
+DROP TABLE IF EXISTS accessEvent;
diff --git a/migrations/eventstore/sqlite/000001_init.up.sql b/migrations/eventstore/sqlite/000001_init.up.sql
@@ -0,0 +1,32 @@
+CREATE TABLE IF NOT EXISTS resourceEvent (
+  id TEXT NOT NULL,
+  type TEXT NOT NULL,
+  source TEXT NOT NULL,
+  resourceType TEXT NOT NULL,
+  resourceId TEXT NOT NULL,
+  meta TEXT DEFAULT NULL,
+  createdAt DATETIME NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX resource_event_idx_created_at_id
+    ON resourceEvent (createdAt, id);
+
+CREATE TABLE IF NOT EXISTS accessEvent (
+  id TEXT NOT NULL,
+  type TEXT NOT NULL,
+  source TEXT NOT NULL,
+  objectType TEXT NOT NULL,
+  objectId TEXT NOT NULL,
+  relation TEXT NOT NULL,
+  subjectType TEXT NOT NULL,
+  subjectId TEXT NOT NULL,
+  subjectRelation TEXT DEFAULT NULL,
+  context TEXT DEFAULT NULL,
+  meta TEXT DEFAULT NULL,
+  createdAt DATETIME NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX access_event_idx_created_at_id
+    ON accessEvent (createdAt, id);
diff --git a/pkg/authz/feature/model.go b/pkg/authz/feature/model.go
@@ -21,14 +21,14 @@ type Model interface {
 }
 
 type Feature struct {
-	ID          int64               `mysql:"id" postgres:"id"`
-	ObjectId    int64               `mysql:"objectId" postgres:"object_id"`
-	FeatureId   string              `mysql:"featureId" postgres:"feature_id"`
-	Name        database.NullString `mysql:"name" postgres:"name"`
-	Description database.NullString `mysql:"description" postgres:"description"`
-	CreatedAt   time.Time           `mysql:"createdAt" postgres:"created_at"`
-	UpdatedAt   time.Time           `mysql:"updatedAt" postgres:"updated_at"`
-	DeletedAt   database.NullTime   `mysql:"deletedAt" postgres:"deleted_at"`
+	ID          int64               `mysql:"id" postgres:"id" sqlite:"id"`
+	ObjectId    int64               `mysql:"objectId" postgres:"object_id" sqlite:"objectId"`
+	FeatureId   string              `mysql:"featureId" postgres:"feature_id" sqlite:"featureId"`
+	Name        database.NullString `mysql:"name" postgres:"name" sqlite:"name"`
+	Description database.NullString `mysql:"description" postgres:"description" sqlite:"description"`
+	CreatedAt   time.Time           `mysql:"createdAt" postgres:"created_at" sqlite:"createdAt"`
+	UpdatedAt   time.Time           `mysql:"updatedAt" postgres:"updated_at" sqlite:"updatedAt"`
+	DeletedAt   database.NullTime   `mysql:"deletedAt" postgres:"deleted_at" sqlite:"deletedAt"`
 }
 
 func (feature Feature) GetID() int64 {

diff --git a/pkg/authz/feature/repository.go b/pkg/authz/feature/repository.go
@@ -33,6 +33,13 @@ func NewRepository(db database.Database) (FeatureRepository, error) {
 		}
 
 		return NewPostgresRepository(postgres), nil
+	case database.TypeSQLite:
+		sqlite, ok := db.(*database.SQLite)
+		if !ok {
+			return nil, fmt.Errorf("invalid %s database config", database.TypeSQLite)
+		}
+
+		return NewSQLiteRepository(sqlite), nil
 	default:
 		return nil, fmt.Errorf("unsupported database type %s specified", db.Type())
 	}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		DELETE FROM objectType
		WHERE typeId IN ('role', 'permission', 'tenant', 'user', 'pricing-tier', 'feature');
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		DROP TABLE IF EXISTS resourceEvent;
		DROP TABLE IF EXISTS accessEvent;