From dbcb828bde27536ff63f302b59c1ede514303e04 Mon Sep 17 00:00:00 2001 From: Mark Steckel Date: Sun, 2 Mar 2014 19:23:07 -0800 Subject: [PATCH 1/5] Reworked things to: 1) maintain separation between "common" and tenant schemas (means that "common" tables are not also in the tenant schema!) 2) taks to process all tenants or a single tenant 3) support for per tenant migrations (potentially scary) 4) changed default so that db:migrate does NOT run apartment migrations. --- lib/apartment.rb | 4 +- lib/apartment/adapters/abstract_adapter.rb | 10 +- lib/apartment/adapters/postgresql_adapter.rb | 26 ++++- lib/apartment/database.rb | 2 +- lib/apartment/migrator.rb | 24 +++- lib/apartment/railtie.rb | 4 +- .../apartment/install/templates/apartment.rb | 9 +- lib/tasks/apartment.rake | 105 ++++++++++++++---- 8 files changed, 137 insertions(+), 47 deletions(-) diff --git a/lib/apartment.rb b/lib/apartment.rb index 67dc2b48..664b24d9 100644 --- a/lib/apartment.rb +++ b/lib/apartment.rb @@ -10,7 +10,7 @@ class << self extend Forwardable - ACCESSOR_METHODS = [:use_schemas, :seed_after_create, :prepend_environment, :append_environment] + ACCESSOR_METHODS = [:use_schemas, :seed_after_create, :prepend_environment, :append_environment, :migration_path] WRITER_METHODS = [:tenant_names, :database_schema_file, :excluded_models, :default_schema, :persistent_schemas, :connection_class, :tld_length, :db_migrate_tenants] attr_accessor(*ACCESSOR_METHODS) @@ -33,7 +33,7 @@ def tenant_names def db_migrate_tenants return @db_migrate_tenants if defined?(@db_migrate_tenants) - @db_migrate_tenants = true + @db_migrate_tenants = false end # Default to empty array diff --git a/lib/apartment/adapters/abstract_adapter.rb b/lib/apartment/adapters/abstract_adapter.rb index 2819c8ce..c0575313 100644 --- a/lib/apartment/adapters/abstract_adapter.rb +++ b/lib/apartment/adapters/abstract_adapter.rb @@ -65,9 +65,9 @@ def drop(tenant) # # @param {String?} tenant Database or schema to connect to # - def process(tenant = nil) + def process(tenant = nil, exclusive = false) previous_tenant = current_tenant - switch(tenant) + switch(tenant, exclusive) yield if block_given? ensure @@ -93,11 +93,11 @@ def reset # # @param {String} tenant Database name # - def switch(tenant = nil) + def switch(tenant = nil, exclusive = false) # Just connect to default db and return return reset if tenant.nil? - connect_to_new(tenant).tap do + connect_to_new(tenant, exclusive).tap do ActiveRecord::Base.connection.clear_query_cache end end @@ -126,7 +126,7 @@ def create_tenant(tenant) # # @param {String} tenant Database name # - def connect_to_new(tenant) + def connect_to_new(tenant, exclusive = false) Apartment.establish_connection multi_tenantify(tenant) Apartment.connection.active? # call active? to manually check if this connection is valid diff --git a/lib/apartment/adapters/postgresql_adapter.rb b/lib/apartment/adapters/postgresql_adapter.rb index f9074005..2925b5f8 100644 --- a/lib/apartment/adapters/postgresql_adapter.rb +++ b/lib/apartment/adapters/postgresql_adapter.rb @@ -49,6 +49,22 @@ def drop(tenant) raise SchemaNotFound, "The schema #{tenant.inspect} cannot be found." end + def load(tenant) + # require 'active_record/schema_dumper' + # filename = File.join(Apartment.migration_path, "#{tenant}_schema.rb") + # File.open(filename, "w:utf-8") do |file| + # ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file) + # end + end + + def dump(tenant) + require 'active_record/schema_dumper' + filename = File.join(Apartment.migration_path, "..", "#{tenant}_schema.rb") + File.open(filename, "w:utf-8") do |file| + ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file) + end + end + # Reset search path to default search_path # Set the table_name to always use the default namespace for excluded models # @@ -84,12 +100,16 @@ def current_tenant # Set schema search path to new schema # - def connect_to_new(tenant = nil) + def connect_to_new(tenant = nil, exclusive = false) return reset if tenant.nil? raise ActiveRecord::StatementInvalid.new("Could not find schema #{tenant}") unless Apartment.connection.schema_exists? tenant @current_tenant = tenant.to_s - Apartment.connection.schema_search_path = full_search_path + if exclusive + Apartment.connection.schema_search_path = @current_tenant + else + Apartment.connection.schema_search_path = full_search_path + end rescue *rescuable_exceptions raise SchemaNotFound, "One of the following schema(s) is invalid: #{tenant}, #{full_search_path}" @@ -113,7 +133,7 @@ def full_search_path end def persistent_schemas - [@current_tenant, Apartment.persistent_schemas].flatten + [@current_tenant, Apartment.persistent_schemas].flatten.uniq end end end diff --git a/lib/apartment/database.rb b/lib/apartment/database.rb index 70fbbb32..68cf5385 100644 --- a/lib/apartment/database.rb +++ b/lib/apartment/database.rb @@ -8,7 +8,7 @@ module Database extend self extend Forwardable - def_delegators :adapter, :create, :current_tenant, :current, :current_database, :drop, :process, :process_excluded_models, :reset, :seed, :switch + def_delegators :adapter, :create, :current_tenant, :current, :current_database, :drop, :process, :process_excluded_models, :reset, :seed, :switch, :dump, :load attr_writer :config diff --git a/lib/apartment/migrator.rb b/lib/apartment/migrator.rb index 90334ab7..3570dbd7 100644 --- a/lib/apartment/migrator.rb +++ b/lib/apartment/migrator.rb @@ -7,27 +7,39 @@ module Migrator # Migrate to latest def migrate(database) - Database.process(database) do + Database.process(database, true) do version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil - ActiveRecord::Migrator.migrate(ActiveRecord::Migrator.migrations_paths, version) do |migration| + ActiveRecord::Migrator.migrate(migration_paths(database), version) do |migration| ENV["SCOPE"].blank? || (ENV["SCOPE"] == migration.scope) end + Database.dump(database) end end # Migrate up/down to a specific version def run(direction, database, version) - Database.process(database) do - ActiveRecord::Migrator.run(direction, ActiveRecord::Migrator.migrations_paths, version) + Database.process(database, true) do + ActiveRecord::Migrator.run(direction, migration_paths(database), version) + Database.dump(database) end end # rollback latest migration `step` number of times def rollback(database, step = 1) - Database.process(database) do - ActiveRecord::Migrator.rollback(ActiveRecord::Migrator.migrations_paths, step) + Database.process(database, true) do + ActiveRecord::Migrator.rollback(migration_paths(database), step) + Database.dump(database) end end + + private + + def migration_paths(tenant) + paths = [Apartment.migration_path] + paths << "#{Apartment.migration_path}/../#{tenant}" if File.exists?("#{Apartment.migration_path}/../#{tenant}") + paths + end + end end diff --git a/lib/apartment/railtie.rb b/lib/apartment/railtie.rb index 30c019c1..c77ebd59 100644 --- a/lib/apartment/railtie.rb +++ b/lib/apartment/railtie.rb @@ -11,6 +11,7 @@ class Railtie < Rails::Railtie # config.before_initialize do Apartment.configure do |config| + config.db_migrate_tenants = false config.excluded_models = [] config.use_schemas = true config.tenant_names = [] @@ -18,9 +19,8 @@ class Railtie < Rails::Railtie config.prepend_environment = false config.append_environment = false config.tld_length = 1 + config.migration_path = "#{Rails.root}/db/apartment/migrate" end - - ActiveRecord::Migrator.migrations_paths = Rails.application.paths['db/migrate'].to_a end # Hook into ActionDispatch::Reloader to ensure Apartment is properly initialized diff --git a/lib/generators/apartment/install/templates/apartment.rb b/lib/generators/apartment/install/templates/apartment.rb index 9c632828..a9033f32 100644 --- a/lib/generators/apartment/install/templates/apartment.rb +++ b/lib/generators/apartment/install/templates/apartment.rb @@ -9,6 +9,11 @@ # Apartment.configure do |config| + # Determines whether db:migrations will automatically + # run apartment migrations or not. Defautls to false + # + # config.db_migrate_tenants = true + # These models will not be multi-tenanted, # but remain in the global (public) namespace # @@ -43,7 +48,3 @@ # Rails.application.config.middleware.use 'Apartment::Elevators::Domain' Rails.application.config.middleware.use 'Apartment::Elevators::Subdomain' - -## -# Rake enhancements so that db:migrate etc... also runs migrations on all tenants -require 'apartment/tasks/enhancements' diff --git a/lib/tasks/apartment.rake b/lib/tasks/apartment.rake index 1c497cd4..6d2aa61c 100644 --- a/lib/tasks/apartment.rake +++ b/lib/tasks/apartment.rake @@ -2,8 +2,10 @@ require 'apartment/migrator' apartment_namespace = namespace :apartment do + task :init => ['environment', 'db:load_config'] + desc "Create all tenants" - task create: 'db:migrate' do + task create: :init do tenants.each do |tenant| begin puts("Creating #{tenant} tenant") @@ -15,8 +17,8 @@ apartment_namespace = namespace :apartment do end desc "Migrate all tenants" - task :migrate do - warn_if_tenants_empty + task migrate: :init do + err_if_tenants_empty tenants.each do |tenant| begin @@ -29,8 +31,8 @@ apartment_namespace = namespace :apartment do end desc "Seed all tenants" - task :seed do - warn_if_tenants_empty + task seed: :init do + err_if_tenants_empty tenants.each do |tenant| begin @@ -45,8 +47,8 @@ apartment_namespace = namespace :apartment do end desc "Rolls the migration back to the previous version (specify steps w/ STEP=n) across all tenants." - task :rollback do - warn_if_tenants_empty + task rollback: :init do + err_if_tenants_empty step = ENV['STEP'] ? ENV['STEP'].to_i : 1 @@ -60,10 +62,75 @@ apartment_namespace = namespace :apartment do end end + namespace :tenant do + desc "Migrate a single tenant: required paramter TENANT=name" + task :migrate,[:tenant] => :init do |t, args| + tenant = args.tenant || ENV['TENANT'] + + err_if_tenants_empty + raise 'TENANT is required' unless tenant + raise "TENANT #{tenant} is unknown" if !tenants.include?(tenant) + + begin + puts("Migrating #{tenant} tenant") + Apartment::Migrator.migrate tenant + rescue Apartment::TenantNotFound => e + puts e.message + end + end + + desc "Rolls the migration back to the previous version (specify steps w/ STEP=n) for one TENANT." + task :rollback,[:tenant, :step] => :init do |t, args| + tenant = args.tenant || ENV['TENANT'] + step = args.step || ENV['STEP'] ? ENV['STEP'].to_i : 1 + + err_if_tenants_empty + raise 'TENANT is required' unless tenant + raise "TENANT #{tenant} is unknown" if !tenants.include?(tenant) + + begin + puts("Rolling back #{tenant} tenant") + Apartment::Migrator.rollback tenant, step + rescue Apartment::TenantNotFound => e + puts e.message + end + end + + desc 'Create a db/schema.rb file that can be portably used against any DB supported by AR' + task :dump, [:tenant] => ['environment', 'db:load_config'] do |t, args| + + tenant = args.tenant || ENV['TENANT'] + + err_if_tenants_empty + raise 'TENANT is required' unless tenant + raise "TENANT #{tenant} is unknown" if !tenants.include?(tenant) + + Apartment::Database.dump(tenant) + + # apartment_namespace['dump'].reenable + end + + # desc 'Load a schema.rb file into the database' + # task :load,[:tenant] => ['environment', 'db:load_config'] do |t, args| + # tenant = args.tenant || ENV['TENANT'] + + # err_if_tenants_empty + # raise 'TENANT is required' unless tenant + # raise "TENANT #{tenant} is unknown" if !tenants.include?(tenant) + + # puts "apartment:tenant:load(#{tenant})" + + # Apartment::Database.load(tenant) + # end + + end + + namespace :migrate do + desc 'Runs the "up" for a given migration VERSION across all tenants.' - task :up do - warn_if_tenants_empty + task up: :init do + err_if_tenants_empty version = ENV['VERSION'] ? ENV['VERSION'].to_i : nil raise 'VERSION is required' unless version @@ -79,8 +146,8 @@ apartment_namespace = namespace :apartment do end desc 'Runs the "down" for a given migration VERSION across all tenants.' - task :down do - warn_if_tenants_empty + task down: :init do + err_if_tenants_empty version = ENV['VERSION'] ? ENV['VERSION'].to_i : nil raise 'VERSION is required' unless version @@ -96,7 +163,7 @@ apartment_namespace = namespace :apartment do end desc 'Rolls back the tenant one migration and re migrate up (options: STEP=x, VERSION=x).' - task :redo do + task redo: :init do if ENV['VERSION'] apartment_namespace['migrate:down'].invoke apartment_namespace['migrate:up'].invoke @@ -111,17 +178,7 @@ apartment_namespace = namespace :apartment do ENV['DB'] ? ENV['DB'].split(',').map { |s| s.strip } : Apartment.tenant_names || [] end - def warn_if_tenants_empty - if tenants.empty? - puts <<-WARNING - [WARNING] - The list of tenants to migrate appears to be empty. This could mean a few things: - - 1. You may not have created any, in which case you can ignore this message - 2. You've run `apartment:migrate` directly without loading the Rails environment - * `apartment:migrate` is now deprecated. Tenants will automatically be migrated with `db:migrate` - - Note that your tenants currently haven't been migrated. You'll need to run `db:migrate` to rectify this. - WARNING - end + def err_if_tenants_empty + raise "tenants is empty" if tenants.empty? end end From 3c0f433120be383b720cf70eb61fdcf4ead914a5 Mon Sep 17 00:00:00 2001 From: Mark Steckel Date: Thu, 6 Mar 2014 11:48:48 -0800 Subject: [PATCH 2/5] Added support for lambdas and hashes for persistent_schema. --- lib/apartment/adapters/postgresql_adapter.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/apartment/adapters/postgresql_adapter.rb b/lib/apartment/adapters/postgresql_adapter.rb index 2925b5f8..7dfd35f1 100644 --- a/lib/apartment/adapters/postgresql_adapter.rb +++ b/lib/apartment/adapters/postgresql_adapter.rb @@ -133,7 +133,19 @@ def full_search_path end def persistent_schemas - [@current_tenant, Apartment.persistent_schemas].flatten.uniq + + case Apartment.persistent_schemas + when Proc + result = Apartment.persistent_schemas.call(@current_tenant) + when Hash + result = Apartment.persistent_schemas[@current_tenant] + when String + result = Apartment.persistent_schemas.split(',') + when NilClass + return [@current_tenant] + end + + [@current_tenant, result].flatten.uniq end end end From 45e14a71c297ce83d75ee7d1adda94cae76abfcd Mon Sep 17 00:00:00 2001 From: Mark Steckel Date: Thu, 6 Mar 2014 12:09:49 -0800 Subject: [PATCH 3/5] Minor bug fix, plus examples in template. --- lib/apartment/adapters/postgresql_adapter.rb | 6 +++--- lib/generators/apartment/install/templates/apartment.rb | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/apartment/adapters/postgresql_adapter.rb b/lib/apartment/adapters/postgresql_adapter.rb index 7dfd35f1..bf3595ad 100644 --- a/lib/apartment/adapters/postgresql_adapter.rb +++ b/lib/apartment/adapters/postgresql_adapter.rb @@ -133,18 +133,18 @@ def full_search_path end def persistent_schemas - case Apartment.persistent_schemas when Proc result = Apartment.persistent_schemas.call(@current_tenant) when Hash - result = Apartment.persistent_schemas[@current_tenant] + result = Apartment.persistent_schemas[@current_tenant] + when Array + result = Apartment.persistent_schemas when String result = Apartment.persistent_schemas.split(',') when NilClass return [@current_tenant] end - [@current_tenant, result].flatten.uniq end end diff --git a/lib/generators/apartment/install/templates/apartment.rb b/lib/generators/apartment/install/templates/apartment.rb index a9033f32..e3eb6071 100644 --- a/lib/generators/apartment/install/templates/apartment.rb +++ b/lib/generators/apartment/install/templates/apartment.rb @@ -28,7 +28,10 @@ config.use_schemas = true # configure persistent schemas (E.g. hstore ) + # config.persistent_schemas = 'foo' # config.persistent_schemas = %w{ hstore } + # config.persistent_schemas = {'fruit' => ['vegtables', 'public']} + # config.persistent_schemas = lambda{|tenant| tenant.gsub('_cc\z', '_ak')} # add the Rails environment to database names? # config.prepend_environment = true From c4495eeeb0c0f97d1b1d83a66f156f4609c20920 Mon Sep 17 00:00:00 2001 From: Mark Steckel Date: Sat, 8 Mar 2014 13:24:13 -0800 Subject: [PATCH 4/5] Ensure that the schema_migrations table exists in the tenant schema before executing any tenant migration. --- lib/apartment/migrator.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/apartment/migrator.rb b/lib/apartment/migrator.rb index 3570dbd7..c07f425d 100644 --- a/lib/apartment/migrator.rb +++ b/lib/apartment/migrator.rb @@ -7,7 +7,8 @@ module Migrator # Migrate to latest def migrate(database) - Database.process(database, true) do + ensure_schema_migrations_table_exists(database) + Database.process(database) do version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil ActiveRecord::Migrator.migrate(migration_paths(database), version) do |migration| @@ -19,7 +20,8 @@ def migrate(database) # Migrate up/down to a specific version def run(direction, database, version) - Database.process(database, true) do + ensure_schema_migrations_table_exists(database) + Database.process(database) do ActiveRecord::Migrator.run(direction, migration_paths(database), version) Database.dump(database) end @@ -27,7 +29,8 @@ def run(direction, database, version) # rollback latest migration `step` number of times def rollback(database, step = 1) - Database.process(database, true) do + ensure_schema_migrations_table_exists(database) + Database.process(database) do ActiveRecord::Migrator.rollback(migration_paths(database), step) Database.dump(database) end @@ -35,6 +38,12 @@ def rollback(database, step = 1) private + def ensure_schema_migrations_table_exists(database) + Database.process(database, true) do + ActiveRecord::SchemaMigration.create_table + end + end + def migration_paths(tenant) paths = [Apartment.migration_path] paths << "#{Apartment.migration_path}/../#{tenant}" if File.exists?("#{Apartment.migration_path}/../#{tenant}") From a0c7b109f7d8ed8a74ba9c3aa2f655f970350a25 Mon Sep 17 00:00:00 2001 From: Mark Steckel Date: Sat, 8 Mar 2014 13:48:14 -0800 Subject: [PATCH 5/5] Only dump the schema for the specified tenant, not the entire schema search path. --- lib/apartment/migrator.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/apartment/migrator.rb b/lib/apartment/migrator.rb index c07f425d..1ba6c117 100644 --- a/lib/apartment/migrator.rb +++ b/lib/apartment/migrator.rb @@ -14,8 +14,8 @@ def migrate(database) ActiveRecord::Migrator.migrate(migration_paths(database), version) do |migration| ENV["SCOPE"].blank? || (ENV["SCOPE"] == migration.scope) end - Database.dump(database) end + dump_schema(database) end # Migrate up/down to a specific version @@ -23,8 +23,8 @@ def run(direction, database, version) ensure_schema_migrations_table_exists(database) Database.process(database) do ActiveRecord::Migrator.run(direction, migration_paths(database), version) - Database.dump(database) end + dump_schema(database) end # rollback latest migration `step` number of times @@ -32,8 +32,8 @@ def rollback(database, step = 1) ensure_schema_migrations_table_exists(database) Database.process(database) do ActiveRecord::Migrator.rollback(migration_paths(database), step) - Database.dump(database) end + dump_schema(database) end private @@ -44,6 +44,13 @@ def ensure_schema_migrations_table_exists(database) end end + def dump_schema(database) + puts "dump_schema(#{database})" + Database.process(database, true) do + Database.dump(database) + end + end + def migration_paths(tenant) paths = [Apartment.migration_path] paths << "#{Apartment.migration_path}/../#{tenant}" if File.exists?("#{Apartment.migration_path}/../#{tenant}")