Skip to content

Commit

Permalink
Merge pull request #55 from lepo-project/v1.1
Browse files Browse the repository at this point in the history 
V1.1
  • Loading branch information
@kyoshizaki
kyoshizaki authored Mar 10, 2021
2 parents baef5d2 + 0c3981e commit ad1fe52
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 105 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,6 @@
.byebug_history
/storage/*
!/storage/sample_csv

/config/master.key
/config/credentials.yml.enc
138 changes: 69 additions & 69 deletions Gemfile.lock
View file
Original file line number Diff line number Diff line change
@@ -1,58 +1,58 @@
GEM
remote: https://rubygems.org/
specs:
actioncable (6.0.3.4)
actionpack (= 6.0.3.4)
actioncable (6.0.3.5)
actionpack (= 6.0.3.5)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
actionmailbox (6.0.3.4)
actionpack (= 6.0.3.4)
activejob (= 6.0.3.4)
activerecord (= 6.0.3.4)
activestorage (= 6.0.3.4)
activesupport (= 6.0.3.4)
actionmailbox (6.0.3.5)
actionpack (= 6.0.3.5)
activejob (= 6.0.3.5)
activerecord (= 6.0.3.5)
activestorage (= 6.0.3.5)
activesupport (= 6.0.3.5)
mail (>= 2.7.1)
actionmailer (6.0.3.4)
actionpack (= 6.0.3.4)
actionview (= 6.0.3.4)
activejob (= 6.0.3.4)
actionmailer (6.0.3.5)
actionpack (= 6.0.3.5)
actionview (= 6.0.3.5)
activejob (= 6.0.3.5)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
actionpack (6.0.3.4)
actionview (= 6.0.3.4)
activesupport (= 6.0.3.4)
actionpack (6.0.3.5)
actionview (= 6.0.3.5)
activesupport (= 6.0.3.5)
rack (~> 2.0, >= 2.0.8)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (6.0.3.4)
actionpack (= 6.0.3.4)
activerecord (= 6.0.3.4)
activestorage (= 6.0.3.4)
activesupport (= 6.0.3.4)
actiontext (6.0.3.5)
actionpack (= 6.0.3.5)
activerecord (= 6.0.3.5)
activestorage (= 6.0.3.5)
activesupport (= 6.0.3.5)
nokogiri (>= 1.8.5)
actionview (6.0.3.4)
activesupport (= 6.0.3.4)
actionview (6.0.3.5)
activesupport (= 6.0.3.5)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
activejob (6.0.3.4)
activesupport (= 6.0.3.4)
activejob (6.0.3.5)
activesupport (= 6.0.3.5)
globalid (>= 0.3.6)
activemodel (6.0.3.4)
activesupport (= 6.0.3.4)
activerecord (6.0.3.4)
activemodel (= 6.0.3.4)
activesupport (= 6.0.3.4)
activerecord-import (1.0.7)
activemodel (6.0.3.5)
activesupport (= 6.0.3.5)
activerecord (6.0.3.5)
activemodel (= 6.0.3.5)
activesupport (= 6.0.3.5)
activerecord-import (1.0.8)
activerecord (>= 3.2)
activestorage (6.0.3.4)
actionpack (= 6.0.3.4)
activejob (= 6.0.3.4)
activerecord (= 6.0.3.4)
activestorage (6.0.3.5)
actionpack (= 6.0.3.5)
activejob (= 6.0.3.5)
activerecord (= 6.0.3.5)
marcel (~> 0.3.1)
activesupport (6.0.3.4)
activesupport (6.0.3.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 0.7, < 2)
minitest (~> 5.1)
Expand All @@ -61,11 +61,11 @@ GEM
annotate (3.1.1)
activerecord (>= 3.2, < 7.0)
rake (>= 10.4, < 14.0)
ast (2.4.1)
ast (2.4.2)
builder (3.2.4)
byebug (11.1.3)
chronic (0.10.2)
concurrent-ruby (1.1.7)
concurrent-ruby (1.1.8)
crass (1.0.6)
diff-lcs (1.4.4)
doorkeeper (5.4.0)
Expand All @@ -78,11 +78,11 @@ GEM
factory_bot_rails (4.11.1)
factory_bot (~> 4.11.1)
railties (>= 3.0.0)
faker (2.15.1)
faker (2.16.0)
i18n (>= 1.6, < 2)
globalid (0.4.2)
activesupport (>= 4.2.0)
i18n (1.8.7)
i18n (1.8.9)
concurrent-ruby (~> 1.0)
loofah (2.9.0)
crass (~> 1.0.2)
Expand All @@ -95,8 +95,8 @@ GEM
mimemagic (0.3.5)
mini_mime (1.0.2)
mini_portile2 (2.5.0)
minitest (5.14.3)
nio4r (2.5.4)
minitest (5.14.4)
nio4r (2.5.7)
nokogiri (1.11.1)
mini_portile2 (~> 2.5.0)
racc (~> 1.4)
Expand All @@ -105,62 +105,62 @@ GEM
parallel (1.20.1)
parser (3.0.0.0)
ast (~> 2.4.1)
puma (5.1.1)
puma (5.2.2)
nio4r (~> 2.0)
racc (1.5.2)
rack (2.2.3)
rack-cors (1.1.1)
rack (>= 2.0.0)
rack-test (1.1.0)
rack (>= 1.0, < 3)
rails (6.0.3.4)
actioncable (= 6.0.3.4)
actionmailbox (= 6.0.3.4)
actionmailer (= 6.0.3.4)
actionpack (= 6.0.3.4)
actiontext (= 6.0.3.4)
actionview (= 6.0.3.4)
activejob (= 6.0.3.4)
activemodel (= 6.0.3.4)
activerecord (= 6.0.3.4)
activestorage (= 6.0.3.4)
activesupport (= 6.0.3.4)
rails (6.0.3.5)
actioncable (= 6.0.3.5)
actionmailbox (= 6.0.3.5)
actionmailer (= 6.0.3.5)
actionpack (= 6.0.3.5)
actiontext (= 6.0.3.5)
actionview (= 6.0.3.5)
activejob (= 6.0.3.5)
activemodel (= 6.0.3.5)
activerecord (= 6.0.3.5)
activestorage (= 6.0.3.5)
activesupport (= 6.0.3.5)
bundler (>= 1.3.0)
railties (= 6.0.3.4)
railties (= 6.0.3.5)
sprockets-rails (>= 2.0.0)
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
rails-html-sanitizer (1.3.0)
loofah (~> 2.3)
railties (6.0.3.4)
actionpack (= 6.0.3.4)
activesupport (= 6.0.3.4)
railties (6.0.3.5)
actionpack (= 6.0.3.5)
activesupport (= 6.0.3.5)
method_source
rake (>= 0.8.7)
thor (>= 0.20.3, < 2.0)
rainbow (3.0.0)
rake (13.0.3)
regexp_parser (2.0.3)
regexp_parser (2.1.1)
rexml (3.2.4)
rspec-core (3.10.1)
rspec-support (~> 3.10.0)
rspec-expectations (3.10.1)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.10.0)
rspec-mocks (3.10.1)
rspec-mocks (3.10.2)
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.10.0)
rspec-rails (4.0.2)
actionpack (>= 4.2)
activesupport (>= 4.2)
railties (>= 4.2)
rspec-rails (5.0.0)
actionpack (>= 5.2)
activesupport (>= 5.2)
railties (>= 5.2)
rspec-core (~> 3.10)
rspec-expectations (~> 3.10)
rspec-mocks (~> 3.10)
rspec-support (~> 3.10)
rspec-support (3.10.1)
rubocop (1.8.1)
rspec-support (3.10.2)
rubocop (1.11.0)
parallel (~> 1.10)
parser (>= 3.0.0.0)
rainbow (>= 2.2.2, < 4.0)
Expand All @@ -169,7 +169,7 @@ GEM
rubocop-ast (>= 1.2.0, < 2.0)
ruby-progressbar (~> 1.7)
unicode-display_width (>= 1.4.0, < 3.0)
rubocop-ast (1.4.0)
rubocop-ast (1.4.1)
parser (>= 2.7.1.5)
ruby-progressbar (1.11.0)
rubyzip (2.3.0)
Expand All @@ -182,11 +182,11 @@ GEM
sprockets (>= 3.0.0)
sqlite3 (1.4.2)
swagger-blocks (3.0.0)
thor (1.0.1)
thor (1.1.0)
thread_safe (0.3.6)
tzinfo (1.2.9)
thread_safe (~> 0.1)
tzinfo-data (1.2020.6)
tzinfo-data (1.2021.1)
tzinfo (>= 1.0.0)
unicode-display_width (2.0.0)
websocket-driver (0.7.3)
Expand Down
21 changes: 17 additions & 4 deletions app/jobs/csv_import_job.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ class CsvImportJob < ApplicationJob # rubocop:disable Metrics/ClassLength
def perform(*)
@logger = ActiveSupport::Logger.new(Rails.root.join(CSV_IMPORT_LOG))
@logger.info("----- Start CSV import job: #{Time.zone.now} -----")
return unless abstract_zip
return unless zip_to_csv
return unless File.exist?(get_filepath('manifest'))

manifest_table = CSV.read(get_filepath('manifest'), headers: true)
Expand All @@ -29,14 +29,15 @@ def perform(*)
end
# backup zip file or csv files depending on the value of ZIP_MODE
ZIP_MODE ? remove_csv : backup_csv
remove_old_backups
@logger.info("----- End CSV import job: #{Time.zone.now} -----")
end

private

def destroy_unused(cls, condition, csv_sourcedIds)
db_sourcedIds = cls.where(condition.merge({ application_id: 0 })).pluck('sourcedId')
db_sourcedIds.reject! { |id| csv_sourcedIds.include? id }
db_sourcedIds -= csv_sourcedIds
return if db_sourcedIds.empty?

cls.where(sourcedId: db_sourcedIds).destroy_all
Expand All @@ -52,9 +53,9 @@ def filename_to_class(str)
end
end

def abstract_zip
def zip_to_csv
return true unless ZIP_MODE # continue importing process if ZIP_MODE=false
return false if extract_to_csv.nil? # stop processing if zip file not exists
return false if extract_to_csv.nil? # stop processing if zip file does not exist

backup_zip
true
Expand Down Expand Up @@ -103,6 +104,18 @@ def remove_csv
end
end

def remove_old_backups
return unless CSV_BACKUP_DAYS.positive?
FileUtils.cd(Rails.root.join(CSV_FILE_PATH, CSV_BACKUP_DIR))
backups = ZIP_MODE ? '*.zip' : '*/'
Dir.glob(backups) do |f|
if File.mtime(f) < Time.zone.now - CSV_BACKUP_DAYS
ZIP_MODE ? FileUtils.rm(f) : FileUtils.remove_dir(f)
@logger.info("Removed old backups: #{f}")
end
end
end

def csv_to_db(csv_file, cls)
metadata_flag = METADATA[csv_file.to_sym].present?
fullname_flag = (cls == User) && FULLNAME_IN_FAMILYNAME
Expand Down
1 change: 1 addition & 0 deletions config/initializers/constants.rb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
APIDOC = 'doc/apidoc.json'.freeze
CSV_BACKUP_DIR = 'backup'.freeze
CSV_BACKUP_DAYS = 7.days.freeze
CSV_FILE_PATH = 'storage/csv'.freeze
CSV_IMPORT_LOG = 'log/csv_import.log'.freeze
CSV_ZIP_FILE = 'oneroster.zip'.freeze
Expand Down
32 changes: 0 additions & 32 deletions config/secrets.yml
View file

This file was deleted.

0 comments on commit ad1fe52

Please sign in to comment.