add logic for uploading migration logs after migration complete

GoogleSharePointMigrationAssistant/web/plumbing/base.py

@@ -1,15 +1,15 @@
 import logging 
 import os 
-from .constants import LOG_FOLDER_PATH
 from datetime import timedelta
 from pathlib import PurePath
 import math 
 
 class BaseUtil: 
-    def __init__(self, name: str = '', verbose: bool = False): 
+    def __init__(self, name: str = '', verbose: bool = False, username: str = '' ): 
         self.name = name
         self.verbose = verbose 
         self.setup_logging() 
+        self.log_folder_path = os.path.join(os.path.dirname(__file__), f'migration-logs-{username}')
 
     def less_than_4mb(self, bytes_size: int = 0): 
         bytes_in_4mb = 1024 * 1024 * 4
@@ -36,8 +36,8 @@ def setup_logging(self):
         handlerStream = logging.StreamHandler()
         handlerStream.setFormatter(formatter) 
         self.logger.addHandler(handlerStream)  
-        os.makedirs(LOG_FOLDER_PATH, exist_ok=True) 
-        handlerFile = logging.FileHandler(f'{LOG_FOLDER_PATH}/{self.name}.log')
+        os.makedirs(self.log_folder_path, exist_ok=True) 
+        handlerFile = logging.FileHandler(f'{self.log_folder_path}/{self.name}.log')
         handlerFile.setFormatter(formatter) 
         self.logger.addHandler(handlerFile)  
         if self.verbose:

GoogleSharePointMigrationAssistant/web/plumbing/googletosharepoint.py

@@ -32,7 +32,7 @@ def __init__(self,
     migration: Migration = None, 
     google_credentials: dict = {},
     ): 
-        super().__init__(name=name, verbose=verbose)
+        super().__init__(name=name, verbose=verbose, username=migration.user.username)
         self.admin_config = AdministrationSettings.objects.first()
         self.migration = migration
         self.file_batch_size = file_batch_size

GoogleSharePointMigrationAssistant/web/plumbing/migrationassistant.py

@@ -1,4 +1,5 @@
 import time 
+import shutil
 from celery import shared_task
 from .constants import *  
 from .sharepoint import SharePointUploader
@@ -9,6 +10,9 @@
 from ..models import Migration
 from django.contrib.auth.models import User
 from django.core.cache import cache as django_cache
+import logging 
+
+logger = logging.getLogger(__name__)
 
 def get_migration_from_cache(migration_id):
     migration = django_cache.get(f'migration-{migration_id}', None)
@@ -26,7 +30,7 @@ def __init__(
         user: User = None,
         m365_token_cache: dict = {}
         ): 
-        super().__init__(name=name, verbose=verbose)      
+        super().__init__(name=name, verbose=verbose, username=user.username)      
         self.migration = migration
         self.m365_token_cache = m365_token_cache
         self.google_credentials = google_credentials
@@ -70,7 +74,7 @@ def set_file_batch_size(self, fbs):
         self.downloader.file_batch_size = fbs 
 
     def notify_completion(self): 
-        self.notifier = Notifier() 
+        self.notifier = Notifier(migration=self.migration)
         self.notifier.notify_completion(
             migration=self.migration,  
             num_files_migrated=self.downloader.num_files_downloaded,# FIXME: should be stored as part of migration data, not downloader
@@ -84,24 +88,22 @@ def upload_logs_to_destination(self):
         Log files are going to get big. When migration is finished, upload 
         logs to the same destination, adjacent to the folder uploaded during migration. 
         """
-        self.info(f'Uploading logs in {LOG_FOLDER_PATH} to sharepoint and then deleting from local system')
+        self.info({
+            'upload_logs_to_destination': {
+                'log_folder_path': self.log_folder_path
+            }
+        })
         self.shutdown_logging()
         self.uploader.shutdown_logging()  
         self.notifier.shutdown_logging() 
-        if self.migration.target_type == 'sharepoint_folder':
-            self.uploader.configure(
-                migration=self.migration,
-                local_temp_dir=self.local_temp_dir,
-                use_multithreading=True
-            )
-            self.uploader.upload(local_folder_base_path=LOG_FOLDER_PATH) 
-
-        elif self.migration.target_type == 'onedrive_folder': 
-            self.uploader.num_completed_uploads = 0 
-            self.uploader.upload(
-                local_folder_base_path=LOG_FOLDER_PATH
-            )  
+        self.uploader.configure(
+            migration=self.migration,
+            use_multithreading=True
+        )
+        self.uploader.num_completed_uploads = 0
+        self.uploader.upload(local_folder_base_path=self.log_folder_path) 
 
+
     def migrate(self):
         start = time.time()
         response = self.downloader.migrate()
@@ -119,6 +121,18 @@ def migrate(self):
     def scan_data_source(self):
         return self.downloader.scan()
 
+
+def clear_logs(assistant: MigrationAssistant = None):
+    if assistant:
+        assistant.shutdown_logging()
+        assistant.uploader.shutdown_logging()
+        assistant.downloader.shutdown_logging() 
+    try:
+        shutil.rmtree(assistant.log_folder_path, ignore_errors=True)
+    except Exception as e:
+        logger.error({'clear_logs': {'error': e}})
+    time.sleep(1)
+
 @shared_task
 def scan_data_source(migration_id: int = 0, google_credentials: dict = {}, user_id: int = 0):
     """ Scan source data asynchronously """
@@ -153,20 +167,10 @@ def migrate_data(migration_id: int = 0, google_credentials: dict = {}, user_id:
     migration_response = assistant.migrate()
     migration.state = Migration.STATES.MIGRATION_COMPLETE
     migration.save()
+
+    assistant.upload_logs_to_destination()
+    clear_logs(assistant)
+
     # TODO: Save something to model for migration report 
     assistant.notify_completion()
     return migration_response
-
-
-# def clear_logs(assistant: MigrationAssistant = None):
-#     print('Clearing logs')
-#     if assistant:
-#         assistant.shutdown_logging()
-#         assistant.uploader.shutdown_logging()
-#         assistant.downloader.shutdown_logging() 
-#     try:
-#         shutil.rmtree(LOG_FOLDER_PATH, ignore_errors=True)
-#     except Exception as e:
-#         print(f'Failed to remove log directory {LOG_FOLDER_PATH}')
-#         print(e)
-#     time.sleep(1)

GoogleSharePointMigrationAssistant/web/plumbing/notif/notifier.py

@@ -10,10 +10,11 @@
 from ..base import BaseUtil
 
 class Notifier(BaseUtil): 
-    def __init__(self, name: str = 'Notifier'):
+    def __init__(self, name: str = 'Notifier', migration: Migration = None):
         super().__init__(
             name=name, 
-            verbose=False
+            verbose=False,
+            username=migration.user.username
             ) 
         config = cache.get('config', AdministrationSettings.objects.first())
         cache.set('config', config)

GoogleSharePointMigrationAssistant/web/plumbing/onedrive.py

@@ -18,7 +18,7 @@ def __init__(self,
                  local_folder_base_path: str = '',
                  verbose: bool = False,
                  username: str = ''):
-        super().__init__(name, verbose)
+        super().__init__(name, verbose, username=username)
 
         self.username = username
         self.m365_token_cache = m365_token_cache
@@ -339,12 +339,16 @@ def upload(self, local_folder_base_path: str = ''):
                 remote_parent_folder_id=remote_parent_folder_id
             )
             if not response:
-                self.error('Folder was not uploaded')
+                self.error({
+                    'upload': {
+                        'error': f'folder not uploaded',
+                        'local_folder_base_path': local_folder_base_path,
+                        'remote_parent_folder_id': remote_parent_folder_id
+                    }
+                })
         else:
             self.error(f'{local_folder_base_path} is not a directory')
         while self._num_active_uploads > 0:
-            self.info(
-                f'Waiting for completion of active uploads ({self._num_active_uploads})')
+            self.info({'upload': {'self._num_active_uploads': self._num_active_uploads}})
             time.sleep(5)
-        self.info("All enqueued upload tasks complete. Upload finished.")
-
+        self.info({'upload': {'complete': 'all enqueued upload tasks complete'}})

GoogleSharePointMigrationAssistant/web/plumbing/sharepoint.py

@@ -19,7 +19,7 @@ def __init__(self,
                  verbose: bool = False,
                  name: str = 'SPUploader'):
 
-        super().__init__(name=name, verbose=verbose)
+        super().__init__(name=name, verbose=verbose, username=migration.user.username)
         self.migration = migration
         self.m365_token_cache = m365_token_cache
         self.use_multithreading = use_multithreading