From 722358a6cdfbe5a35e7b16f586675df4b598f74f Mon Sep 17 00:00:00 2001 From: Sergei Ivashchenko Date: Thu, 30 Sep 2021 12:52:05 +0300 Subject: [PATCH] [ext] Storage delete toggle (#1541) * [ext] Storage delete toggle * Remove DELETION_FROM_S3_ENABLED_FOR_ORGS * Remove fixture Co-authored-by: nik --- label_studio/core/settings/base.py | 2 - label_studio/io_storages/base_models.py | 2 + .../migrations/0007_auto_20210928_1252.py | 38 +++++++++++++++++++ label_studio/io_storages/s3/form_layout.yml | 9 +++++ label_studio/io_storages/s3/models.py | 16 +++----- label_studio/tests/io_storages.tavern.yml | 6 +++ 6 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 label_studio/io_storages/migrations/0007_auto_20210928_1252.py diff --git a/label_studio/core/settings/base.py b/label_studio/core/settings/base.py index d38833316f28..e593e04f2f64 100644 --- a/label_studio/core/settings/base.py +++ b/label_studio/core/settings/base.py @@ -368,8 +368,6 @@ TASK_LOCK_DEFAULT_TTL = int(get_env('TASK_LOCK_DEFAULT_TTL', 3600)) TASK_LOCK_MIN_TTL = int(get_env('TASK_LOCK_MIN_TTL', 120)) -DELETION_FROM_S3_ENABLED_FOR_ORGS = get_env_list_int('DELETION_FROM_S3_ENABLED_FOR_ORGS', []) - # Email backend FROM_EMAIL = get_env('FROM_EMAIL', 'Label Studio ') EMAIL_BACKEND = get_env('EMAIL_BACKEND', 'django.core.mail.backends.dummy.EmailBackend') diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py index 5ad736018ba5..c19437c60022 100644 --- a/label_studio/io_storages/base_models.py +++ b/label_studio/io_storages/base_models.py @@ -160,6 +160,8 @@ def sync_background(storage_class, storage_id): class ExportStorage(Storage): + can_delete_objects = models.BooleanField(_('can_delete_objects'), null=True, blank=True, help_text='Deletion from storage enabled') + def _get_serialized_data(self, annotation): if get_bool_env('FUTURE_SAVE_TASK_TO_STORAGE', default=False): # export task with annotations diff --git a/label_studio/io_storages/migrations/0007_auto_20210928_1252.py b/label_studio/io_storages/migrations/0007_auto_20210928_1252.py new file mode 100644 index 000000000000..6b45e56cd227 --- /dev/null +++ b/label_studio/io_storages/migrations/0007_auto_20210928_1252.py @@ -0,0 +1,38 @@ +# Generated by Django 3.1.13 on 2021-09-28 12:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('io_storages', '0006_auto_20210906_1323'), + ] + + operations = [ + migrations.AddField( + model_name='azureblobexportstorage', + name='can_delete_objects', + field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'), + ), + migrations.AddField( + model_name='gcsexportstorage', + name='can_delete_objects', + field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'), + ), + migrations.AddField( + model_name='localfilesexportstorage', + name='can_delete_objects', + field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'), + ), + migrations.AddField( + model_name='redisexportstorage', + name='can_delete_objects', + field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'), + ), + migrations.AddField( + model_name='s3exportstorage', + name='can_delete_objects', + field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'), + ), + ] diff --git a/label_studio/io_storages/s3/form_layout.yml b/label_studio/io_storages/s3/form_layout.yml index c2997c0e5998..7ed2de2e19e7 100644 --- a/label_studio/io_storages/s3/form_layout.yml +++ b/label_studio/io_storages/s3/form_layout.yml @@ -89,3 +89,12 @@ ExportStorage: fields: *title_bucket_prefix - columnCount: 3 fields: *aws_params + # 1 columns grid + - columnCount: 1 + columns: + - width: 345 + fields: + - type: toggle + name: can_delete_objects + label: Can delete objects from storage + description: If unchecked, annotations will not be deleted from storage diff --git a/label_studio/io_storages/s3/models.py b/label_studio/io_storages/s3/models.py index 4d72795d1e79..7ce686995b9c 100644 --- a/label_studio/io_storages/s3/models.py +++ b/label_studio/io_storages/s3/models.py @@ -78,6 +78,7 @@ def get_client_and_bucket(self, validate_connection=True): return client, s3.Bucket(self.bucket) def validate_connection(self, client=None): + print('validate_connection') if client is None: client = self.get_client() if self.prefix: @@ -203,14 +204,7 @@ def delete_annotation(self, annotation): key = str(self.prefix) + '/' + key if self.prefix else key # delete object from storage - try: - s3.Object(self.bucket, key).delete() - except ClientError as e: - # we ignore access denied errors - logger.exception(e) - if not 'Access Denied' in str(e): - raise - + s3.Object(self.bucket, key).delete() # delete link if everything ok S3ExportStorageLink.objects.filter(storage=self, annotation=annotation).delete() @@ -228,9 +222,9 @@ def export_annotation_to_s3_storages(sender, instance, **kwargs): @receiver(post_delete, sender=Annotation) def delete_annotation_from_s3_storages(sender, instance, **kwargs): project = instance.task.project - if project.organization_id in settings.DELETION_FROM_S3_ENABLED_FOR_ORGS: - if hasattr(project, 'io_storages_s3exportstorages'): - for storage in project.io_storages_s3exportstorages.all(): + if hasattr(project, 'io_storages_s3exportstorages'): + for storage in project.io_storages_s3exportstorages.all(): + if storage.can_delete_objects: logger.debug(f'Delete {instance} from S3 storage {storage}') storage.delete_annotation(instance) diff --git a/label_studio/tests/io_storages.tavern.yml b/label_studio/tests/io_storages.tavern.yml index 7414d9f40cb8..ba78d4733bff 100644 --- a/label_studio/tests/io_storages.tavern.yml +++ b/label_studio/tests/io_storages.tavern.yml @@ -573,6 +573,12 @@ stages: url: '{django_live_url}/api/storages/export/s3/{storage_pk}/sync' response: status_code: 200 +- name: stage + request: + method: POST + url: '{django_live_url}/api/dm/actions?id=delete_tasks_annotations&project={project_pk}' + response: + status_code: 200 --- test_name: test_export_gcs_storage strict: false