[ext] Storage delete toggle (HumanSignal#1541)

* [ext] Storage delete toggle * Remove DELETION_FROM_S3_ENABLED_FOR_ORGS * Remove fixture Co-authored-by: nik <[email protected]>
sharpboy2008 · Sep 30, 2021 · 722358a · 722358a
1 parent de05dc9
commit 722358a
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 13 deletions.
diff --git a/label_studio/core/settings/base.py b/label_studio/core/settings/base.py
@@ -368,8 +368,6 @@
 TASK_LOCK_DEFAULT_TTL = int(get_env('TASK_LOCK_DEFAULT_TTL', 3600))
 TASK_LOCK_MIN_TTL = int(get_env('TASK_LOCK_MIN_TTL', 120))
 
-DELETION_FROM_S3_ENABLED_FOR_ORGS = get_env_list_int('DELETION_FROM_S3_ENABLED_FOR_ORGS', [])
-
 # Email backend
 FROM_EMAIL = get_env('FROM_EMAIL', 'Label Studio <[email protected]>')
 EMAIL_BACKEND = get_env('EMAIL_BACKEND', 'django.core.mail.backends.dummy.EmailBackend')

diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py
@@ -160,6 +160,8 @@ def sync_background(storage_class, storage_id):
 
 
 class ExportStorage(Storage):
+    can_delete_objects = models.BooleanField(_('can_delete_objects'), null=True, blank=True, help_text='Deletion from storage enabled')
+
     def _get_serialized_data(self, annotation):
         if get_bool_env('FUTURE_SAVE_TASK_TO_STORAGE', default=False):
             # export task with annotations

diff --git a/label_studio/io_storages/migrations/0007_auto_20210928_1252.py b/label_studio/io_storages/migrations/0007_auto_20210928_1252.py
@@ -0,0 +1,38 @@
+# Generated by Django 3.1.13 on 2021-09-28 12:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('io_storages', '0006_auto_20210906_1323'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='azureblobexportstorage',
+            name='can_delete_objects',
+            field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'),
+        ),
+        migrations.AddField(
+            model_name='gcsexportstorage',
+            name='can_delete_objects',
+            field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'),
+        ),
+        migrations.AddField(
+            model_name='localfilesexportstorage',
+            name='can_delete_objects',
+            field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'),
+        ),
+        migrations.AddField(
+            model_name='redisexportstorage',
+            name='can_delete_objects',
+            field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'),
+        ),
+        migrations.AddField(
+            model_name='s3exportstorage',
+            name='can_delete_objects',
+            field=models.BooleanField(blank=True, help_text='Deletion from storage enabled', null=True, verbose_name='can_delete_objects'),
+        ),
+    ]
diff --git a/label_studio/io_storages/s3/form_layout.yml b/label_studio/io_storages/s3/form_layout.yml
@@ -89,3 +89,12 @@ ExportStorage:
     fields: *title_bucket_prefix
   - columnCount: 3
     fields: *aws_params
+  # 1 columns grid
+  - columnCount: 1
+    columns:
+      - width: 345
+        fields:
+          - type: toggle
+            name: can_delete_objects
+            label: Can delete objects from storage
+            description: If unchecked, annotations will not be deleted from storage
diff --git a/label_studio/io_storages/s3/models.py b/label_studio/io_storages/s3/models.py
@@ -78,6 +78,7 @@ def get_client_and_bucket(self, validate_connection=True):
         return client, s3.Bucket(self.bucket)
 
     def validate_connection(self, client=None):
+        print('validate_connection')
         if client is None:
             client = self.get_client()
         if self.prefix:
@@ -203,14 +204,7 @@ def delete_annotation(self, annotation):
         key = str(self.prefix) + '/' + key if self.prefix else key
 
         # delete object from storage
-        try:
-            s3.Object(self.bucket, key).delete()
-        except ClientError as e:
-            # we ignore access denied errors
-            logger.exception(e)
-            if not 'Access Denied' in str(e):
-                raise
-
+        s3.Object(self.bucket, key).delete()
 
         # delete link if everything ok
         S3ExportStorageLink.objects.filter(storage=self, annotation=annotation).delete()
@@ -228,9 +222,9 @@ def export_annotation_to_s3_storages(sender, instance, **kwargs):
 @receiver(post_delete, sender=Annotation)
 def delete_annotation_from_s3_storages(sender, instance, **kwargs):
     project = instance.task.project
-    if project.organization_id in settings.DELETION_FROM_S3_ENABLED_FOR_ORGS:
-        if hasattr(project, 'io_storages_s3exportstorages'):
-            for storage in project.io_storages_s3exportstorages.all():
+    if hasattr(project, 'io_storages_s3exportstorages'):
+        for storage in project.io_storages_s3exportstorages.all():
+            if storage.can_delete_objects:
                 logger.debug(f'Delete {instance} from S3 storage {storage}')
                 storage.delete_annotation(instance)
 

diff --git a/label_studio/tests/io_storages.tavern.yml b/label_studio/tests/io_storages.tavern.yml
@@ -573,6 +573,12 @@ stages:
     url: '{django_live_url}/api/storages/export/s3/{storage_pk}/sync'
   response:
     status_code: 200
+- name: stage
+  request:
+    method: POST
+    url: '{django_live_url}/api/dm/actions?id=delete_tasks_annotations&project={project_pk}'
+  response:
+    status_code: 200
 ---
 test_name: test_export_gcs_storage
 strict: false