Version 1.0.14

Fixing bug in database arn resolution for tagging Adding exception handler for where caller doesn't have iam::PassRole on the crawler role they supply
rmr1154 · Mar 4, 2022 · a1218ff · a1218ff
1 parent 9917d2f
commit a1218ff
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 35 deletions.
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = aws-data-mesh-utils
-version = 1.0.13
+version = 1.0.14
 author = Ian Meyers
 author_email = [email protected]
 license = Apache 2.0

diff --git a/src/data_mesh_util/DataMeshProducer.py b/src/data_mesh_util/DataMeshProducer.py
@@ -47,7 +47,8 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo
         self._session, _producer_credentials, _producer_arn = utils.assume_iam_role(
             role_name=DATA_MESH_PRODUCER_ROLENAME,
             region_name=self._current_region,
-            use_credentials=use_credentials)
+            use_credentials=use_credentials
+        )
 
         self._iam_client = self._session.client('iam')
         self._sts_client = self._session.client('sts')
@@ -282,7 +283,7 @@ def create_data_products(self, source_database_name: str,
         # get or create a data mesh shared database in the producer account
         self._producer_automator.get_or_create_database(
             database_name=data_mesh_database_name,
-            database_desc="Database to contain objects objects shared with the Data Mesh Account",
+            database_desc="Database to contain objects objects shared with the Data Mesh Account"
         )
         self._logger.info("Validated Producer Account Database %s" % data_mesh_database_name)
 

diff --git a/src/data_mesh_util/lib/ApiAutomator.py b/src/data_mesh_util/lib/ApiAutomator.py
@@ -917,36 +917,36 @@ def create_crawler(self, crawler_role: str, database_name: str, table_name: str,
         try:
             glue_client.get_crawler(Name=crawler_name)
         except glue_client.exceptions.from_code('EntityNotFoundException'):
-            glue_client.create_crawler(
-                Name=crawler_name,
-                Role=crawler_role,
-                DatabaseName=database_name,
-                Description="S3 Crawler to sync structure of %s.%s to Data Mesh" % (database_name, table_name),
-                Targets={
-                    'S3Targets': [
-                        {
-                            'Path': s3_location
-                        },
-                    ]
-                },
-                Schedule="cron(0 */4 * * ? *)" if sync_schedule is None else sync_schedule,
-                SchemaChangePolicy={
-                    'UpdateBehavior': 'LOG',
-                    'DeleteBehavior': 'LOG'
-                },
-                RecrawlPolicy={
-                    'RecrawlBehavior': 'CRAWL_NEW_FOLDERS_ONLY'
-                },
-                LineageConfiguration={
-                    'CrawlerLineageSettings': 'ENABLE' if enable_lineage is True else 'DISABLE'
-                },
-                Tags=DEFAULT_TAGS
-            )
-            self._logger.info("Created new Glue Crawler %s" % crawler_name)
-
-        # create lakeformation permissions in the mesh account for the glue crawler role
-
-        # create s3 permission for glue crawler role
+            try:
+                glue_client.create_crawler(
+                    Name=crawler_name,
+                    Role=crawler_role,
+                    DatabaseName=database_name,
+                    Description="S3 Crawler to sync structure of %s.%s to Data Mesh" % (database_name, table_name),
+                    Targets={
+                        'S3Targets': [
+                            {
+                                'Path': s3_location
+                            },
+                        ]
+                    },
+                    Schedule="cron(0 */4 * * ? *)" if sync_schedule is None else sync_schedule,
+                    SchemaChangePolicy={
+                        'UpdateBehavior': 'LOG',
+                        'DeleteBehavior': 'LOG'
+                    },
+                    RecrawlPolicy={
+                        'RecrawlBehavior': 'CRAWL_NEW_FOLDERS_ONLY'
+                    },
+                    LineageConfiguration={
+                        'CrawlerLineageSettings': 'ENABLE' if enable_lineage is True else 'DISABLE'
+                    },
+                    Tags=DEFAULT_TAGS
+                )
+                self._logger.info("Created new Glue Crawler %s" % crawler_name)
+            except glue_client.exceptions.AccessDeniedException as ade:
+                self._logger.error(
+                    "Cannot create Glue Crawler - caller doesn't have permissions or is missing iam::PassRole")
 
         return crawler_name
 
@@ -992,8 +992,11 @@ def get_or_create_database(self, database_name: str, database_desc: str, source_
                 **args
             )
 
+            this_account_id = self._get_client('sts').get_caller_identity().get('Account')
+
             # tag the database with default tags
-            db_arn = utils.get_db_arn(region_name=glue_client.meta.region_name, database_name=database_name)
+            db_arn = utils.get_db_arn(region_name=glue_client.meta.region_name, catalog_id=this_account_id,
+                                      database_name=database_name)
             glue_client.tag_resource(
                 ResourceArn=db_arn,
                 TagsToAdd=DEFAULT_TAGS

diff --git a/src/data_mesh_util/resource/producer_account_policy.pystache b/src/data_mesh_util/resource/producer_account_policy.pystache
@@ -23,7 +23,8 @@
                 "glue:CreateTable",
                 "glue:CreateTables",
                 "glue:CreateCrawler",
-                "glue:Update*"
+                "glue:Update*",
+                "glue:TagResource"
             ],
             "Resource": "*"
         },