Parameterizing DagBag import timeouts

airflow/configuration.py

@@ -78,6 +78,7 @@ class AirflowConfigException(Exception):
         'dags_are_paused_at_creation': False,
         'sql_alchemy_pool_size': 5,
         'sql_alchemy_pool_recycle': 3600,
+        'dagbag_import_timeout': 30,
     },
     'webserver': {
         'base_url': 'http://localhost:8080',
@@ -183,6 +184,9 @@ class AirflowConfigException(Exception):
 # Whether to disable pickling dags
 donot_pickle = False
 
+# How long before timing out a python file import while filing the DagBag
+dagbag_import_timeout = 30
+
 [webserver]
 # The base url of your website as airflow cannot guess what domain or
 # cname you are using. This is use in automated emails that

airflow/models.py

@@ -204,7 +204,8 @@ def process_file(self, filepath, only_if_updated=True, safe_mode=True):
                 self.logger.info("Importing " + filepath)
                 if mod_name in sys.modules:
                     del sys.modules[mod_name]
-                with utils.timeout(30):
+                with utils.timeout(
+                        configuration.getint('core', "DAGBAG_IMPORT_TIMEOUT")):
                     m = imp.load_source(mod_name, filepath)
             except Exception as e:
                 self.logger.exception("Failed to import: " + filepath)