Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incu…

…bator-airflow
askmeegs · Aug 23, 2017 · b8b71ff · b8b71ff
2 parents 553744a + 4a4b024
commit b8b71ff
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/airflow/contrib/hooks/bigquery_hook.py b/airflow/contrib/hooks/bigquery_hook.py
@@ -385,6 +385,7 @@ def run_load(self,
                  field_delimiter=',',
                  max_bad_records=0,
                  quote_character=None,
+                 allow_quoted_newlines=False,
                  schema_update_options=()):
         """
         Executes a BigQuery load command to load data from Google Cloud Storage
@@ -421,6 +422,8 @@ def run_load(self,
         :type max_bad_records: int
         :param quote_character: The value that is used to quote data sections in a CSV file.
         :type quote_character: string
+        :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).
+        :type allow_quoted_newlines: boolean
         :param schema_update_options: Allows the schema of the desitination
             table to be updated as a side effect of the load job.
         :type schema_update_options: list
@@ -500,6 +503,9 @@ def run_load(self,
         if quote_character:
             configuration['load']['quote'] = quote_character
 
+        if allow_quoted_newlines:
+            configuration['load']['allowQuotedNewlines'] = allow_quoted_newlines
+
         return self.run_with_configuration(configuration)
 
     def run_with_configuration(self, configuration):

diff --git a/airflow/contrib/operators/gcs_to_bq.py b/airflow/contrib/operators/gcs_to_bq.py
@@ -45,6 +45,7 @@ def __init__(
         field_delimiter=',',
         max_bad_records=0,
         quote_character=None,
+        allow_quoted_newlines=False,
         max_id_key=None,
         bigquery_conn_id='bigquery_default',
         google_cloud_storage_conn_id='google_cloud_storage_default',
@@ -87,6 +88,8 @@ def __init__(
         :type max_bad_records: int
         :param quote_character: The value that is used to quote data sections in a CSV file.
         :type quote_character: string
+        :param allow_quoted_newlines: Whether to allow quoted newlines (true) or not (false).
+        :type allow_quoted_newlines: boolean
         :param max_id_key: If set, the name of a column in the BigQuery table
             that's to be loaded. Thsi will be used to select the MAX value from
             BigQuery after the load occurs. The results will be returned by the
@@ -124,6 +127,7 @@ def __init__(
         self.field_delimiter = field_delimiter
         self.max_bad_records = max_bad_records
         self.quote_character = quote_character
+        self.allow_quoted_newlines = allow_quoted_newlines
 
         self.max_id_key = max_id_key
         self.bigquery_conn_id = bigquery_conn_id
@@ -161,6 +165,7 @@ def execute(self, context):
             field_delimiter=self.field_delimiter,
             max_bad_records=self.max_bad_records,
             quote_character=self.quote_character,
+            allow_quoted_newlines=self.allow_quoted_newlines,
             schema_update_options=self.schema_update_options)
 
         if self.max_id_key: