Skip to content

Commit

Permalink
Unpin pandas-gbq and remove unused code (apache#21915)
Browse files Browse the repository at this point in the history
* Unpin ``pandas-gbq`` and remove unused code

`BigQueryPandasConnector` was previously used by `BigqueryHook.get_pandas_df`. This was fixed in apache@ad308ea (**in 2018**). However we forgot to remove `BigQueryPandasConnector` which is age-old code (2016) and used private methods.

2016 code - https://github.com/apache/airflow/pull/1452/files
  • Loading branch information
kaxil authored Mar 2, 2022
1 parent 3052268 commit 84ed747
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 31 deletions.
1 change: 0 additions & 1 deletion airflow/contrib/hooks/bigquery_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
BigQueryConnection,
BigQueryCursor,
BigQueryHook,
BigQueryPandasConnector,
GbqConnector,
)

Expand Down
28 changes: 1 addition & 27 deletions airflow/providers/google/cloud/hooks/bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,7 @@
from googleapiclient.discovery import Resource, build
from pandas import DataFrame
from pandas_gbq import read_gbq
from pandas_gbq.gbq import (
GbqConnector,
_check_google_client_version as gbq_check_google_client_version,
_test_google_api_imports as gbq_test_google_api_imports,
)
from pandas_gbq.gbq import GbqConnector # noqa
from sqlalchemy import create_engine

from airflow.exceptions import AirflowException
Expand Down Expand Up @@ -2193,28 +2189,6 @@ def run_query(
return job.job_id


class BigQueryPandasConnector(GbqConnector):
"""
This connector behaves identically to GbqConnector (from Pandas), except
that it allows the service to be injected, and disables a call to
self.get_credentials(). This allows Airflow to use BigQuery with Pandas
without forcing a three legged OAuth connection. Instead, we can inject
service account credentials into the binding.
"""

def __init__(
self, project_id: str, service: str, reauth: bool = False, verbose: bool = False, dialect="legacy"
) -> None:
super().__init__(project_id)
gbq_check_google_client_version()
gbq_test_google_api_imports()
self.project_id = project_id
self.reauth = reauth
self.service = service
self.verbose = verbose
self.dialect = dialect


class BigQueryConnection:
"""
BigQuery does not have a notion of a persistent connection. Thus, these
Expand Down
4 changes: 1 addition & 3 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,9 +355,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
'grpcio-gcp>=0.2.2',
'httpx',
'json-merge-patch>=0.2',
# pandas-gbq 0.15.0 release broke google provider's bigquery import
# _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49)
'pandas-gbq<0.15.0',
'pandas-gbq',
pandas_requirement,
'sqlalchemy-bigquery>=1.2.1',
]
Expand Down

0 comments on commit 84ed747

Please sign in to comment.