RFC: Upgrade sqlalchemy

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.22
+current_version = 1.4.23
 commit = False
 tag = False
 

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.4.23] - 2023-06-09
+
+### Added
+
+- Support for SQLAlchemy >=1.3,<2.0.
+
 ## [1.4.22] - 2023-04-05
 
 ### Fixed

muttlib/dbconn/base.py

@@ -6,8 +6,9 @@
 from contextlib import closing
 
 import pandas as pd
-from sqlalchemy import create_engine
+from packaging import version
 from sqlalchemy.engine.url import URL
+from sqlalchemy import __version__ as sqlalchemy_version, create_engine
 
 import muttlib.utils as utils
 
@@ -126,6 +127,10 @@ def __setattr__(self, attr, value):
                 elif attr == 'driver':
                     value = format_drivername(self.dialect, value)
                 attr = 'drivername'
+
+            if version.parse(sqlalchemy_version) >= version.parse("1.4"):
+                self.conn_url = self.conn_url.set(**{attr: value})
+                return self.conn_url
         else:
             _cls = object
             target = self

muttlib/dbconn/ibis.py

@@ -115,11 +115,11 @@ def _connect(self):
             and self.hdfs_port is not None
             and self.hdfs_username is not None
         ):
-            self.hdfs_client = ibis.backends.impala.hdfs_connect(
+            self.hdfs_client = ibis.impala.hdfs_connect(
                 host=self.hdfs_host, port=self.hdfs_port, user=self.hdfs_username
             )
 
-        client = ibis.backends.impala.connect(
+        client = ibis.impala.connect(
             host=self.host,
             port=self.port,
             database=self.database,

muttlib/utils.py

@@ -167,7 +167,13 @@ def wrapped_f(*args, **kwargs):
 
 
 def df_read_multi(fn, index_col=False, quoting=0):
-    """Read multiple table disk-formats into a pandas DataFrame."""
+    """
+    Read multiple table disk-formats into a pandas DataFrame.
+
+    If the file has a '.pickle' or '.pkl' extension, keep in mind that
+    unpickling untrusted data is not secure.
+    https://docs.python.org/3/library/pickle.html
+    """
     ext = Path(fn).suffix[1:]
     if ext == 'csv':
         df = pd.read_csv(fn, index_col=index_col, quoting=quoting)
@@ -183,7 +189,7 @@ def clean_quotes(s):
     elif ext == 'feather':
         return pd.read_feather(fn)
     elif ext in ['pickle', 'pkl']:
-        return pd.read_pickle(fn)
+        return pd.read_pickle(fn)  # nosec
     else:
         raise ValueError(f"File format '{ext}' not supported!")
 
@@ -522,7 +528,7 @@ def str_normalize_pandas(data, str_replace_kws=None):
     """
 
     if isinstance(data, pd.DataFrame):
-        obj_cols = data.select_dtypes(include=[np.object]).columns
+        obj_cols = data.select_dtypes(include=[np.object_]).columns
         for col in obj_cols:
             data[col] = (
                 data[col]
@@ -534,7 +540,7 @@ def str_normalize_pandas(data, str_replace_kws=None):
             if str_replace_kws:
                 data[col] = data[col].str.replace(**str_replace_kws)
         return data
-    elif isinstance(data, pd.Series) and data.dtype == np.object:
+    elif isinstance(data, pd.Series) and data.dtype == np.object_:
         data = (
             data.str.lower()
             .str.lower()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "muttlib"
-version = "1.4.21"
+version = "1.4.23"
 description = "Collection of helper modules by Mutt Data."
 readme = "README.md"
 authors = [ "Mutt Data <[email protected]>",]
@@ -59,28 +59,27 @@ progressbar2 = "^3.55.0"
 PyYAML = ">=5.1,<7.0"
 scikit-learn = "^1.0.1"
 scipy = "^1.7.2"
-SQLAlchemy = "<1.4.0,>=1.3.0"
+SQLAlchemy = ">=1.3,<2.0"
 numpy = ">=1.22.0,<2.0"
 jinjasql = "^0.1.8"
-ipython = "^7.29.0"
+ipython = ">=8.0.1"
 matplotlib = "^3.5.0"
 pyarrow = { version = "6.0.0", optional = true }
 cx-Oracle = { version = "^8.3.0", optional = true }
 PyHive = { version = ">=0.6.1", optional = true }
-thrift = { version = "^0.15.0", optional = true }
+thrift = {version = "^0.16.0", optional = true}
 psycopg2-binary = { version = "^2.9.2", optional = true }
 PyMySQL = { version = "^1.0.2", optional = true }
 pymssql = { version = "^2.2.2", optional = true }
 pymongo = { version = "^3.12.1", optional = true }
 snowflake-connector-python = { version = "^2.7.1", optional = true }
 cryptography = { version = "3.4.7", optional = true }
-snowflake-sqlalchemy = { version = "1.2.4", optional = true }
+snowflake-sqlalchemy = { version = ">=1.2.4, <=1.4.4, !=1.2.5", optional = true } # https://github.com/snowflakedb/snowflake-sqlalchemy/issues/234
 teradatasql = { version = "17.0.0.8", optional = true }
-sqlalchemy-trino = { version = "0.4.0", optional = true }
 sqlalchemy-redshift = { version = "0.8.6", optional = true }
 pyparsing = { version = "<3", optional = true }
+ibis-framework = { version = "3.2.0", extras = [ "impala",], optional = true }
 google-cloud-bigquery = { version = "~2.34.4", optional = true }
-ibis-framework = { version = "1.4.0", extras = [ "impala",], optional = true }
 oauth2client = { version = "^4.1.3", optional = true }
 requests = { version = "^2.26.0", optional = true }
 freezegun = { version = "^1.1.0", optional = true }
@@ -95,7 +94,6 @@ betamax = { version = "^0.8.1", optional = true }
 betamax-serializers = { version = "^0.2.1", optional = true }
 holidays = { version = ">=0.10.2", optional = true }
 gspread-pandas = { version = "^2.3.0", optional = true }
-wrapt = "1.11.2"
 types-requests = "^2.27.7"
 types-PyMySQL = "^1.0.11"
 bandit = { version = "^1.7.2", optional = true }
@@ -106,6 +104,7 @@ Pillow = ">=9.1.1"
 ujson = ">=5.4.0"
 PyJWT = ">=2.4.0"
 lxml = ">=4.9.1"
+trino = {extras = ["sqlalchemy"], version = "0.320.0"}
 
 [tool.poetry.extras]
 pyarrow = [ "pyarrow",]

tests/dbconn/test_ibis.py

@@ -48,7 +48,7 @@ def test_temp_hdfs_path_changes_config():
 
 
 def test_execute_with_no_client():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         q = "SELECT *"
         ibis_cli.execute(q, return_cursor=False)
@@ -57,7 +57,7 @@ def test_execute_with_no_client():
 
 
 def test_execute_with_client():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         client = MagicMock()
         q = "SELECT *"
@@ -67,7 +67,7 @@ def test_execute_with_client():
 
 
 def test_execute_returns_cursor():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         client = MagicMock()
         q = "SELECT *"
@@ -76,7 +76,7 @@ def test_execute_returns_cursor():
 
 
 def test_to_frame_via_hdfs_and_no_hdfs_client_fails():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         q = "SELECT *"
         with pytest.raises(ValueError, match=r".*hdfs\Wclient.*"):
@@ -85,7 +85,7 @@ def test_to_frame_via_hdfs_and_no_hdfs_client_fails():
 
 
 def test_to_frame_via_hdfs_and_no_cache_dir_fails():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient(
             "host",
             hdfs_host="",
@@ -99,7 +99,7 @@ def test_to_frame_via_hdfs_and_no_cache_dir_fails():
 
 
 def test_to_frame_via_hdfs_and_refresh_cache():
-    with patch("ibis.backends.impala") as impala, patch("shutil.rmtree") as rm, patch(
+    with patch("ibis.impala") as impala, patch("shutil.rmtree") as rm, patch(
         "pyarrow.parquet.read_table"
     ) as pq:
         ibis_cli = IbisClient("host", hdfs_host="", hdfs_port="", hdfs_username="")
@@ -112,7 +112,7 @@ def test_to_frame_via_hdfs_and_refresh_cache():
 
 
 def test_to_frame_via_hdfs_creates_tmp_table():
-    with patch("ibis.backends.impala") as impala, patch(
+    with patch("ibis.impala") as impala, patch(
         "muttlib.dbconn.ibis.urlparse"
     ) as parse, patch("muttlib.utils.make_dirs") as make_dirs, patch(
         "muttlib.dbconn.ibis.pq"
@@ -153,7 +153,7 @@ def test_to_frame_via_hdfs_creates_tmp_table():
 
 
 def test_to_frame_via_hdfs_create_tmp_table_fails():
-    with patch("ibis.backends.impala") as impala, patch(
+    with patch("ibis.impala") as impala, patch(
         "muttlib.dbconn.ibis.sleep"
     ) as sleep, patch("muttlib.dbconn.ibis.urlparse") as parse, patch(
         "muttlib.utils.make_dirs"
@@ -170,7 +170,7 @@ def test_to_frame_via_hdfs_create_tmp_table_fails():
 
 
 def test_to_frame_via_hdfs_and_erase_cache_dir():
-    with patch("ibis.backends.impala") as impala, patch("shutil.rmtree") as rm, patch(
+    with patch("ibis.impala") as impala, patch("shutil.rmtree") as rm, patch(
         "pyarrow.parquet.read_table"
     ) as pq:
         local_tmp_table_dir = MagicMock()
@@ -184,7 +184,7 @@ def test_to_frame_via_hdfs_and_erase_cache_dir():
 
 
 def test_to_frame_no_hdfs_returns_empty_df():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         q = "example query"
         impala.connect.return_value.raw_sql.return_value.fetchall.return_value = None
@@ -197,7 +197,7 @@ def test_to_frame_no_hdfs_returns_empty_df():
 
 
 def test_to_frame_no_hdfs_returns_non_empty_df():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         q = "example query"
         values = [[1, 2], [3, 4]]
@@ -214,7 +214,7 @@ def test_to_frame_no_hdfs_returns_non_empty_df():
 
 
 def test_to_frame_no_hdfs_fails_and_closes_connection():
-    with patch("ibis.backends.impala") as impala:
+    with patch("ibis.impala") as impala:
         ibis_cli = IbisClient("host")
         q = "example query"
         impala.connect.return_value.raw_sql.side_effect = ValueError("test error")