(WIP) dialect compliance: run compliance tests

Author: Valeriya Popova

setup.cfg

@@ -1,7 +1,7 @@
 [tool:pytest]
-addopts= --tb native -v -r fxX --maxfail=25 -p no:warnings
+addopts= --tb native -v -r fxX -p no:warnings
 testpaths =
-    tests
+    test
     ydb_sqlalchemy
 
 [sqla_testing]

tests/__init__.py test/__init__.py

@@ -8,7 +8,7 @@
 registry.register("yql.ydb", "ydb_sqlalchemy.sqlalchemy", "YqlDialect")
 pytest.register_assert_rewrite("sqlalchemy.testing.assertions")
 
-# from sqlalchemy.testing.plugin.pytestplugin import *
+from sqlalchemy.testing.plugin.pytestplugin import *
 
 
 def wait_container_ready(driver):

tests/conftest.py test/conftest.py

@@ -151,6 +151,7 @@ def test_sa_types(connection):
         "test_types",
         sa.MetaData(),
         Column("id", Integer, primary_key=True),
+        # Column("bin", sa.BINARY),
         Column("str", sa.String),
         Column("num", sa.Float),
         Column("bl", sa.Boolean),
@@ -163,7 +164,8 @@ def test_sa_types(connection):
 
     stm = types_tb.insert().values(
         id=1,
-        str=b"Hello World!",
+        # bin=b"abc",
+        str="Hello World!",
         num=3.1415,
         bl=True,
         dt=datetime.now(),

tests/test_core.py test/test_core.py

@@ -8,9 +8,9 @@ def test_get_columns(engine, test_table):
         c["type"] = type(c["type"])
 
     assert columns == [
-        {"name": "id", "type": sa.INTEGER, "nullable": False},
-        {"name": "value", "type": sa.TEXT, "nullable": True},
-        {"name": "num", "type": sa.DECIMAL, "nullable": True},
+        {"name": "id", "type": sa.INTEGER, "nullable": False, "default": None},
+        {"name": "value", "type": sa.TEXT, "nullable": True, "default": None},
+        {"name": "num", "type": sa.DECIMAL, "nullable": True, "default": None},
     ]
 
 

tests/test_dbapi.py test/test_dbapi.py

@@ -0,0 +1,261 @@
+import pytest
+import sqlalchemy as sa
+import sqlalchemy.testing.suite.test_types
+from sqlalchemy.testing.suite import *
+
+from sqlalchemy.testing.suite.test_select import CompoundSelectTest as _CompoundSelectTest
+from sqlalchemy.testing.suite.test_reflection import (
+    HasTableTest as _HasTableTest,
+    HasIndexTest as _HasIndexTest,
+    ComponentReflectionTest as _ComponentReflectionTest,
+    CompositeKeyReflectionTest as _CompositeKeyReflectionTest,
+    ComponentReflectionTestExtra as _ComponentReflectionTestExtra,
+    QuotedNameArgumentTest as _QuotedNameArgumentTest,
+)
+from sqlalchemy.testing.suite.test_types import (
+    IntegerTest as _IntegerTest,
+    TrueDivTest as _TrueDivTest,
+    TimeTest as _TimeTest,
+    TimeMicrosecondsTest as _TimeMicrosecondsTest,
+    DateTimeCoercedToDateTimeTest as _DateTimeCoercedToDateTimeTest,
+)
+from sqlalchemy.testing.suite.test_dialect import DifficultParametersTest as _DifficultParametersTest
+from sqlalchemy.testing.suite.test_select import JoinTest as _JoinTest
+
+
+test_types_suite = sqlalchemy.testing.suite.test_types
+col_creator = test_types_suite.Column
+
+
+def column_getter(*args, **kwargs):
+    col = col_creator(*args, **kwargs)
+    if col.name == "x":
+        col.primary_key = True
+    return col
+
+
+test_types_suite.Column = column_getter
+
+
+class ComponentReflectionTest(_ComponentReflectionTest):
+    @property
+    def _required_column_keys(self):
+        # nullable had changed so don't check it.
+        return {"name", "type", "default"}
+
+    def _check_list(self, result, exp, req_keys=None, msg=None):
+        try:
+            return super()._check_list(result, exp, req_keys, msg)
+        except AssertionError as err:
+            if "nullable" in err.args[0]:
+                return "We changed nullable in define_reflected_tables method so won't check it."
+            raise
+
+    @classmethod
+    def define_reflected_tables(cls, metadata, schema):
+        Table(
+            "users",
+            metadata,
+            Column("user_id", sa.INT, primary_key=True),
+            Column("test1", sa.CHAR(5)),
+            Column("test2", sa.Float()),
+            Column("parent_user_id", sa.Integer),
+            schema=schema,
+            test_needs_fk=True,
+        )
+
+        Table(
+            "dingalings",
+            metadata,
+            Column("dingaling_id", sa.Integer, primary_key=True),
+            Column("address_id", sa.Integer),
+            Column("id_user", sa.Integer),
+            Column("data", sa.String(30)),
+            schema=schema,
+            test_needs_fk=True,
+        )
+
+        Table(
+            "email_addresses",
+            metadata,
+            Column("address_id", sa.Integer, primary_key=True),
+            Column("remote_user_id", sa.Integer),
+            Column("email_address", sa.String(20)),
+            schema=schema,
+            test_needs_fk=True,
+        )
+
+        Table(
+            "comment_test",
+            metadata,
+            Column("id", sa.Integer, primary_key=True, comment="id comment"),
+            Column("data", sa.String(20), comment="data % comment"),
+            Column("d2", sa.String(20), comment=r"""Comment types type speedily ' " \ '' Fun!"""),
+            schema=schema,
+            comment=r"""the test % ' " \ table comment""",
+        )
+
+        Table(
+            "no_constraints",
+            metadata,
+            Column("data", sa.String(20), primary_key=True, nullable=True),
+            schema=schema,
+        )
+
+    @pytest.mark.skip("views unsupported")
+    def test_get_view_names(self, connection, use_schema):
+        pass
+
+
+class CompositeKeyReflectionTest(_CompositeKeyReflectionTest):
+    @classmethod
+    def define_tables(cls, metadata):
+        Table(
+            "tb1",
+            metadata,
+            Column("id", Integer),
+            Column("attr", Integer),
+            Column("name", sa.VARCHAR(20)),
+            # named pk unsupported
+            sa.PrimaryKeyConstraint("name", "id", "attr"),
+            schema=None,
+            test_needs_fk=True,
+        )
+
+    @pytest.mark.skip("TODO: pk key reflection unsupported")
+    def test_pk_column_order(self, connection):
+        pass
+
+
+class ComponentReflectionTestExtra(_ComponentReflectionTestExtra):
+    def _type_round_trip(self, connection, metadata, *types):
+        t = Table(
+            "t",
+            metadata,
+            # table without pk unsupported
+            *[Column("t%d" % i, type_, primary_key=True) for i, type_ in enumerate(types)],
+        )
+        t.create(connection)
+        return [c["type"] for c in inspect(connection).get_columns("t")]
+
+    @pytest.mark.skip("TODO: numeric now int64??")
+    def test_numeric_reflection(self):
+        pass
+
+    @pytest.mark.skip("TODO: varchar with length unsupported")
+    def test_varchar_reflection(self):
+        pass
+
+    @testing.requires.table_reflection
+    def test_nullable_reflection(self, connection, metadata):
+        t = Table(
+            "t",
+            metadata,
+            # table without pk unsupported
+            Column("a", Integer, nullable=True, primary_key=True),
+            Column("b", Integer, nullable=False, primary_key=True),
+        )
+        t.create(connection)
+        eq_(
+            {
+                col["name"]: col["nullable"]
+                for col in inspect(connection).get_columns("t")
+            },
+            {"a": True, "b": False},
+        )
+
+
+class HasTableTest(_HasTableTest):
+    @classmethod
+    def define_tables(cls, metadata):
+        Table(
+            "test_table",
+            metadata,
+            Column("id", Integer, primary_key=True),
+            Column("data", String(50)),
+        )
+
+    @pytest.mark.skip("TODO: reflection cache unsupported")
+    def test_has_table_cache(self, metadata):
+        pass
+
+
+@pytest.mark.skip("CREATE INDEX syntax unsupported")
+class HasIndexTest(_HasIndexTest):
+    pass
+
+
+@pytest.mark.skip("quotes unsupported in table names")
+class QuotedNameArgumentTest(_QuotedNameArgumentTest):
+    pass
+
+
+class IntegerTest(_IntegerTest):
+    @pytest.mark.skip("YQL doesn't support select with where without from")
+    def test_huge_int_auto_accommodation(self, connection, intvalue):
+        pass
+
+
+class TrueDivTest(_TrueDivTest):
+    @pytest.mark.skip("Unsupported builtin: FLOOR")
+    def test_floordiv_numeric(self, connection, left, right, expected):
+        pass
+
+    @pytest.mark.skip("Truediv unsupported for int")
+    def test_truediv_integer(self, connection, left, right, expected):
+        pass
+
+    @pytest.mark.skip("Truediv unsupported for int")
+    def test_truediv_integer_bound(self, connection):
+        pass
+
+    @pytest.mark.skip("Numeric is not Decimal")
+    def test_truediv_numeric(self):
+        pass
+
+
+class CompoundSelectTest(_CompoundSelectTest):
+    @pytest.mark.skip("limit don't work")
+    def test_distinct_selectable_in_unions(self):
+        pass
+
+    @pytest.mark.skip("limit don't work")
+    def test_limit_offset_in_unions_from_alias(self):
+        pass
+
+    @pytest.mark.skip("limit don't work")
+    def test_limit_offset_aliased_selectable_in_unions(self):
+        pass
+
+    @pytest.mark.skip("union with brackets don't work")
+    def test_order_by_selectable_in_unions(self):
+        pass
+
+    @pytest.mark.skip("union with brackets don't work")
+    def test_limit_offset_selectable_in_unions(self):
+        pass
+
+
+@pytest.mark.skip("unsupported tricky names for columns")
+class DifficultParametersTest(_DifficultParametersTest):
+    pass
+
+
+@pytest.mark.skip("JOIN ON expression must be a conjunction of equality predicates")
+class JoinTest(_JoinTest):
+    pass
+
+
+@pytest.mark.skip("unsupported Time data type")
+class TimeTest(_TimeTest):
+    pass
+
+
+@pytest.mark.skip("unsupported Time data type")
+class TimeMicrosecondsTest(_TimeMicrosecondsTest):
+    pass
+
+
+@pytest.mark.skip("unsupported coerce dates from datetime")
+class DateTimeCoercedToDateTimeTest(_DateTimeCoercedToDateTimeTest):
+    pass

tests/test_inspect.py test/test_inspect.py

@@ -14,11 +14,11 @@ deps =
 
 [testenv:py]
 commands =
-    pytest -v -m "not tls" --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
+    pytest -v --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
 
 [testenv:py-cov]
 commands =
-    pytest -v -m "not tls" \
+    pytest -v \
         --cov-report html:cov_html --cov=ydb_sqlalchemy \
         --docker-compose-remove-volumes --docker-compose=docker-compose.yml {posargs}
 

test/test_suite.py

@@ -7,9 +7,10 @@
 
 import sqlalchemy as sa
 from sqlalchemy import Table
-from sqlalchemy.exc import CompileError
+from sqlalchemy.exc import CompileError, NoSuchTableError
 from sqlalchemy.sql import functions, literal_column
 from sqlalchemy.sql.compiler import (
+    selectable,
     IdentifierPreparer,
     StrSQLTypeCompiler,
     StrSQLCompiler,
@@ -24,6 +25,16 @@
 from .types import UInt32, UInt64
 
 
+COMPOUND_KEYWORDS = {
+    selectable.CompoundSelect.UNION: "UNION ALL",
+    selectable.CompoundSelect.UNION_ALL: "UNION ALL",
+    selectable.CompoundSelect.EXCEPT: "EXCEPT",
+    selectable.CompoundSelect.EXCEPT_ALL: "EXCEPT ALL",
+    selectable.CompoundSelect.INTERSECT: "INTERSECT",
+    selectable.CompoundSelect.INTERSECT_ALL: "INTERSECT ALL",
+}
+
+
 class YqlIdentifierPreparer(IdentifierPreparer):
     def __init__(self, dialect):
         super(YqlIdentifierPreparer, self).__init__(
@@ -38,8 +49,11 @@ def _requires_quotes(self, value):
 
 
 class YqlTypeCompiler(StrSQLTypeCompiler):
+    def visit_CHAR(self, type_, **kw):
+        return "UTF8"
+
     def visit_VARCHAR(self, type_, **kw):
-        return "STRING"
+        return "UTF8"
 
     def visit_unicode(self, type_, **kw):
         return "UTF8"
@@ -71,6 +85,15 @@ def visit_INTEGER(self, type_, **kw):
     def visit_NUMERIC(self, type_, **kw):
         return "Int64"
 
+    def visit_BINARY(self, type_, **kw):
+        return "String"
+
+    def visit_BLOB(self, type_, **kw):
+        return "String"
+
+    def visit_DATETIME(self, type_, **kw):
+        return "Timestamp"
+
 
 class ParametrizedFunction(functions.Function):
     __visit_name__ = "parametrized_function"
@@ -83,6 +106,8 @@ def __init__(self, name, params, *args, **kwargs):
 
 
 class YqlCompiler(StrSQLCompiler):
+    compound_keywords = COMPOUND_KEYWORDS
+
     def render_bind_cast(self, type_, dbapi_type, sqltext):
         pass
 
@@ -178,7 +203,7 @@ def upsert(table):
     ydb.PrimitiveType.Uint64: UInt64,
     ydb.PrimitiveType.Float: sa.FLOAT,
     ydb.PrimitiveType.Double: sa.FLOAT,
-    ydb.PrimitiveType.String: sa.TEXT,
+    ydb.PrimitiveType.String: sa.BINARY,
     ydb.PrimitiveType.Utf8: sa.TEXT,
     ydb.PrimitiveType.Json: sa.JSON,
     ydb.PrimitiveType.JsonDocument: sa.JSON,
@@ -216,9 +241,15 @@ class YqlDialect(StrCompileDialect):
     supports_native_boolean = True
     supports_native_decimal = True
     supports_smallserial = False
+    supports_schemas = False
+    supports_constraint_comments = False
+
+    insert_returning = False
+    update_returning = False
+    delete_returning = False
 
     supports_sequences = False
-    sequences_optional = True
+    sequences_optional = False
     preexecute_autoincrement_sequences = True
     postfetch_lastrowid = False
 
@@ -242,12 +273,15 @@ def import_dbapi(cls: Any):
 
     def get_columns(self, connection, table_name, schema=None, **kw):
         if schema is not None:
-            raise dbapi.errors.NotSupportedError("unsupported on non empty schema")
-
-        qt = table_name.name if isinstance(table_name, Table) else table_name
+            raise dbapi.NotSupportedError("unsupported on non empty schema")
 
+        qt = table_name if isinstance(table_name, str) else table_name.name
         raw_conn = connection.connection
-        columns = raw_conn.describe(qt)
+        try:
+            columns = raw_conn.describe(qt)
+        except dbapi.DatabaseError as e:
+            raise NoSuchTableError(qt) from e
+
         as_compatible = []
         for column in columns:
             col_type, nullable = _get_column_info(column.type)
@@ -256,24 +290,31 @@ def get_columns(self, connection, table_name, schema=None, **kw):
                     "name": column.name,
                     "type": col_type,
                     "nullable": nullable,
+                    "default": None,
                 }
             )
 
         return as_compatible
 
-    def has_table(self, connection, table_name, schema=None, **kwargs):
+    def get_table_names(self, connection, schema=None, **kw):
         if schema:
-            raise dbapi.errors.NotSupportedError("unsupported on non empty schema")
+            raise dbapi.NotSupportedError("unsupported on non empty schema")
 
-        quote = self.identifier_preparer.quote_identifier
-        qtable = quote(table_name)
+        driver = connection.connection.driver_connection.driver
+        db_path = driver._driver_config.database
+        children = driver.scheme_client.list_directory(db_path).children
 
-        # TODO: use `get_columns` instead.
-        statement = "SELECT * FROM " + qtable
+        return [child.name for child in children if child.is_table()]
+
+    def has_table(self, connection, table_name, schema=None, **kwargs):
+        if schema:
+            raise dbapi.NotSupportedError("unsupported on non empty schema")
+
+        raw_conn = connection.connection
         try:
-            connection.execute(sa.text(statement))
+            raw_conn.describe(table_name)
             return True
-        except Exception:
+        except dbapi.DatabaseError:
             return False
 
     def get_pk_constraint(self, connection, table_name, schema=None, **kwargs):

tox.ini

@@ -10,3 +10,52 @@ def array_type(self):
     @property
     def uuid_data_type(self):
         return exclusions.open()
+
+    @property
+    def nullable_booleans(self):
+        return exclusions.closed()
+
+    @property
+    def foreign_keys(self):
+        # foreign keys unsupported
+        return exclusions.closed()
+
+    @property
+    def self_referential_foreign_keys(self):
+        return exclusions.closed()
+
+    @property
+    def foreign_key_ddl(self):
+        return exclusions.closed()
+
+    @property
+    def foreign_key_constraint_reflection(self):
+        return exclusions.closed()
+
+    @property
+    def temp_table_reflection(self):
+        return exclusions.closed()
+
+    @property
+    def temporary_tables(self):
+        return exclusions.closed()
+
+    @property
+    def temporary_views(self):
+        return exclusions.closed()
+
+    @property
+    def index_reflection(self):
+        return exclusions.closed()
+
+    @property
+    def view_reflection(self):
+        return exclusions.closed()
+
+    @property
+    def unique_constraint_reflection(self):
+        return exclusions.closed()
+
+    @property
+    def insert_returning(self):
+        return exclusions.closed()