diff --git a/README.md b/README.md index c623037..e4df325 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ An InterSystems IRIS dialect for SQLAlchemy. Pre-requisites --- -This dialect requires SQLAlchemy, InterSystems DB-API driver. They are specified as requirements so ``pip`` -will install them if they are not already in place. To install, just: +This dialect requires SQLAlchemy, InterSystems DB-API driver, and iris-embedded-python-wrapper. They are +specified as requirements so ``pip`` will install them if they are not already in place. To install, just: ```shell pip install sqlalchemy-iris @@ -29,13 +29,15 @@ from sqlalchemy import create_engine engine = create_engine("iris://_SYSTEM:SYS@localhost:1972/USER") ``` -To use with Python Embedded mode, when run next to IRIS +To use with Python Embedded mode through iris-embedded-python-wrapper, when run next to IRIS ```python from sqlalchemy import create_engine -engine = create_engine("iris+emb:///USER") +engine = create_engine("iris+emb://USER") ``` +The legacy path form `iris+emb:///USER` is also supported. + To use with InterSystems official driver, does not work in Python Embedded mode ```python diff --git a/requirements-iris.txt b/requirements-iris.txt index 4f103c1..88b59b6 100644 --- a/requirements-iris.txt +++ b/requirements-iris.txt @@ -1 +1 @@ -https://github.com/intersystems-community/intersystems-irispython/releases/download/3.9.3/intersystems_iris-3.9.3-py3-none-any.whl \ No newline at end of file +intersystems-irispython~=5.3.2 diff --git a/requirements.txt b/requirements.txt index 2a57409..546ef97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ SQLAlchemy>=1.3 -intersystems-irispython~=5.3.2 \ No newline at end of file +intersystems-irispython~=5.3.2 +iris-embedded-python-wrapper>=0.5.23 diff --git a/setup.cfg b/setup.cfg index 49c72ad..521f662 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sqlalchemy-iris -version = 0.19.2 +version = 0.20.0 description = InterSystems IRIS for SQLAlchemy long_description = file: README.md url = https://github.com/caretdev/sqlalchemy-iris @@ -41,8 +41,7 @@ addopts= --tb native -v -r fxX -p no:warnings default=iris://_SYSTEM:SYS@localhost:1972/USER iris=iris://_SYSTEM:SYS@localhost:1972/USER irisintersystems=iris+intersystems://_SYSTEM:SYS@localhost:1972/USER -# irisasync=iris+irisasync://_SYSTEM:SYS@localhost:1972/USER -# irisemb=iris+emb:/// +irisemb=iris+emb:/// sqlite=sqlite:///:memory: [sqla_testing] diff --git a/setup.py b/setup.py index ff14cc9..733b0fd 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,12 @@ install_requires=[ "SQLAlchemy>=1.3", "intersystems-irispython~=5.3.2", + "iris-embedded-python-wrapper>=0.5.23", ], entry_points={ "sqlalchemy.dialects": [ - # "iris = sqlalchemy_iris.iris:IRISDialect_iris", - # "iris.emb = sqlalchemy_iris.embedded:IRISDialect_emb", - # "iris.irisasync = sqlalchemy_iris.irisasync:IRISDialect_irisasync", "iris = sqlalchemy_iris.intersystems:IRISDialect_intersystems", + "iris.emb = sqlalchemy_iris.embedded:IRISDialect_emb", "iris.intersystems = sqlalchemy_iris.intersystems:IRISDialect_intersystems", ] }, diff --git a/sqlalchemy_iris/__init__.py b/sqlalchemy_iris/__init__.py index c8ffef1..4bae159 100644 --- a/sqlalchemy_iris/__init__.py +++ b/sqlalchemy_iris/__init__.py @@ -30,10 +30,8 @@ base.dialect = dialect = intersystems_dialect -# _registry.register("iris.iris", "sqlalchemy_iris.iris", "IRISDialect_iris") -# _registry.register("iris.emb", "sqlalchemy_iris.embedded", "IRISDialect_emb") -# _registry.register("iris.irisasync", "sqlalchemy_iris.irisasync", "IRISDialect_irisasync") _registry.register("iris.iris", "sqlalchemy_iris.intersystems", "IRISDialect_intersystems") +_registry.register("iris.emb", "sqlalchemy_iris.embedded", "IRISDialect_emb") _registry.register("iris.intersystems", "sqlalchemy_iris.intersystems", "IRISDialect_intersystems") __all__ = [ diff --git a/sqlalchemy_iris/embedded.py b/sqlalchemy_iris/embedded.py index 3e631ae..4323d68 100644 --- a/sqlalchemy_iris/embedded.py +++ b/sqlalchemy_iris/embedded.py @@ -1,6 +1,14 @@ +from sqlalchemy import exc +from sqlalchemy import util + from .base import IRISDialect +def _parse_version_number(server_version): + server_version = str(server_version).split(" ")[0].split(".") + return tuple([int("".join(filter(str.isdigit, v)) or 0) for v in server_version]) + + class IRISDialect_emb(IRISDialect): driver = "emb" @@ -8,22 +16,139 @@ class IRISDialect_emb(IRISDialect): supports_statement_cache = True + insert_returning = False + insert_executemany_returning = False + insert_executemany_returning_sort_by_parameter_order = False + + _isolation_lookup = set( + [ + "READ UNCOMMITTED", + "READ COMMITTED", + "REPEATABLE READ", + "SERIALIZABLE", + ] + ) + def _get_option(self, connection, option): - return connection.iris.cls("%SYSTEM.SQL.Util").GetOption(option) + import iris + + return iris.cls("%SYSTEM.SQL.Util").GetOption(option) def _set_option(self, connection, option, value): - return connection.iris.cls("%SYSTEM.SQL.Util").SetOption(option) + import iris + + return iris.cls("%SYSTEM.SQL.Util").SetOption(option, value) @classmethod def import_dbapi(cls): - import intersystems_iris.dbapi._DBAPI as dbapi + import iris + + return iris.dbapi + + def create_connect_args(self, url): + if url.port or url.username or url.password: + raise exc.ArgumentError( + "iris+emb:// URLs are local-only; use iris:// or iris+intersystems:// " + "for host, port, username, or password connections" + ) + + if url.host and url.database: + raise exc.ArgumentError( + "iris+emb:// URLs accept the namespace as either " + "iris+emb://NAMESPACE or iris+emb:///NAMESPACE, not both" + ) + + supported_query_args = {"path"} + unsupported_query_args = set(url.query).difference(supported_query_args) + if unsupported_query_args: + raise exc.ArgumentError( + "Unsupported iris+emb:// query argument(s): " + + ", ".join(sorted(unsupported_query_args)) + ) + + opts = { + "mode": "embedded", + "namespace": url.host or url.database or "USER", + } + path = url.query.get("path") + if path is not None: + if isinstance(path, tuple): + path = path[-1] + opts["path"] = path - return dbapi + return ([], opts) def _get_server_version_info(self, connection): - server_version = connection._dbapi_connection.iris.system.Version.GetNumber() - server_version = server_version.split(".") - return tuple([int("".join(filter(str.isdigit, v))) for v in server_version]) + import iris + + version_api = getattr(getattr(iris, "system", None), "Version", None) + get_number = getattr(version_api, "GetNumber", None) + if callable(get_number): + return _parse_version_number(get_number()) + + return _parse_version_number(iris.cls("%SYSTEM.Version").GetNumber()) + + def on_connect(self): + def on_connect(conn): + try: + with conn.cursor() as cursor: + cursor.execute( + "select vector_cosine(to_vector('1'), to_vector('1'))" + ) + cursor.execute("select to_vector('1')") + cursor.fetchone() + self.supports_vectors = True + except Exception: + self.supports_vectors = False + + try: + with conn.cursor() as cursor: + cursor.execute("SELECT TOP 1 Name FROM %Dictionary.PropertyDefinition") + cursor.fetchone() + self._dictionary_access = True + except Exception: + self._dictionary_access = False + + if not self._dictionary_access: + util.warn( + """ +There are no access to %Dictionary, may be required for some advanced features, + such as Calculated fields, and include columns in indexes + """.replace( + "\n", "" + ) + ) + + return on_connect + + def get_isolation_level(self, connection): + if getattr(connection, "autocommit", False): + return "AUTOCOMMIT" + + isolation_level = getattr(connection, "isolation_level", None) + if isolation_level: + return isolation_level.upper() + + return "READ COMMITTED" + + def set_isolation_level(self, connection, level_str): + if level_str == "AUTOCOMMIT": + connection.autocommit = True + else: + connection.autocommit = False + connection.isolation_level = level_str + + def do_execute(self, cursor, query, params, context=None): + if query.endswith(";"): + query = query[:-1] + self._debug(query, params) + cursor.execute(query, params) + + def do_executemany(self, cursor, query, params, context=None): + if query.endswith(";"): + query = query[:-1] + self._debug(query, params, True) + cursor.executemany(query, params) dialect = IRISDialect_emb diff --git a/sqlalchemy_iris/intersystems/__init__.py b/sqlalchemy_iris/intersystems/__init__.py index b5abb5a..560e8dd 100644 --- a/sqlalchemy_iris/intersystems/__init__.py +++ b/sqlalchemy_iris/intersystems/__init__.py @@ -4,7 +4,6 @@ from ..base import IRISExecutionContext from . import dbapi from .dbapi import connect -from .dbapi import IntegrityError, OperationalError, DatabaseError from sqlalchemy.engine.cursor import CursorFetchStrategy @@ -17,6 +16,7 @@ def wrapper(cursor, *args, **kwargs): cursor.sqlcode = 0 return func(cursor, *args, **kwargs) except RuntimeError as ex: + dbapi._sync_exception_classes() # [SQLCODE: <-119>:... message = ex.args[0] if "" in message: @@ -27,10 +27,13 @@ def wrapper(cursor, *args, **kwargs): raise Exception(message) sqlcode = int(sqlcode[0]) if abs(sqlcode) in [108, 119, 121, 122]: - raise IntegrityError(sqlcode, message) + raise dbapi.IntegrityError(sqlcode, message) if abs(sqlcode) in [1, 12]: - raise OperationalError(sqlcode, message) - raise DatabaseError(sqlcode, message) + raise dbapi.OperationalError(sqlcode, message) + raise dbapi.DatabaseError(sqlcode, message) + except Exception: + dbapi._sync_exception_classes() + raise return wrapper @@ -153,7 +156,6 @@ def set_isolation_level(self, connection, level_str): with connection.cursor() as cursor: cursor.execute("SET TRANSACTION ISOLATION LEVEL " + level_str) -""" @remap_exception def do_execute(self, cursor, query, params, context=None): if query.endswith(";"): @@ -170,6 +172,5 @@ def do_executemany(self, cursor, query, params, context=None): params = [param[0] if len(param) else None for param in params] cursor.executemany(query, params) -""" dialect = IRISDialect_intersystems diff --git a/sqlalchemy_iris/intersystems/dbapi.py b/sqlalchemy_iris/intersystems/dbapi.py index 59d556a..25b529b 100644 --- a/sqlalchemy_iris/intersystems/dbapi.py +++ b/sqlalchemy_iris/intersystems/dbapi.py @@ -7,12 +7,16 @@ class Cursor(iris.irissdk.dbapiCursor): class DataRow(iris.irissdk.dbapiDataRow): pass -except ImportError: - pass +except (AttributeError, ImportError, TypeError): + iris = None def connect(*args, **kwargs): - return iris.connect(*args, **kwargs) + _sync_exception_classes() + try: + return iris.connect(*args, **kwargs) + finally: + _sync_exception_classes() def createIRIS(*args, **kwargs): @@ -30,7 +34,6 @@ def createIRIS(*args, **kwargs): NUMBER = float ROWID = str - class Error(Exception): pass @@ -69,3 +72,30 @@ class DataError(DatabaseError): class NotSupportedError(DatabaseError): pass + + +_EXCEPTION_NAMES = ( + "Error", + "Warning", + "InterfaceError", + "DatabaseError", + "InternalError", + "OperationalError", + "ProgrammingError", + "IntegrityError", + "DataError", + "NotSupportedError", +) + + +def _sync_exception_classes(): + if iris is None or not hasattr(iris, "dbapi"): + return + + for name in _EXCEPTION_NAMES: + cls = getattr(iris.dbapi, name, None) + if cls is not None: + globals()[name] = cls + + +_sync_exception_classes() diff --git a/sqlalchemy_iris/irisasync.py b/sqlalchemy_iris/irisasync.py deleted file mode 100644 index 0ded9e4..0000000 --- a/sqlalchemy_iris/irisasync.py +++ /dev/null @@ -1,17 +0,0 @@ -from .base import IRISDialect - - -class IRISDialect_irisasync(IRISDialect): - driver = "irisasync" - - is_async = True - supports_statement_cache = True - - @classmethod - def import_dbapi(cls): - import intersystems_iris.dbapi._DBAPI as dbapi - - return dbapi - - -dialect = IRISDialect_irisasync diff --git a/sqlalchemy_iris/requirements.py b/sqlalchemy_iris/requirements.py index 324e23d..2ed7942 100644 --- a/sqlalchemy_iris/requirements.py +++ b/sqlalchemy_iris/requirements.py @@ -243,7 +243,11 @@ def autocommit(self): def get_isolation_levels(self, config): levels = set(config.db.dialect._isolation_lookup) - default = "READ UNCOMMITTED" + default = ( + "READ COMMITTED" + if config.db.dialect.driver == "emb" + else "READ UNCOMMITTED" + ) levels.add("AUTOCOMMIT") return {"default": default, "supported": levels} @@ -944,7 +948,10 @@ def unicode_ddl(self): """Target driver must support some degree of non-ascii symbol names. """ - return exclusions.open() + return exclusions.skip_if( + lambda config: getattr(config.db.dialect, "embedded", False), + "embedded DBAPI crashes on non-ASCII result column names", + ) @property def datetime_interval(self): @@ -960,8 +967,13 @@ def datetime_literals(self): literal string, e.g. via the TypeEngine.literal_processor() method. """ - # works stable only on Community driver - return self.community_driver + return exclusions.only_if( + lambda config: ( + config.db.dialect.driver != "intersystems" + and not getattr(config.db.dialect, "embedded", False) + ), + "datetime literal rendering is not stable on this driver", + ) @property def datetime(self): @@ -1153,7 +1165,10 @@ def implicit_decimal_binds(self): a string. """ - return exclusions.open() + return exclusions.skip_if( + lambda config: getattr(config.db.dialect, "embedded", False), + "embedded DBAPI returns untyped Decimal binds as strings", + ) @property def numeric_received_as_decimal_untyped(self): diff --git a/sqlalchemy_iris/types.py b/sqlalchemy_iris/types.py index 0245ef8..5435950 100644 --- a/sqlalchemy_iris/types.py +++ b/sqlalchemy_iris/types.py @@ -6,14 +6,33 @@ from uuid import UUID as _python_UUID from sqlalchemy import __version__ as sqlalchemy_version -try: - from intersystems_iris import IRISList -except ImportError: - pass - HOROLOG_ORDINAL = datetime.date(1840, 12, 31).toordinal() +def _decode_uuid_value(value): + if isinstance(value, bytes): + return value.decode() + return value + + +def _get_iris_list_class(): + try: + from intersystems_iris import IRISList + + return IRISList + except ImportError: + pass + + try: + from iris import IRISList + + return IRISList + except ImportError: + pass + + raise ImportError("IRISList is not available in this Python runtime") + + class IRISBoolean(sqltypes.Boolean): def _should_create_constraint(self, compiler, **kw): return False @@ -63,6 +82,8 @@ def process(value): def literal_processor(self, dialect): def process(value): if isinstance(value, datetime.date): + if getattr(dialect, "embedded", False): + return str(value.toordinal() - HOROLOG_ORDINAL) return "'%s'" % value.strftime("%Y-%m-%d") return value @@ -100,6 +121,8 @@ def literal_processor(self, dialect): def process(value): if isinstance(value, datetime.datetime): return "'%s'" % value.strftime("%Y-%m-%d %H:%M:%S.%f") + if isinstance(value, datetime.date): + return "'%s 00:00:00.000000'" % value.strftime("%Y-%m-%d") return value return process @@ -124,6 +147,10 @@ def process(value): if "." not in value: value += ".0" return datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S.%f") + if isinstance(value, int): + value -= (2**60) if value > 0 else -(2**61 * 3) + value = value / 1000000 + return datetime.datetime.utcfromtimestamp(value) return value return process @@ -132,6 +159,8 @@ def literal_processor(self, dialect): def process(value): if isinstance(value, datetime.datetime): return "'%s'" % value.strftime("%Y-%m-%d %H:%M:%S.%f") + if isinstance(value, datetime.date): + return "'%s 00:00:00.000000'" % value.strftime("%Y-%m-%d") return value return process @@ -140,9 +169,18 @@ def process(value): class IRISTime(sqltypes.DateTime): __visit_name__ = "TIME" + @staticmethod + def _to_horolog(value): + result = value.hour * 3600 + value.minute * 60 + value.second + if value.microsecond: + result += value.microsecond / 1000000 + return result + def bind_processor(self, dialect): def process(value): if value is not None: + if getattr(dialect, "embedded", False): + return self._to_horolog(value) return value.strftime("%H:%M:%S.%f") return value @@ -156,13 +194,13 @@ def process(value): if "." not in value: value += ".0" return datetime.datetime.strptime(value, "%H:%M:%S.%f").time() - if isinstance(value, int) or isinstance(value, Decimal): + if isinstance(value, int) or isinstance(value, float) or isinstance(value, Decimal): horolog = value hour = int(horolog // 3600) horolog -= int(hour * 3600) minute = int(horolog // 60) second = int(horolog % 60) - micro = int(value % 1 * 1000000) + micro = round(value % 1 * 1000000) return datetime.time(hour, minute, second, micro) return value @@ -171,6 +209,8 @@ def process(value): def literal_processor(self, dialect): def process(value): if isinstance(value, datetime.time): + if getattr(dialect, "embedded", False): + return str(self._to_horolog(value)) return "'%s'" % value.strftime("%H:%M:%S.%f") return value @@ -226,6 +266,7 @@ def result_processor(self, dialect, coltype): if self.as_uuid: def process(value): + value = _decode_uuid_value(value) if value and not isinstance(value, _python_UUID): value = _python_UUID(value) return value @@ -234,6 +275,7 @@ def process(value): else: def process(value): + value = _decode_uuid_value(value) if value and isinstance(value, _python_UUID): value = str(value) return value @@ -243,6 +285,7 @@ def process(value): if not self.as_uuid: def process(value): + value = _decode_uuid_value(value) if value and isinstance(value, _python_UUID): value = str(value) return value @@ -272,6 +315,7 @@ def get_col_spec(self, **kw): def bind_processor(self, dialect): def process(value): + IRISList = _get_iris_list_class() irislist = IRISList() if not value: return value @@ -286,8 +330,12 @@ def process(value): def result_processor(self, dialect, coltype): def process(value): if value: + IRISList = _get_iris_list_class() irislist = IRISList(value) - return irislist._list_data + list_data = getattr(irislist, "_list_data", None) + if list_data is not None: + return list_data + return [irislist.get(index) for index in range(1, irislist.count() + 1)] return value return process @@ -296,6 +344,7 @@ class comparator_factory(UserDefinedType.Comparator): def func(self, funcname: str, other): if not isinstance(other, list) and not isinstance(other, tuple): raise ValueError("expected list or tuple, got '%s'" % type(other)) + IRISList = _get_iris_list_class() irislist = IRISList() for item in other: irislist.add(item) @@ -336,6 +385,9 @@ def process(value): return process + def bind_expression(self, bindvalue): + return func.to_vector(bindvalue, text(self.item_type_server)) + def result_processor(self, dialect, coltype): def process(value): if not value: diff --git a/tests/conftest.py b/tests/conftest.py index 0444130..850d1ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,6 @@ registry.register("iris.iris", "sqlalchemy_iris.iris", "IRISDialect_iris") registry.register("iris.emb", "sqlalchemy_iris.embedded", "IRISDialect_emb") -registry.register( - "iris.irisasync", "sqlalchemy_iris.irisasync", "IRISDialect_irisasync" -) registry.register( "iris.intersystems", "sqlalchemy_iris.intersystems", "IRISDialect_intersystems" ) diff --git a/tests/test_embedded.py b/tests/test_embedded.py new file mode 100644 index 0000000..f64ecf1 --- /dev/null +++ b/tests/test_embedded.py @@ -0,0 +1,51 @@ +import pytest + +from sqlalchemy import exc +from sqlalchemy.engine import make_url +from sqlalchemy.testing import fixtures + +from sqlalchemy_iris.embedded import IRISDialect_emb + + +def _connect_opts(url): + args, opts = IRISDialect_emb().create_connect_args(make_url(url)) + assert args == [] + return opts + + +class EmbeddedURLTest(fixtures.TestBase): + @pytest.mark.parametrize( + ("url", "namespace"), + [ + ("iris+emb://", "USER"), + ("iris+emb:///", "USER"), + ("iris+emb://SAMPLES", "SAMPLES"), + ("iris+emb:///SAMPLES", "SAMPLES"), + ], + ) + def test_embedded_url_namespace(self, url, namespace): + opts = _connect_opts(url) + + assert opts["mode"] == "embedded" + assert opts["namespace"] == namespace + + def test_embedded_url_namespace_with_path_option(self): + opts = _connect_opts("iris+emb://SAMPLES?path=/opt/iris") + + assert opts["namespace"] == "SAMPLES" + assert opts["path"] == "/opt/iris" + + @pytest.mark.parametrize( + "url", + [ + "iris+emb://localhost:1972/USER", + "iris+emb://user:pass@USER", + ], + ) + def test_embedded_url_rejects_remote_connection_parts(self, url): + with pytest.raises(exc.ArgumentError, match="local-only"): + _connect_opts(url) + + def test_embedded_url_rejects_duplicate_namespace(self): + with pytest.raises(exc.ArgumentError, match="not both"): + _connect_opts("iris+emb://SAMPLES/USER")