Skip to content

⚡ Optimize database table existence checks to avoid N+1 schema queries #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zrt219 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

⚡ Optimize database table existence checks to avoid N+1 schema queries #16

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion openmed/openmed/interop/omop/cdm_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -846,10 +846,15 @@ def validate_omop_tables(
def validate_omop_database(con: Any) -> tuple[OmopConstraintViolation, ...]:
"""Validate persisted OMOP tables using the same PHI-free violation shape."""

existing_tables = _database_tables(con)
tables = {
table: tuple(_select_all(con, table))
for table in _TABLE_ORDER
if _table_exists(con, table)
if (
table in existing_tables
if existing_tables is not None
else _table_exists(con, table)
)
}
return validate_omop_tables(
OmopCdmTables(
Expand Down Expand Up @@ -1181,10 +1186,15 @@ def _ordered_row(table: str, row: Mapping[str, Any]) -> dict[str, Any]:


def _upsert_sql_tables(con: Any, tables: OmopCdmTables) -> None:
existing_tables = _database_tables(con)
for table in _TABLE_ORDER:
rows = tables.table(table)
if not rows:
continue
if existing_tables is not None and table not in existing_tables:
continue
if existing_tables is None and not _table_exists(con, table):
continue
columns = _SCHEMA_COLUMNS[table]
column_sql = ", ".join(_quote_identifier(column) for column in columns)
placeholder_sql = ", ".join("?" for _ in columns)
Expand Down Expand Up @@ -1237,6 +1247,21 @@ def _select_all(con: Any, table: str) -> list[dict[str, Any]]:
return [dict(zip(columns, row)) for row in rows]


def _database_tables(con: Any) -> frozenset[str] | None:
# Only use fast path for supported in-memory or file-based DBs
# where sqlite_master is safe and does not poison transactions.
dialect = type(con).__module__.split(".")[0]
if dialect in {"sqlite3", "duckdb"}:
try:
rows = con.execute(
"SELECT name FROM sqlite_master WHERE type IN ('table', 'view')"
).fetchall()
return frozenset(r[0] for r in rows)
except Exception:
return None
return None


def _table_exists(con: Any, table: str) -> bool:
try:
con.execute(f"SELECT 1 FROM {table} LIMIT 1")
Expand Down