diff --git a/backend/backend/application/session/connection_session.py b/backend/backend/application/session/connection_session.py index 991a8391..13a334e5 100644 --- a/backend/backend/application/session/connection_session.py +++ b/backend/backend/application/session/connection_session.py @@ -1,8 +1,11 @@ +import logging from typing import Any from visitran.utils import import_file from backend.application.utils import get_filter + +logger = logging.getLogger(__name__) from backend.core.models.connection_models import ConnectionDetails from backend.core.models.environment_models import EnvironmentModels from backend.core.models.project_details import ProjectDetails @@ -14,6 +17,32 @@ from backend.utils.pagination import CustomPaginator +def _get_host_display(con_model): + """Extract a human-readable host string from connection details. + + Only reads non-sensitive plaintext fields (host, port, account, etc.) + so no Fernet decryption is needed. + """ + try: + details = con_model.connection_details or {} + ds = con_model.datasource_name + if ds in ("postgres", "mysql", "trino"): + host = details.get("host", "") + port = details.get("port", "") + return f"{host}:{port}" if host and port else host or None + if ds == "snowflake": + return details.get("account") or None + if ds == "bigquery": + return details.get("project_id") or None + if ds == "databricks": + return details.get("host") or None + if ds == "duckdb": + return details.get("file_path") or None + except Exception: + logger.warning("Failed to derive host display for connection %s", con_model.connection_id, exc_info=True) + return None + + class ConnectionSession: @staticmethod @@ -43,35 +72,53 @@ def create_connection(connection_details: dict[str, Any]) -> ConnectionDetails: @staticmethod def get_all_connections(page: int, limit: int, filter_condition: dict[str, Any]) -> Any: + from django.db.models import Count, Q, Exists, OuterRef + filter_condition.update(get_filter()) if "is_deleted" not in filter_condition: filter_condition["is_deleted"] = False - con_models = ConnectionDetails.objects.filter(**filter_condition).order_by("-modified_at") - custom_paginator = CustomPaginator(queryset=con_models, limit=limit, page=page) + # Annotate counts + sample flag in a single query (no N+1) + con_qs = ( + ConnectionDetails.objects.filter(**filter_condition) + .annotate( + env_count=Count( + "environment_model", + filter=Q(environment_model__is_deleted=False), + distinct=True, + ), + project_count=Count("project", distinct=True), + is_sample=Exists( + ProjectDetails.objects.filter( + connection_model_id=OuterRef("connection_id"), + is_sample=True, + ) + ), + ) + .order_by("-modified_at") + ) + + custom_paginator = CustomPaginator(queryset=con_qs, limit=limit, page=page) con_models = custom_paginator.paginate() connection_list = [] for con_model in con_models.get("page_items"): - project_con = ProjectDetails.objects.filter(connection_model_id=con_model.connection_id).first() - if project_con: - is_sample_project = project_con.is_sample - else: - is_sample_project = False connection_list.append( { "id": con_model.connection_id, "name": con_model.connection_name, "description": con_model.connection_description, "datasource_name": con_model.datasource_name, + "host": _get_host_display(con_model), "created_by": con_model.created_by, "last_modified_by": con_model.last_modified_by, "db_icon": import_file(f"visitran.adapters.{con_model.datasource_name}").ICON, "is_connection_exist": con_model.is_connection_exist, "is_connection_valid": con_model.is_connection_valid, "connection_flag": con_model.connection_flag, - "is_sample_project": is_sample_project, - # "connection_details": con_model.connection_details, # skipping connection_details + "is_sample_project": con_model.is_sample, + "env_count": con_model.env_count, + "project_count": con_model.project_count, } ) diff --git a/backend/backend/application/session/env_session.py b/backend/backend/application/session/env_session.py index 08bbfb6a..75b744a6 100644 --- a/backend/backend/application/session/env_session.py +++ b/backend/backend/application/session/env_session.py @@ -2,7 +2,7 @@ from visitran.utils import import_file -from backend.application.session.connection_session import ConnectionSession +from backend.application.session.connection_session import ConnectionSession, _get_host_display from backend.application.utils import get_filter from backend.core.models.environment_models import EnvironmentModels from backend.core.models.project_details import ProjectDetails @@ -59,6 +59,7 @@ def create_environment(self, environment_details: dict[str, Any]) -> Environment env_connection_data=env_connection_data, env_custom_data=environment_details.get("custom_data", {}), connection_model=connection_model, + is_tested=True, ) env_model.save() return env_model @@ -82,13 +83,24 @@ def get_all_environment_models(filter_condition: dict[str, Any]) -> Any: return env_models def get_all_environments(self, page: int, limit: int, filter_condition: dict[str, Any]) -> Any: - env_models = self.get_all_environment_models(filter_condition=filter_condition).order_by("-modified_at") + from django.db.models import Count + + env_qs = ( + self.get_all_environment_models(filter_condition=filter_condition) + .select_related("connection_model") + .annotate( + job_count=Count("usertaskdetails", distinct=True), + project_count=Count("project", distinct=True), + ) + .order_by("-modified_at") + ) - custom_paginator = CustomPaginator(queryset=env_models, limit=limit, page=page) + custom_paginator = CustomPaginator(queryset=env_qs, limit=limit, page=page) env_models = custom_paginator.paginate() env_data = [] for env_model in env_models.get("page_items"): + conn = env_model.connection_model env_data.append( { "id": env_model.environment_id, @@ -96,12 +108,16 @@ def get_all_environments(self, page: int, limit: int, filter_condition: dict[str "description": env_model.environment_description, "deployment_type": env_model.deployment_type, "connection": { - "id": env_model.connection_model.connection_id, - "name": env_model.connection_model.connection_name, - "datasource_name": env_model.connection_model.datasource_name, - "db_icon": import_file(f"visitran.adapters.{env_model.connection_model.datasource_name}").ICON, + "id": conn.connection_id, + "name": conn.connection_name, + "datasource_name": conn.datasource_name, + "db_icon": import_file(f"visitran.adapters.{conn.datasource_name}").ICON, + "host": _get_host_display(conn), + "connection_flag": conn.connection_flag, }, "is_tested": env_model.is_tested, + "job_count": env_model.job_count, + "project_count": env_model.project_count, } ) env_models["page_items"] = env_data @@ -118,6 +134,9 @@ def get_environment(self, environment_id: str) -> dict[str, Any]: "id": env_model.connection_model.connection_id, "name": env_model.connection_model.connection_name, "datasource_name": env_model.connection_model.datasource_name, + "db_icon": import_file(f"visitran.adapters.{env_model.connection_model.datasource_name}").ICON, + "host": _get_host_display(env_model.connection_model), + "connection_flag": env_model.connection_model.connection_flag, }, "connection_details": env_model.masked_connection_data, "custom_data": env_model.env_custom_data, @@ -136,6 +155,7 @@ def update_environment(self, environment_id: str, environment_details: dict[str, ) env_model.env_connection_data = env_connection_data env_model.env_custom_data = environment_details.get("custom_data", {}) + env_model.is_tested = True env_model.save() return env_model except KeyError as e: diff --git a/frontend/src/base/components/connection/ConnectionDrawer.jsx b/frontend/src/base/components/connection/ConnectionDrawer.jsx new file mode 100644 index 00000000..58906002 --- /dev/null +++ b/frontend/src/base/components/connection/ConnectionDrawer.jsx @@ -0,0 +1,883 @@ +/* eslint-disable react/prop-types */ +import { useState, useEffect, useCallback, useMemo, useRef } from "react"; +import Cookies from "js-cookie"; +import { + Drawer, + Form, + Input, + Button, + Space, + Typography, + Alert, + Divider, + Tag, + Row, + Col, + Card, + Tooltip, + Segmented, + Empty, +} from "antd"; +import { + LinkOutlined, + DatabaseOutlined, + SafetyCertificateOutlined, + ThunderboltOutlined, + CheckCircleFilled, + LockOutlined, + EyeOutlined, +} from "@ant-design/icons"; +import RjsfForm from "@rjsf/antd"; +import validator from "@rjsf/validator-ajv8"; + +import { useAxiosPrivate } from "../../../service/axios-service"; +import { orgStore } from "../../../store/org-store"; +import encryptionService from "../../../service/encryption-service"; +import { + fetchDataSources, + fetchDataSourceFields, + createConnectionApi, + updateConnectionApi, + fetchSingleConnection, + revealConnectionCredentials, + testConnectionApi, +} from "../environment/environment-api-service"; +import { useNotificationService } from "../../../service/notification-service"; +import { + validateFormFieldName, + validateFormFieldDescription, + collapseSpaces, +} from "../environment/helper"; +import isEqual from "lodash/isEqual.js"; +import { SpinnerLoader } from "../../../widgets/spinner_loader"; +import { GridObjectFieldTemplate } from "./shared"; + +const { Text } = Typography; +const { TextArea } = Input; + +/* ── DB Tile component — uses real logo from API ── */ +const DBTile = ({ db, isActive, isDisabled, onClick }) => ( +
+
+ {db.label} +
+
{db.label}
+
+); + +const ConnectionDrawer = ({ + open, + onClose, + connectionId, + onSaved, + getContainer, +}) => { + const axiosRef = useAxiosPrivate(); + const { selectedOrgId } = orgStore(); + const csrfToken = Cookies.get("csrftoken"); + const { notify } = useNotificationService(); + const [form] = Form.useForm(); + const watchedName = Form.useWatch("name", form); + const watchedDesc = Form.useWatch("description", form); + + // Data sources + const [dataSources, setDataSources] = useState([]); + const [dsLoading, setDsLoading] = useState(false); + + // Selected DB and fields + const [selectedDb, setSelectedDb] = useState(""); + const [connectionDetails, setConnectionDetails] = useState({}); + const [inputFields, setInputFields] = useState({}); + const [connType, setConnType] = useState("host"); + + // Edit mode + const [originalInfo, setOriginalInfo] = useState(null); + const [isCredentialsRevealed, setIsCredentialsRevealed] = useState(false); + const [isRevealLoading, setIsRevealLoading] = useState(false); + + // Schema for RJSF + const [schema, setSchema] = useState(null); + const [uiSchema, setUiSchema] = useState({}); + + // Actions + const [isTestLoading, setIsTestLoading] = useState(false); + const [isTestSuccess, setIsTestSuccess] = useState(false); + const [testError, setTestError] = useState(null); + const [showErrorDetail, setShowErrorDetail] = useState(false); + const [isSaveLoading, setIsSaveLoading] = useState(false); + const [isEncryptionLoading, setIsEncryptionLoading] = useState(true); + + const hasCapturedOriginalRef = useRef(false); + const [originalConnectionData, setOriginalConnectionData] = useState({}); + + const isEditing = Boolean(connectionId); + + /* ── Init encryption ── */ + useEffect(() => { + if (!open) return; + const init = async () => { + setIsEncryptionLoading(true); + try { + await encryptionService.initialize(selectedOrgId || "default_org"); + } catch { + // Encryption unavailable — proceed without + } finally { + setIsEncryptionLoading(false); + } + }; + init(); + }, [open, selectedOrgId]); + + /* ── Fetch datasources ── */ + useEffect(() => { + if (!open) return; + const load = async () => { + setDsLoading(true); + try { + const ds = await fetchDataSources(axiosRef, selectedOrgId); + setDataSources(ds); + if (!connectionId && ds.length > 0 && !selectedDb) { + setSelectedDb(ds[0].value); + } + } catch (error) { + notify({ error }); + } finally { + setDsLoading(false); + } + }; + load(); + }, [open, selectedOrgId]); + + /* ── Fetch field schema when DB changes ── */ + useEffect(() => { + if (!selectedDb || !open) return; + const load = async () => { + try { + const details = await fetchDataSourceFields( + axiosRef, + selectedOrgId, + selectedDb + ); + setConnectionDetails(details); + } catch (error) { + notify({ error }); + } + }; + load(); + }, [selectedDb, selectedOrgId, open]); + + /* ── Build RJSF schema from connectionDetails + connType ── */ + useEffect(() => { + if (Object.keys(connectionDetails).length === 0) { + setSchema(null); + return; + } + if (["postgres", "snowflake"].includes(selectedDb)) { + const updatedProperties = { ...connectionDetails.properties }; + delete updatedProperties["connection_type"]; + const updatedRequired = + connType === "url" + ? ["connection_url"] + : connectionDetails?.required?.filter( + (el) => + !["connection_url", "schema", "connection_type"].includes(el) + ); + setSchema({ + type: "object", + properties: updatedProperties, + required: updatedRequired, + }); + const ui = {}; + Object.keys(updatedProperties).forEach((key) => { + ui[key] = { + "ui:disabled": + connType === "url" + ? key !== "connection_url" + : key === "connection_url", + }; + }); + setUiSchema({ ...ui, schema: { "ui:disabled": false } }); + } else { + setSchema(connectionDetails); + setUiSchema({}); + } + }, [connectionDetails, connType, selectedDb]); + + /* ── Load existing connection for edit ── */ + useEffect(() => { + if (!connectionId || !open) return; + hasCapturedOriginalRef.current = false; + const load = async () => { + try { + const data = await fetchSingleConnection( + axiosRef, + selectedOrgId, + connectionId + ); + const { name, description, datasource_name, connection_details } = data; + setSelectedDb(datasource_name); + setOriginalInfo({ name: collapseSpaces(name || ""), description }); + form.setFieldsValue({ name, description }); + + const processed = { ...connection_details }; + if ( + datasource_name === "bigquery" && + processed.credentials && + typeof processed.credentials === "object" + ) { + processed.credentials = JSON.stringify( + processed.credentials, + null, + 2 + ); + } + setInputFields(processed); + if (["postgres", "snowflake"].includes(datasource_name)) { + setConnType(connection_details.connection_type || "host"); + } + setIsCredentialsRevealed(false); + } catch (error) { + notify({ error }); + } + }; + load(); + }, [connectionId, open]); + + /* ── Capture original data for change detection ── */ + useEffect(() => { + if ( + connectionId && + Object.keys(inputFields).length > 0 && + !hasCapturedOriginalRef.current + ) { + setOriginalConnectionData(structuredClone(inputFields)); + hasCapturedOriginalRef.current = true; + } else if (!connectionId) { + setOriginalConnectionData({}); + hasCapturedOriginalRef.current = false; + } + }, [connectionId, inputFields]); + + /* ── Reset on close ── */ + useEffect(() => { + if (!open) { + form.resetFields(); + setSelectedDb(""); + setInputFields({}); + setConnectionDetails({}); + setSchema(null); + setIsTestSuccess(false); + setTestError(null); + setShowErrorDetail(false); + setOriginalInfo(null); + setIsCredentialsRevealed(false); + hasCapturedOriginalRef.current = false; + setOriginalConnectionData({}); + } + }, [open]); + + /* ── Has credential data changed? ── */ + const hasCredChanges = useMemo(() => { + if (!connectionId) { + return Object.values(inputFields).some( + (v) => v !== undefined && v !== null && v !== "" + ); + } + if (Object.keys(originalConnectionData).length === 0) return false; + const curr = { ...inputFields }; + const orig = { ...originalConnectionData }; + delete curr.connection_type; + delete orig.connection_type; + return !isEqual(curr, orig); + }, [connectionId, inputFields, originalConnectionData]); + + const hasDetailsChanged = useMemo(() => { + if (!connectionId || !originalInfo) return false; + return ( + watchedName !== originalInfo.name || + watchedDesc !== originalInfo.description + ); + }, [connectionId, originalInfo, watchedName, watchedDesc]); + + const hasValidData = useMemo(() => { + return Object.values(inputFields).some( + (v) => v !== undefined && v !== null && v !== "" + ); + }, [inputFields]); + + /* ── Reveal credentials ── */ + const handleReveal = useCallback(async () => { + if (!connectionId || isCredentialsRevealed) return; + setIsRevealLoading(true); + try { + const creds = await revealConnectionCredentials( + axiosRef, + selectedOrgId, + connectionId + ); + const processed = { ...creds }; + if ( + selectedDb === "bigquery" && + processed.credentials && + typeof processed.credentials === "object" + ) { + processed.credentials = JSON.stringify(processed.credentials, null, 2); + } + setInputFields(processed); + setIsCredentialsRevealed(true); + } catch (error) { + notify({ error }); + } finally { + setIsRevealLoading(false); + } + }, [connectionId, selectedOrgId, isCredentialsRevealed, selectedDb]); + + /* ── Test connection ── */ + const handleTest = useCallback(async () => { + setIsTestLoading(true); + setIsTestSuccess(false); + setTestError(null); + setShowErrorDetail(false); + try { + const testData = { + ...inputFields, + ...(["postgres", "snowflake"].includes(selectedDb) && { + schema: inputFields.schema || "", + connection_type: connType, + }), + }; + const data = encryptionService.isAvailable() + ? await encryptionService.encryptSensitiveFields(testData) + : testData; + await testConnectionApi( + axiosRef, + selectedOrgId, + csrfToken, + selectedDb, + data, + connectionId || null + ); + setIsTestSuccess(true); + } catch (error) { + const errorData = error?.response?.data; + const errMsg = + errorData?.error_message || + errorData?.message || + errorData?.error || + error?.message || + "Connection test failed"; + const statusCode = error?.response?.status; + setTestError({ + summary: statusCode ? `Error ${statusCode}` : "Connection test failed", + detail: errMsg, + }); + } finally { + setIsTestLoading(false); + } + }, [ + inputFields, + selectedDb, + connType, + selectedOrgId, + csrfToken, + connectionId, + ]); + + /* ── Save connection ── */ + const handleSave = useCallback(async () => { + try { + await form.validateFields(); + } catch { + return; + } + setIsSaveLoading(true); + try { + const { name, description } = form.getFieldsValue(); + const payload = { + datasource_name: selectedDb, + name, + description, + connection_details: { + ...inputFields, + ...(["postgres", "snowflake"].includes(selectedDb) && { + schema: inputFields.schema || "", + connection_type: connType, + }), + }, + ...(connectionId && + hasDetailsChanged && + !hasCredChanges && { metadata_only: true }), + }; + if (encryptionService.isAvailable()) { + payload.connection_details = + await encryptionService.encryptSensitiveFields( + payload.connection_details + ); + } + if (!connectionId) { + const res = await createConnectionApi( + axiosRef, + selectedOrgId, + csrfToken, + payload + ); + if (res.status === 200) { + notify({ + type: "success", + message: "Connection created successfully", + }); + onSaved?.(); + onClose(); + } + } else { + const res = await updateConnectionApi( + axiosRef, + selectedOrgId, + csrfToken, + connectionId, + payload + ); + if (res.status === 200) { + notify({ + type: "success", + message: "Connection updated successfully", + }); + onSaved?.(); + onClose(); + } + } + } catch (error) { + notify({ error }); + } finally { + setIsSaveLoading(false); + } + }, [ + form, + selectedDb, + inputFields, + connType, + connectionId, + hasDetailsChanged, + hasCredChanges, + selectedOrgId, + csrfToken, + ]); + + /* ── RJSF handlers ── */ + const handleFieldChange = ({ formData }) => { + setInputFields(formData); + if (isTestSuccess) setIsTestSuccess(false); + if (testError) { + setTestError(null); + setShowErrorDetail(false); + } + }; + + const handleFieldSubmit = ({ formData }) => { + setInputFields(formData); + handleTest(); + }; + + const handleConnTypeChange = (value) => { + setConnType(value); + if (!connectionId) setInputFields({}); + }; + + /* ── Derive selected DB info ── */ + const selectedDbInfo = dataSources.find((d) => d.value === selectedDb); + const dbLabel = + selectedDbInfo?.label || + selectedDb.charAt(0).toUpperCase() + selectedDb.slice(1); + + const canSave = + isTestSuccess || (connectionId && hasDetailsChanged && !hasCredChanges); + + return ( + + + {isEditing ? "Edit Connection" : "New Connection"} + + } + width={640} + open={open} + onClose={onClose} + destroyOnClose + keyboard={false} + maskClosable={false} + getContainer={getContainer} + className="conn-drawer" + footer={ + + + {isTestSuccess && ( + + + Tested + + )} + + + + + + + + + } + > +
+ {/* ── STEP 1: Database picker ── */} +
+ 1.{" "} + {isEditing ? ( + + Database{" "} + + · Locked after creation + + + ) : ( + "Pick your database" + )} +
+ {!isEditing && ( + + The fields below will adjust based on your choice. + + )} + {isEditing && ( + } + message={ + + Database can't be changed after creation.{" "} + Create a new connection for a different database. + + } + style={{ marginBottom: 14 }} + /> + )} + {dsLoading ? ( + + ) : ( +
+ {dataSources.map((db) => { + const isActive = selectedDb === db.value; + const isDisabled = isEditing && !isActive; + return ( + { + if (!isEditing) { + setSelectedDb(db.value); + setInputFields({}); + setIsTestSuccess(false); + } + }} + /> + ); + })} +
+ )} + + + + {/* ── STEP 2: Name & Describe ── */} +
2. Name & describe
+ + + } + /> + + +