[KM-436][KM-437] add and modify for database and document pipeline

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+software/** filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -26,6 +26,8 @@ test/users/user_accounts.csv
 .env.prd
 .env.example
 
+CLAUDE.md
+
 erd/
 playground/
 playground_retriever.py
@@ -33,4 +35,5 @@ playground_chat.py
 playground_flush_cache.py
 playground_create_user.py
 API_CONTRACT.md
-context_engineering/
+context_engineering/
+sample_file/

main.py

@@ -10,6 +10,7 @@ from src.api.v1.chat import router as chat_router
 from src.api.v1.room import router as room_router
 from src.api.v1.users import router as users_router
 from src.api.v1.knowledge import router as knowledge_router
+from src.api.v1.db_client import router as db_client_router
 from src.db.postgres.init_db import init_db
 import uvicorn
 
@@ -35,6 +36,7 @@ app.include_router(document_router)
 app.include_router(knowledge_router)
 app.include_router(room_router)
 app.include_router(chat_router)
+app.include_router(db_client_router)
 
 
 @app.on_event("startup")

pyproject.toml

@@ -79,12 +79,13 @@ dependencies = [
     "jsonpatch>=1.33",
     "pymongo>=4.14.0",
     "psycopg2>=2.9.11",
-]
-
-[project.optional-dependencies]
-dev = [
-    "pytest==8.3.4",
-    "pytest-asyncio==0.24.0",
+    "pdf2image>=1.17.0",
+    "pytesseract>=0.3.13",
+    "pypdf2>=3.0.1",
+    "pdf2image>=1.17.0",
+    "pytesseract>=0.3.13",
+    "pypdf2>=3.0.1",
+    "pdf24.0",
     "pytest-cov==6.0.0",
     "httpx==0.28.1",
     "ruff==0.8.4",

src/api/v1/db_client.py

@@ -1,5 +1,357 @@
-from typing import Literal, Dict
+"""API endpoints for user-registered database connections.
 
+Credential schemas (DbType, PostgresCredentials, etc.) live in
+`src/models/credentials.py` — they are imported below (with noqa: F401) so
+FastAPI/Swagger picks them up for OpenAPI schema generation even though they
+are not referenced by name in this file.
+"""
 
-dbtypes: Literal["postgresql", "mysql", "sqlite"] = Literal["postgresql", "mysql", "sqlite"]
-creds: Dict[str, str]
+from typing import Any, Dict, List, Literal, Optional
+from datetime import datetime
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
+from pydantic import BaseModel, Field
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.database_client.database_client_service import database_client_service
+from src.db.postgres.connection import get_db
+from src.middlewares.logging import get_logger, log_execution
+from src.middlewares.rate_limit import limiter
+from src.models.credentials import (  # noqa: F401 — re-exported for Swagger schema discovery
+    BigQueryCredentials,
+    CredentialSchemas,
+    DbType,
+    MysqlCredentials,
+    PostgresCredentials,
+    SnowflakeCredentials,
+    SqlServerCredentials,
+    SupabaseCredentials,
+)
+from src.pipeline.db_pipeline import db_pipeline_service
+from src.utils.db_credential_encryption import decrypt_credentials_dict
+
+logger = get_logger("database_client_api")
+
+router = APIRouter(prefix="/api/v1", tags=["Database Clients"])
+
+
+# ---------------------------------------------------------------------------
+# Request / Response schemas
+# ---------------------------------------------------------------------------
+
+
+class DatabaseClientCreate(BaseModel):
+    """
+    Payload to register a new external database connection.
+
+    The `credentials` object shape depends on `db_type`:
+
+    | db_type     | Required fields                                          |
+    |-------------|----------------------------------------------------------|
+    | postgres    | host, port, database, username, password, ssl_mode       |
+    | mysql       | host, port, database, username, password, ssl            |
+    | sqlserver   | host, port, database, username, password, driver?        |
+    | supabase    | host, port, database, username, password, ssl_mode       |
+    | bigquery    | project_id, dataset_id, location?, service_account_json  |
+    | snowflake   | account, warehouse, database, schema?, username, password, role? |
+
+    Sensitive fields (`password`, `service_account_json`) are encrypted
+    at rest using Fernet symmetric encryption.
+    """
+
+    name: str = Field(..., description="Display name for this connection.", examples=["Production DB"])
+    db_type: DbType = Field(..., description="Type of the database engine.", examples=["postgres"])
+    credentials: Dict[str, Any] = Field(
+        ...,
+        description="Connection credentials. Shape depends on db_type. See schema descriptions above.",
+        examples=[
+            {
+                "host": "db.example.com",
+                "port": 5432,
+                "database": "mydb",
+                "username": "admin",
+                "password": "s3cr3t!",
+                "ssl_mode": "require",
+            }
+        ],
+    )
+
+
+class DatabaseClientUpdate(BaseModel):
+    """
+    Payload to update an existing database connection.
+
+    All fields are optional — only provided fields will be updated.
+    If `credentials` is provided, it replaces the entire credentials object
+    and sensitive fields are re-encrypted.
+    """
+
+    name: Optional[str] = Field(None, description="New display name for this connection.", examples=["Staging DB"])
+    credentials: Optional[Dict[str, Any]] = Field(
+        None,
+        description="Updated credentials object. Replaces existing credentials entirely if provided.",
+        examples=[{"host": "new-host.example.com", "port": 5432, "database": "mydb", "username": "admin", "password": "n3wP@ss!", "ssl_mode": "require"}],
+    )
+    status: Optional[Literal["active", "inactive"]] = Field(
+        None,
+        description="Set to 'inactive' to soft-disable the connection without deleting it.",
+        examples=["inactive"],
+    )
+
+
+class DatabaseClientResponse(BaseModel):
+    """
+    Database connection record returned by the API.
+
+    Credentials are **never** included in the response for security reasons.
+    """
+
+    id: str = Field(..., description="Unique identifier of the database connection.")
+    user_id: str = Field(..., description="ID of the user who owns this connection.")
+    name: str = Field(..., description="Display name of the connection.")
+    db_type: str = Field(..., description="Database engine type.")
+    status: str = Field(..., description="Connection status: 'active' or 'inactive'.")
+    created_at: datetime = Field(..., description="Timestamp when the connection was registered.")
+    updated_at: Optional[datetime] = Field(None, description="Timestamp of the last update, if any.")
+
+    model_config = {"from_attributes": True}
+
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.post(
+    "/database-clients",
+    response_model=DatabaseClientResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Register a new database connection",
+    response_description="The newly created database connection record (credentials excluded).",
+    responses={
+        201: {"description": "Connection registered successfully."},
+        422: {"description": "Validation error — check the credentials shape for the given db_type."},
+        500: {"description": "Internal server error."},
+    },
+)
+@limiter.limit("10/minute")
+@log_execution(logger)
+async def create_database_client(
+    request: Request,
+    payload: DatabaseClientCreate,
+    user_id: str = Query(..., description="ID of the user registering the connection."),
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Register a new external database connection for a user.
+
+    The `credentials` object must match the shape for the chosen `db_type`
+    (see **CredentialSchemas** in the schema section below for exact fields).
+    Sensitive fields (`password`, `service_account_json`) are encrypted
+    before being persisted — they are never returned in any response.
+    """
+    try:
+        client = await database_client_service.create(
+            db=db,
+            user_id=user_id,
+            name=payload.name,
+            db_type=payload.db_type,
+            credentials=payload.credentials,
+        )
+        return DatabaseClientResponse.model_validate(client)
+    except Exception as e:
+        logger.error(f"Failed to create database client for user {user_id}", error=str(e))
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Failed to create database client: {str(e)}",
+        )
+
+
+@router.get(
+    "/database-clients/{user_id}",
+    response_model=List[DatabaseClientResponse],
+    summary="List all database connections for a user",
+    response_description="List of database connections (credentials excluded).",
+    responses={
+        200: {"description": "Returns an empty list if the user has no connections."},
+    },
+)
+@log_execution(logger)
+async def list_database_clients(
+    user_id: str,
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Return all database connections registered by the specified user,
+    ordered by creation date (newest first).
+
+    Credentials are never included in the response.
+    """
+    clients = await database_client_service.get_user_clients(db, user_id)
+    return [DatabaseClientResponse.model_validate(c) for c in clients]
+
+
+@router.get(
+    "/database-clients/{user_id}/{client_id}",
+    response_model=DatabaseClientResponse,
+    summary="Get a single database connection",
+    response_description="Database connection detail (credentials excluded).",
+    responses={
+        404: {"description": "Connection not found."},
+        403: {"description": "Access denied — user_id does not own this connection."},
+    },
+)
+@log_execution(logger)
+async def get_database_client(
+    user_id: str,
+    client_id: str,
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Return the detail of a single database connection.
+
+    Returns **403** if the `user_id` in the path does not match the owner
+    of the requested connection.
+    """
+    client = await database_client_service.get(db, client_id)
+
+    if not client:
+        raise HTTPException(status_code=404, detail="Database client not found")
+
+    if client.user_id != user_id:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+    return DatabaseClientResponse.model_validate(client)
+
+
+@router.put(
+    "/database-clients/{client_id}",
+    response_model=DatabaseClientResponse,
+    summary="Update a database connection",
+    response_description="Updated database connection record (credentials excluded).",
+    responses={
+        404: {"description": "Connection not found."},
+        403: {"description": "Access denied — user_id does not own this connection."},
+    },
+)
+@log_execution(logger)
+async def update_database_client(
+    client_id: str,
+    payload: DatabaseClientUpdate,
+    user_id: str = Query(..., description="ID of the user who owns the connection."),
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Update an existing database connection.
+
+    Only fields present in the request body are updated.
+    If `credentials` is provided it **replaces** the entire credentials object
+    and sensitive fields are re-encrypted automatically.
+    """
+    client = await database_client_service.get(db, client_id)
+
+    if not client:
+        raise HTTPException(status_code=404, detail="Database client not found")
+
+    if client.user_id != user_id:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+    updated = await database_client_service.update(
+        db=db,
+        client_id=client_id,
+        name=payload.name,
+        credentials=payload.credentials,
+        status=payload.status,
+    )
+    return DatabaseClientResponse.model_validate(updated)
+
+
+@router.delete(
+    "/database-clients/{client_id}",
+    status_code=status.HTTP_200_OK,
+    summary="Delete a database connection",
+    responses={
+        200: {"description": "Connection deleted successfully."},
+        404: {"description": "Connection not found."},
+        403: {"description": "Access denied — user_id does not own this connection."},
+    },
+)
+@log_execution(logger)
+async def delete_database_client(
+    client_id: str,
+    user_id: str = Query(..., description="ID of the user who owns the connection."),
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Permanently delete a database connection.
+
+    This action is irreversible. The stored credentials are also removed.
+    """
+    client = await database_client_service.get(db, client_id)
+
+    if not client:
+        raise HTTPException(status_code=404, detail="Database client not found")
+
+    if client.user_id != user_id:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+    await database_client_service.delete(db, client_id)
+    return {"status": "success", "message": "Database client deleted successfully"}
+
+
+@router.post(
+    "/database-clients/{client_id}/ingest",
+    status_code=status.HTTP_200_OK,
+    summary="Ingest schema from a registered database into the vector store",
+    response_description="Count of chunks ingested.",
+    responses={
+        200: {"description": "Ingestion completed successfully."},
+        403: {"description": "Access denied — user_id does not own this connection."},
+        404: {"description": "Connection not found."},
+        501: {"description": "The connection's db_type is not yet supported by the pipeline."},
+        500: {"description": "Ingestion failed (connection error, profiling error, etc.)."},
+    },
+)
+@limiter.limit("5/minute")
+@log_execution(logger)
+async def ingest_database_client(
+    request: Request,
+    client_id: str,
+    user_id: str = Query(..., description="ID of the user who owns the connection."),
+    db: AsyncSession = Depends(get_db),
+):
+    """
+    Decrypt the stored credentials, connect to the user's database, introspect
+    its schema, profile each column, embed the descriptions, and store them in
+    the shared PGVector collection tagged with `source_type="database"`.
+
+    Chunks become retrievable via the same retriever used for document chunks.
+    """
+    client = await database_client_service.get(db, client_id)
+
+    if not client:
+        raise HTTPException(status_code=404, detail="Database client not found")
+
+    if client.user_id != user_id:
+        raise HTTPException(status_code=403, detail="Access denied")
+
+    creds = decrypt_credentials_dict(client.credentials)
+
+    try:
+        with db_pipeline_service.engine_scope(
+            db_type=client.db_type,
+            credentials=creds,
+        ) as engine:
+            total = await db_pipeline_service.run(user_id=user_id, engine=engine)
+    except NotImplementedError as e:
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(e))
+    except Exception as e:
+        logger.error(
+            f"Ingestion failed for client {client_id}", user_id=user_id, error=str(e)
+        )
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Ingestion failed: {e}",
+        )
+
+    return {"status": "success", "client_id": client_id, "chunks_ingested": total}

src/api/v1/document.py

@@ -1,21 +1,20 @@
 """Document management API endpoints."""
-
-from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, status
+ 
+from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File
 from sqlalchemy.ext.asyncio import AsyncSession
 from src.db.postgres.connection import get_db
 from src.document.document_service import document_service
-from src.knowledge.processing_service import knowledge_processor
-from src.storage.az_blob.az_blob import blob_storage
 from src.middlewares.logging import get_logger, log_execution
 from src.middlewares.rate_limit import limiter
+from src.pipeline.document_pipeline.document_pipeline import document_pipeline
 from pydantic import BaseModel
 from typing import List
-
+ 
 logger = get_logger("document_api")
-
+ 
 router = APIRouter(prefix="/api/v1", tags=["Documents"])
-
-
+ 
+ 
 class DocumentResponse(BaseModel):
     id: str
     filename: str
@@ -23,8 +22,8 @@ class DocumentResponse(BaseModel):
     file_size: int
     file_type: str
     created_at: str
-
-
+ 
+ 
 @router.get("/documents/{user_id}", response_model=List[DocumentResponse])
 @log_execution(logger)
 async def list_documents(
@@ -44,8 +43,8 @@ async def list_documents(
         )
         for doc in documents
     ]
-
-
+ 
+ 
 @router.post("/document/upload")
 @limiter.limit("10/minute")
 @log_execution(logger)
@@ -57,57 +56,12 @@ async def upload_document(
 ):
     """Upload a document."""
     if not user_id:
-        raise HTTPException(
-            status_code=400,
-            detail="user_id is required"
-        )
-
-    try:
-        # Read file content
-        content = await file.read()
-        file_size = len(content)
-
-        # Get file type
-        filename = file.filename
-        file_type = filename.split('.')[-1].lower() if '.' in filename else 'txt'
-
-        if file_type not in ['pdf', 'docx', 'txt']:
-            raise HTTPException(
-                status_code=400,
-                detail="Unsupported file type. Supported: pdf, docx, txt"
-            )
-
-        # Upload to blob storage
-        blob_name = await blob_storage.upload_file(content, filename, user_id)
-
-        # Create document record
-        document = await document_service.create_document(
-            db=db,
-            user_id=user_id,
-            filename=filename,
-            blob_name=blob_name,
-            file_size=file_size,
-            file_type=file_type
-        )
-
-        return {
-            "status": "success",
-            "message": "Document uploaded successfully",
-            "data": {
-                "id": document.id,
-                "filename": document.filename,
-                "status": document.status
-            }
-        }
-
-    except Exception as e:
-        logger.error(f"Upload failed for user {user_id}", error=str(e))
-        raise HTTPException(
-            status_code=500,
-            detail=f"Upload failed: {str(e)}"
-        )
-
-
+        raise HTTPException(status_code=400, detail="user_id is required")
+ 
+    data = await document_pipeline.upload(file, user_id, db)
+    return {"status": "success", "message": "Document uploaded successfully", "data": data}
+ 
+ 
 @router.delete("/document/delete")
 @log_execution(logger)
 async def delete_document(
@@ -116,31 +70,10 @@ async def delete_document(
     db: AsyncSession = Depends(get_db)
 ):
     """Delete a document."""
-    document = await document_service.get_document(db, document_id)
-
-    if not document:
-        raise HTTPException(
-            status_code=404,
-            detail="Document not found"
-        )
-
-    if document.user_id != user_id:
-        raise HTTPException(
-            status_code=403,
-            detail="Access denied"
-        )
-
-    success = await document_service.delete_document(db, document_id)
-
-    if success:
-        return {"status": "success", "message": "Document deleted successfully"}
-    else:
-        raise HTTPException(
-            status_code=500,
-            detail="Failed to delete document"
-        )
-
-
+    await document_pipeline.delete(document_id, user_id, db)
+    return {"status": "success", "message": "Document deleted successfully"}
+ 
+ 
 @router.post("/document/process")
 @log_execution(logger)
 async def process_document(
@@ -149,45 +82,6 @@ async def process_document(
     db: AsyncSession = Depends(get_db)
 ):
     """Process document and ingest to vector index."""
-    document = await document_service.get_document(db, document_id)
-
-    if not document:
-        raise HTTPException(
-            status_code=404,
-            detail="Document not found"
-        )
-
-    if document.user_id != user_id:
-        raise HTTPException(
-            status_code=403,
-            detail="Access denied"
-        )
-
-    try:
-        # Update status to processing
-        await document_service.update_document_status(db, document_id, "processing")
-
-        # Process document
-        chunks_count = await knowledge_processor.process_document(document, db)
-
-        # Update status to completed
-        await document_service.update_document_status(db, document_id, "completed")
-
-        return {
-            "status": "success",
-            "message": "Document processed successfully",
-            "data": {
-                "document_id": document_id,
-                "chunks_processed": chunks_count
-            }
-        }
-
-    except Exception as e:
-        logger.error(f"Processing failed for document {document_id}", error=str(e))
-        await document_service.update_document_status(
-            db, document_id, "failed", str(e)
-        )
-        raise HTTPException(
-            status_code=500,
-            detail=f"Processing failed: {str(e)}"
-        )
+    data = await document_pipeline.process(document_id, user_id, db)
+    return {"status": "success", "message": "Document processed successfully", "data": data}
+ 

src/config/settings.py

@@ -61,6 +61,11 @@ class Settings(BaseSettings):
     # Bcrypt salt (for users - existing)
     emarcal_bcrypt_salt: str = Field(alias="emarcal__bcrypt__salt", default="")
 
+    # DB credential encryption (Fernet key for user-registered database creds)
+    dataeyond_db_credential_key: str = Field(
+        alias="dataeyond__db__credential__key", default=""
+    )
+
 
 # Singleton instance
 settings = Settings()

src/database_client/database_client_service.py

@@ -0,0 +1,118 @@
+"""Service for managing user-registered external database connections."""
+
+import uuid
+from typing import List, Optional
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.db.postgres.models import DatabaseClient
+from src.middlewares.logging import get_logger
+from src.utils.db_credential_encryption import (
+    decrypt_credentials_dict,
+    encrypt_credentials_dict,
+)
+
+logger = get_logger("database_client_service")
+
+
+class DatabaseClientService:
+    """Service for managing user-registered external database connections."""
+
+    async def create(
+        self,
+        db: AsyncSession,
+        user_id: str,
+        name: str,
+        db_type: str,
+        credentials: dict,
+    ) -> DatabaseClient:
+        """Register a new database client connection.
+
+        Credentials are encrypted before being stored.
+        """
+        client = DatabaseClient(
+            id=str(uuid.uuid4()),
+            user_id=user_id,
+            name=name,
+            db_type=db_type,
+            credentials=encrypt_credentials_dict(credentials),
+            status="active",
+        )
+        db.add(client)
+        await db.commit()
+        await db.refresh(client)
+        logger.info(f"Created database client {client.id} for user {user_id}")
+        return client
+
+    async def get_user_clients(
+        self,
+        db: AsyncSession,
+        user_id: str,
+    ) -> List[DatabaseClient]:
+        """Return all active and inactive database clients for a user."""
+        result = await db.execute(
+            select(DatabaseClient)
+            .where(DatabaseClient.user_id == user_id)
+            .order_by(DatabaseClient.created_at.desc())
+        )
+        return result.scalars().all()
+
+    async def get(
+        self,
+        db: AsyncSession,
+        client_id: str,
+    ) -> Optional[DatabaseClient]:
+        """Return a single database client by its ID."""
+        result = await db.execute(
+            select(DatabaseClient).where(DatabaseClient.id == client_id)
+        )
+        return result.scalars().first()
+
+    async def update(
+        self,
+        db: AsyncSession,
+        client_id: str,
+        name: Optional[str] = None,
+        credentials: Optional[dict] = None,
+        status: Optional[str] = None,
+    ) -> Optional[DatabaseClient]:
+        """Update an existing database client connection.
+
+        Only non-None fields are updated.
+        Credentials are re-encrypted if provided.
+        """
+        client = await self.get(db, client_id)
+        if not client:
+            return None
+
+        if name is not None:
+            client.name = name
+        if credentials is not None:
+            client.credentials = encrypt_credentials_dict(credentials)
+        if status is not None:
+            client.status = status
+
+        await db.commit()
+        await db.refresh(client)
+        logger.info(f"Updated database client {client_id}")
+        return client
+
+    async def delete(
+        self,
+        db: AsyncSession,
+        client_id: str,
+    ) -> bool:
+        """Permanently delete a database client connection."""
+        result = await db.execute(
+            delete(DatabaseClient).where(DatabaseClient.id == client_id)
+        )
+        await db.commit()
+        deleted = result.rowcount > 0
+        if deleted:
+            logger.info(f"Deleted database client {client_id}")
+        return deleted
+
+
+database_client_service = DatabaseClientService()
+ 

src/db/postgres/init_db.py

@@ -2,7 +2,14 @@
 
 from sqlalchemy import text
 from src.db.postgres.connection import engine, Base
-from src.db.postgres.models import Document, Room, ChatMessage, User, MessageSource
+from src.db.postgres.models import (
+    ChatMessage,
+    DatabaseClient,
+    Document,
+    MessageSource,
+    Room,
+    User,
+)
 
 
 async def init_db():

src/db/postgres/models.py

@@ -4,6 +4,7 @@ from uuid import uuid4
 from sqlalchemy import Column, String, DateTime, Text, Integer, ForeignKey
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
+from sqlalchemy.dialects.postgresql import JSONB
 from src.db.postgres.connection import Base
 
 
@@ -81,3 +82,18 @@ class MessageSource(Base):
     created_at = Column(DateTime(timezone=True), server_default=func.now())
 
     message = relationship("ChatMessage", back_populates="sources")
+
+
+class DatabaseClient(Base):
+    """User-registered external database connections."""
+    __tablename__ = "databases"
+
+    id = Column(String, primary_key=True, default=lambda: str(uuid4()))
+    user_id = Column(String, nullable=False, index=True)
+    name = Column(String, nullable=False)       # display name, e.g. "Prod DB"
+    db_type = Column(String, nullable=False)    # postgres|mysql|sqlserver|supabase|bigquery|snowflake
+    credentials = Column(JSONB, nullable=False) # per-type JSON; sensitive fields Fernet-encrypted
+    status = Column(String, nullable=False, default="active")  # active | inactive
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+ 

src/knowledge/processing_service.py

@@ -5,14 +5,14 @@ from langchain_core.documents import Document as LangChainDocument
 from src.db.postgres.vector_store import get_vector_store
 from src.storage.az_blob.az_blob import blob_storage
 from src.db.postgres.models import Document as DBDocument
-from src.config.settings import settings
 from sqlalchemy.ext.asyncio import AsyncSession
 from src.middlewares.logging import get_logger
-from azure.ai.documentintelligence.aio import DocumentIntelligenceClient
-from azure.core.credentials import AzureKeyCredential
 from typing import List
-import pypdf
+import sys
 import docx
+import pandas as pd
+import pytesseract
+from pdf2image import convert_from_bytes
 from io import BytesIO
 
 logger = get_logger("knowledge_processing")
@@ -40,6 +40,10 @@ class KnowledgeProcessingService:
 
             if db_doc.file_type == "pdf":
                 documents = await self._build_pdf_documents(content, db_doc)
+            elif db_doc.file_type == "csv":
+                documents = self._build_csv_documents(content, db_doc)
+            elif db_doc.file_type == "xlsx":
+                documents = self._build_excel_documents(content, db_doc)
             else:
                 text = self._extract_text(content, db_doc.file_type)
                 if not text.strip():
@@ -49,10 +53,14 @@ class KnowledgeProcessingService:
                     LangChainDocument(
                         page_content=chunk,
                         metadata={
-                            "document_id": db_doc.id,
                             "user_id": db_doc.user_id,
-                            "filename": db_doc.filename,
-                            "chunk_index": i,
+                            "source_type": "document",
+                            "data": {
+                                "document_id": db_doc.id,
+                                "filename": db_doc.filename,
+                                "file_type": db_doc.file_type,
+                                "chunk_index": i,
+                            },
                         }
                     )
                     for i, chunk in enumerate(chunks)
@@ -74,62 +82,98 @@ class KnowledgeProcessingService:
     async def _build_pdf_documents(
         self, content: bytes, db_doc: DBDocument
     ) -> List[LangChainDocument]:
-        """Build LangChain documents from PDF with page_label metadata.
-
-        Uses Azure Document Intelligence (per-page) when credentials are present,
-        falls back to pypdf (also per-page) otherwise.
-        """
+        """Build LangChain documents from PDF with page_label metadata using Tesseract OCR."""
         documents: List[LangChainDocument] = []
 
-        if settings.azureai_docintel_endpoint and settings.azureai_docintel_key:
-            async with DocumentIntelligenceClient(
-                endpoint=settings.azureai_docintel_endpoint,
-                credential=AzureKeyCredential(settings.azureai_docintel_key),
-            ) as client:
-                poller = await client.begin_analyze_document(
-                    model_id="prebuilt-read",
-                    body=BytesIO(content),
-                    content_type="application/pdf",
-                )
-                result = await poller.result()
-                logger.info(f"Azure DI extracted {len(result.pages or [])} pages")
-
-                for page in result.pages or []:
-                    page_text = "\n".join(
-                        line.content for line in (page.lines or [])
-                    )
-                    if not page_text.strip():
-                        continue
-                    for chunk in self.text_splitter.split_text(page_text):
-                        documents.append(LangChainDocument(
-                            page_content=chunk,
-                            metadata={
-                                "document_id": db_doc.id,
-                                "user_id": db_doc.user_id,
-                                "filename": db_doc.filename,
-                                "chunk_index": len(documents),
-                                "page_label": page.page_number,
-                            }
-                        ))
-        else:
-            logger.warning("Azure DI not configured, using pypdf")
-            pdf_reader = pypdf.PdfReader(BytesIO(content))
-            for page_num, page in enumerate(pdf_reader.pages, start=1):
-                page_text = page.extract_text() or ""
-                if not page_text.strip():
-                    continue
-                for chunk in self.text_splitter.split_text(page_text):
-                    documents.append(LangChainDocument(
-                        page_content=chunk,
-                        metadata={
+        poppler_path = None
+        if sys.platform == "win32":
+            pytesseract.pytesseract.tesseract_cmd = r"./software/Tesseract-OCR/tesseract.exe"
+            poppler_path = "./software/poppler-24.08.0/Library/bin"
+
+        images = convert_from_bytes(content, poppler_path=poppler_path)
+        logger.info(f"Tesseract OCR: converting {len(images)} pages")
+
+        for page_num, image in enumerate(images, start=1):
+            page_text = pytesseract.image_to_string(image)
+            if not page_text.strip():
+                continue
+            for chunk in self.text_splitter.split_text(page_text):
+                documents.append(LangChainDocument(
+                    page_content=chunk,
+                    metadata={
+                        "user_id": db_doc.user_id,
+                        "source_type": "document",
+                        "data": {
                             "document_id": db_doc.id,
-                            "user_id": db_doc.user_id,
                             "filename": db_doc.filename,
+                            "file_type": db_doc.file_type,
                             "chunk_index": len(documents),
                             "page_label": page_num,
-                        }
-                    ))
+                        },
+                    }
+                ))
+
+        return documents
+
+    def _profile_dataframe(
+        self, df: pd.DataFrame, source_name: str, db_doc: DBDocument
+    ) -> List[LangChainDocument]:
+        """Profile each column of a dataframe → one chunk per column."""
+        documents = []
+        row_count = len(df)
+
+        for col_name in df.columns:
+            col = df[col_name]
+            is_numeric = pd.api.types.is_numeric_dtype(col)
+            null_count = int(col.isnull().sum())
+            distinct_count = int(col.nunique())
+            distinct_ratio = distinct_count / row_count if row_count > 0 else 0
+
+            text = f"Source: {source_name} ({row_count} rows)\n"
+            text += f"Column: {col_name} ({col.dtype})\n"
+            text += f"Null count: {null_count}\n"
+            text += f"Distinct count: {distinct_count} ({distinct_ratio:.1%})\n"
+
+            if is_numeric:
+                text += f"Min: {col.min()}, Max: {col.max()}\n"
+                text += f"Mean: {col.mean():.4f}, Median: {col.median():.4f}\n"
+
+            if 0 < distinct_ratio <= 0.05:
+                top_values = col.value_counts().head(10)
+                top_str = ", ".join(f"{v} ({c})" for v, c in top_values.items())
+                text += f"Top values: {top_str}\n"
+
+            text += f"Sample values: {col.dropna().head(5).tolist()}"
+
+            documents.append(LangChainDocument(
+                page_content=text,
+                metadata={
+                    "user_id": db_doc.user_id,
+                    "source_type": "document",
+                    "data": {
+                        "document_id": db_doc.id,
+                        "filename": db_doc.filename,
+                        "file_type": db_doc.file_type,
+                        "source": source_name,
+                        "column_name": col_name,
+                        "column_type": str(col.dtype),
+                    }
+                }
+            ))
+        return documents
 
+    def _build_csv_documents(self, content: bytes, db_doc: DBDocument) -> List[LangChainDocument]:
+        """Profile each column of a CSV file."""
+        df = pd.read_csv(BytesIO(content))
+        return self._profile_dataframe(df, db_doc.filename, db_doc)
+
+    def _build_excel_documents(self, content: bytes, db_doc: DBDocument) -> List[LangChainDocument]:
+        """Profile each column of every sheet in an Excel file."""
+        sheets = pd.read_excel(BytesIO(content), sheet_name=None)
+        documents = []
+        for sheet_name, df in sheets.items():
+            source_name = f"{db_doc.filename} / sheet: {sheet_name}"
+            documents.extend(self._profile_dataframe(df, source_name, db_doc))
         return documents
 
     def _extract_text(self, content: bytes, file_type: str) -> str:

src/models/credentials.py

@@ -0,0 +1,164 @@
+"""Pydantic credential schemas for user-registered external databases.
+
+Imported by the `/database-clients` API router (`src/api/v1/db_client.py`) and,
+via `DbType`, by the db pipeline connector (`src/pipeline/db_pipeline/connector.py`).
+
+Sensitive fields (`password`, `service_account_json`) are Fernet-encrypted by
+the database_client service before being stored in the JSONB column; these
+schemas describe the plaintext wire format, not the stored shape.
+"""
+
+from typing import Literal, Optional, Union
+
+from pydantic import BaseModel, Field
+
+# ---------------------------------------------------------------------------
+# Supported DB types
+# ---------------------------------------------------------------------------
+
+DbType = Literal["postgres", "mysql", "sqlserver", "supabase", "bigquery", "snowflake"]
+
+
+# ---------------------------------------------------------------------------
+# Typed credential schemas per DB type
+# ---------------------------------------------------------------------------
+
+
+class PostgresCredentials(BaseModel):
+    """Connection credentials for PostgreSQL."""
+
+    host: str = Field(..., description="Hostname or IP address of the PostgreSQL server.", examples=["db.example.com"])
+    port: int = Field(5432, description="Port number (default: 5432).", examples=[5432])
+    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
+    username: str = Field(..., description="Database username.", examples=["admin"])
+    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
+    ssl_mode: Literal["disable", "require", "verify-ca", "verify-full"] = Field(
+        "require",
+        description="SSL mode for the connection.",
+        examples=["require"],
+    )
+
+
+class MysqlCredentials(BaseModel):
+    """Connection credentials for MySQL."""
+
+    host: str = Field(..., description="Hostname or IP address of the MySQL server.", examples=["db.example.com"])
+    port: int = Field(3306, description="Port number (default: 3306).", examples=[3306])
+    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
+    username: str = Field(..., description="Database username.", examples=["admin"])
+    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
+    ssl: bool = Field(True, description="Enable SSL for the connection.", examples=[True])
+
+
+class SqlServerCredentials(BaseModel):
+    """Connection credentials for Microsoft SQL Server."""
+
+    host: str = Field(..., description="Hostname or IP address of the SQL Server.", examples=["sqlserver.example.com"])
+    port: int = Field(1433, description="Port number (default: 1433).", examples=[1433])
+    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
+    username: str = Field(..., description="Database username.", examples=["sa"])
+    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
+    driver: Optional[str] = Field(
+        None,
+        description="ODBC driver name. Leave empty to use the default driver.",
+        examples=["ODBC Driver 17 for SQL Server"],
+    )
+
+
+class SupabaseCredentials(BaseModel):
+    """Connection credentials for Supabase (PostgreSQL-based).
+
+    Use the connection string details from your Supabase project dashboard
+    under Settings > Database.
+    """
+
+    host: str = Field(
+        ...,
+        description="Supabase database host (e.g. db.<project-ref>.supabase.co, or the pooler host).",
+        examples=["db.xxxx.supabase.co"],
+    )
+    port: int = Field(
+        5432,
+        description="Port number. Use 5432 for direct connection, 6543 for the connection pooler.",
+        examples=[5432],
+    )
+    database: str = Field("postgres", description="Database name (always 'postgres' for Supabase).", examples=["postgres"])
+    username: str = Field(
+        ...,
+        description="Database user. Use 'postgres' for direct connection, or 'postgres.<project-ref>' for the pooler.",
+        examples=["postgres"],
+    )
+    password: str = Field(..., description="Database password (set in Supabase dashboard). Will be encrypted at rest.", examples=["s3cr3t!"])
+    ssl_mode: Literal["require", "verify-ca", "verify-full"] = Field(
+        "require",
+        description="SSL mode. Supabase always requires SSL.",
+        examples=["require"],
+    )
+
+
+class BigQueryCredentials(BaseModel):
+    """Connection credentials for Google BigQuery.
+
+    Requires a GCP Service Account with at least BigQuery Data Viewer
+    and BigQuery Job User roles.
+    """
+
+    project_id: str = Field(..., description="GCP project ID where the BigQuery dataset resides.", examples=["my-gcp-project"])
+    dataset_id: str = Field(..., description="BigQuery dataset name to connect to.", examples=["my_dataset"])
+    location: Optional[str] = Field(
+        "US",
+        description="Dataset location/region (default: US).",
+        examples=["US", "EU", "asia-southeast1"],
+    )
+    service_account_json: str = Field(
+        ...,
+        description=(
+            "Full content of the GCP Service Account key JSON file as a string. "
+            "Will be encrypted at rest."
+        ),
+        examples=['{"type":"service_account","project_id":"my-gcp-project","private_key_id":"..."}'],
+    )
+
+
+class SnowflakeCredentials(BaseModel):
+    """Connection credentials for Snowflake."""
+
+    account: str = Field(
+        ...,
+        description="Snowflake account identifier, including region if applicable (e.g. myaccount.us-east-1).",
+        examples=["myaccount.us-east-1"],
+    )
+    warehouse: str = Field(..., description="Name of the virtual warehouse to use for queries.", examples=["COMPUTE_WH"])
+    database: str = Field(..., description="Name of the target Snowflake database.", examples=["MY_DB"])
+    db_schema: Optional[str] = Field("PUBLIC", alias="schema", description="Schema name (default: PUBLIC).", examples=["PUBLIC"])
+    username: str = Field(..., description="Snowflake username.", examples=["admin"])
+    password: str = Field(..., description="Snowflake password. Will be encrypted at rest.", examples=["s3cr3t!"])
+    role: Optional[str] = Field(None, description="Snowflake role to assume for the session.", examples=["SYSADMIN"])
+
+
+# Union of all credential shapes — reserved for future typed validation on
+# DatabaseClientCreate.credentials (currently Dict[str, Any]). Kept exported
+# so downstream code can reference it without re-declaring.
+CredentialsUnion = Union[
+    PostgresCredentials,
+    MysqlCredentials,
+    SqlServerCredentials,
+    SupabaseCredentials,
+    BigQueryCredentials,
+    SnowflakeCredentials,
+]
+
+
+# Doc-only helper: surfaces per-type credential shapes in the Swagger "Schemas"
+# panel so API consumers can discover the exact field set for each db_type.
+# Not referenced by any endpoint — importing it in db_client.py is enough for
+# FastAPI's OpenAPI generator to pick it up.
+class CredentialSchemas(BaseModel):
+    """Reference schemas for `credentials` per `db_type` (Swagger-only, not used by endpoints)."""
+
+    postgres: PostgresCredentials
+    mysql: MysqlCredentials
+    sqlserver: SqlServerCredentials
+    supabase: SupabaseCredentials
+    bigquery: BigQueryCredentials
+    snowflake: SnowflakeCredentials

src/pipeline/db_pipeline/__init__.py

@@ -0,0 +1,3 @@
+from src.pipeline.db_pipeline.db_pipeline_service import DbPipelineService, db_pipeline_service
+
+__all__ = ["DbPipelineService", "db_pipeline_service"]

src/pipeline/db_pipeline/db_pipeline_service.py

@@ -0,0 +1,197 @@
+"""Service for ingesting a user's external database into the vector store.
+
+End-to-end flow: connect -> introspect schema -> profile columns -> build text
+-> embed + store in the shared PGVector collection (tagged with
+`source_type="database"`, retrievable via the same retriever used for docs).
+
+Sync DB work (SQLAlchemy inspect, pandas read_sql) runs in a threadpool;
+async vector writes stay on the event loop.
+"""
+
+import asyncio
+from contextlib import contextmanager
+from typing import Any, Iterator, Optional
+
+from langchain_core.documents import Document as LangChainDocument
+from sqlalchemy import URL, create_engine
+from sqlalchemy.engine import Engine
+
+from src.db.postgres.vector_store import get_vector_store
+from src.middlewares.logging import get_logger
+from src.models.credentials import DbType
+from src.pipeline.db_pipeline.extractor import get_schema, profile_table
+
+logger = get_logger("db_pipeline")
+
+
+class DbPipelineService:
+    """End-to-end DB ingestion: connect -> introspect -> profile -> embed -> store."""
+
+    def connect(self, db_type: DbType, credentials: dict[str, Any]) -> Engine:
+        """Build a SQLAlchemy engine for the user's database.
+
+        `credentials` is the plaintext dict matching the per-type schema in
+        `src/models/credentials.py`. BigQuery/Snowflake auth models differ
+        from host/port/user/pass, so every shape flows through one dict.
+
+        Optional driver imports (snowflake-sqlalchemy, json for BigQuery) are
+        done lazily so an env missing one driver doesn't break module import.
+        """
+        logger.info("connecting to user db", db_type=db_type)
+
+        if db_type in ("postgres", "supabase"):
+            query = (
+                {"sslmode": credentials["ssl_mode"]} if credentials.get("ssl_mode") else {}
+            )
+            url = URL.create(
+                drivername="postgresql+psycopg2",
+                username=credentials["username"],
+                password=credentials["password"],
+                host=credentials["host"],
+                port=credentials["port"],
+                database=credentials["database"],
+                query=query,
+            )
+            return create_engine(url)
+
+        if db_type == "mysql":
+            url = URL.create(
+                drivername="mysql+pymysql",
+                username=credentials["username"],
+                password=credentials["password"],
+                host=credentials["host"],
+                port=credentials["port"],
+                database=credentials["database"],
+            )
+            # pymysql only activates TLS when the `ssl` dict is truthy
+            # (empty dict is falsy and silently disables TLS). Use system-
+            # default CAs via certifi + hostname verification — required by
+            # managed MySQL providers like TiDB Cloud / PlanetScale / Aiven.
+            if credentials.get("ssl", True):
+                import certifi
+
+                connect_args = {
+                    "ssl": {
+                        "ca": certifi.where(),
+                        "check_hostname": True,
+                    }
+                }
+            else:
+                connect_args = {}
+            return create_engine(url, connect_args=connect_args)
+
+        if db_type == "sqlserver":
+            # `driver` applies to pyodbc only; we ship pymssql. Accept-and-ignore
+            # keeps the credential schema stable.
+            if credentials.get("driver"):
+                logger.info(
+                    "sqlserver driver hint ignored (using pymssql)",
+                    driver=credentials["driver"],
+                )
+            url = URL.create(
+                drivername="mssql+pymssql",
+                username=credentials["username"],
+                password=credentials["password"],
+                host=credentials["host"],
+                port=credentials["port"],
+                database=credentials["database"],
+            )
+            return create_engine(url)
+
+        if db_type == "bigquery":
+            import json
+
+            sa_info = json.loads(credentials["service_account_json"])
+            # sqlalchemy-bigquery URL shape: bigquery://<project>/<dataset>
+            url = f"bigquery://{credentials['project_id']}/{credentials['dataset_id']}"
+            return create_engine(
+                url,
+                credentials_info=sa_info,
+                location=credentials.get("location", "US"),
+            )
+
+        if db_type == "snowflake":
+            from snowflake.sqlalchemy import URL as SnowflakeURL
+
+            url = SnowflakeURL(
+                account=credentials["account"],
+                user=credentials["username"],
+                password=credentials["password"],
+                database=credentials["database"],
+                schema=(
+                    credentials.get("db_schema")
+                    or credentials.get("schema")
+                    or "PUBLIC"
+                ),
+                warehouse=credentials["warehouse"],
+                role=credentials.get("role") or "",
+            )
+            return create_engine(url)
+
+        raise ValueError(f"Unsupported db_type: {db_type}")
+
+    @contextmanager
+    def engine_scope(
+        self, db_type: DbType, credentials: dict[str, Any]
+    ) -> Iterator[Engine]:
+        """Yield a connected Engine and dispose its pool on exit.
+
+        API callers should prefer this over raw `connect(...)` so user DB
+        connection pools do not leak between pipeline runs.
+        """
+        engine = self.connect(db_type, credentials)
+        try:
+            yield engine
+        finally:
+            engine.dispose()
+
+    def _to_document(
+        self, user_id: str, table_name: str, entry: dict
+    ) -> LangChainDocument:
+        col = entry["col"]
+        return LangChainDocument(
+            page_content=entry["text"],
+            metadata={
+                "user_id": user_id,
+                "source_type": "database",
+                "data": {
+                    "table_name": table_name,
+                    "column_name": col["name"],
+                    "column_type": col["type"],
+                    "is_primary_key": col.get("is_primary_key", False),
+                    "foreign_key": col.get("foreign_key"),
+                },
+            },
+        )
+
+    async def run(
+        self,
+        user_id: str,
+        engine: Engine,
+        exclude_tables: Optional[frozenset[str]] = None,
+    ) -> int:
+        """Introspect the user's DB, profile columns, embed descriptions, store in PGVector.
+
+        Returns:
+            Total number of chunks ingested.
+        """
+        vector_store = get_vector_store()
+        logger.info("db pipeline start", user_id=user_id)
+
+        schema = await asyncio.to_thread(get_schema, engine, exclude_tables)
+
+        total = 0
+        for table_name, columns in schema.items():
+            logger.info("profiling table", table=table_name, columns=len(columns))
+            entries = await asyncio.to_thread(profile_table, engine, table_name, columns)
+            docs = [self._to_document(user_id, table_name, e) for e in entries]
+            if docs:
+                await vector_store.aadd_documents(docs)
+                total += len(docs)
+                logger.info("ingested chunks", table=table_name, count=len(docs))
+
+        logger.info("db pipeline complete", user_id=user_id, total=total)
+        return total
+
+
+db_pipeline_service = DbPipelineService()

src/pipeline/db_pipeline/extractor.py

@@ -0,0 +1,213 @@
+"""Schema introspection and per-column profiling for a user's database.
+
+Identifiers (table/column names) are quoted via the engine's dialect preparer,
+which handles reserved words, mixed case, and embedded quotes correctly across
+dialects. Values used in SQL come from SQLAlchemy inspection of the DB itself,
+not user input.
+"""
+
+from typing import Optional
+
+import pandas as pd
+from sqlalchemy import Float, Integer, Numeric, inspect
+from sqlalchemy.engine import Engine
+
+from src.middlewares.logging import get_logger
+
+logger = get_logger("db_extractor")
+
+TOP_VALUES_THRESHOLD = 0.05  # show top values if distinct_ratio <= 5%
+
+# Dialects where PERCENTILE_CONT(...) WITHIN GROUP is supported as an aggregate.
+# MySQL has no percentile aggregate; BigQuery has PERCENTILE_CONT only as an
+# analytic (window) function — both drop median and keep min/max/mean.
+_MEDIAN_DIALECTS = frozenset({"postgresql", "mssql", "snowflake"})
+
+
+def _supports_median(engine: Engine) -> bool:
+    return engine.dialect.name in _MEDIAN_DIALECTS
+
+
+def _head_query(
+    engine: Engine,
+    select_clause: str,
+    from_clause: str,
+    n: int,
+    order_by: str = "",
+) -> str:
+    """LIMIT/TOP-equivalent head query for the engine's dialect."""
+    if engine.dialect.name == "mssql":
+        return f"SELECT TOP {n} {select_clause} FROM {from_clause} {order_by}".strip()
+    return f"SELECT {select_clause} FROM {from_clause} {order_by} LIMIT {n}".strip()
+
+
+def _qi(engine: Engine, name: str) -> str:
+    """Dialect-correct identifier quoting (schema.table also handled if dotted)."""
+    preparer = engine.dialect.identifier_preparer
+    if "." in name:
+        schema, _, table = name.partition(".")
+        return f"{preparer.quote(schema)}.{preparer.quote(table)}"
+    return preparer.quote(name)
+
+
+def get_schema(
+    engine: Engine, exclude_tables: Optional[frozenset[str]] = None
+) -> dict[str, list[dict]]:
+    """Returns {table_name: [{name, type, is_numeric, is_primary_key, foreign_key}, ...]}."""
+    exclude = exclude_tables or frozenset()
+    inspector = inspect(engine)
+    schema = {}
+    for table_name in inspector.get_table_names():
+        if table_name in exclude:
+            continue
+
+        pk = inspector.get_pk_constraint(table_name)
+        pk_cols = set(pk["constrained_columns"]) if pk else set()
+
+        fk_map = {}
+        for fk in inspector.get_foreign_keys(table_name):
+            for col, ref_col in zip(fk["constrained_columns"], fk["referred_columns"]):
+                fk_map[col] = f"{fk['referred_table']}.{ref_col}"
+
+        cols = inspector.get_columns(table_name)
+        schema[table_name] = [
+            {
+                "name": c["name"],
+                "type": str(c["type"]),
+                "is_numeric": isinstance(c["type"], (Integer, Numeric, Float)),
+                "is_primary_key": c["name"] in pk_cols,
+                "foreign_key": fk_map.get(c["name"]),
+            }
+            for c in cols
+        ]
+    logger.info("extracted schema", table_count=len(schema))
+    return schema
+
+
+def get_row_count(engine: Engine, table_name: str) -> int:
+    return pd.read_sql(f"SELECT COUNT(*) FROM {_qi(engine, table_name)}", engine).iloc[0, 0]
+
+
+def profile_column(
+    engine: Engine,
+    table_name: str,
+    col_name: str,
+    is_numeric: bool,
+    row_count: int,
+) -> dict:
+    """Returns null_count, distinct_count, min/max, top values, and sample values."""
+    if row_count == 0:
+        return {
+            "null_count": 0,
+            "distinct_count": 0,
+            "distinct_ratio": 0.0,
+            "sample_values": [],
+        }
+
+    qt = _qi(engine, table_name)
+    qc = _qi(engine, col_name)
+
+    # Combined stats query: null_count, distinct_count, and min/max (if numeric).
+    # One round-trip instead of two.
+    select_cols = [
+        f"COUNT(*) - COUNT({qc}) AS nulls",
+        f"COUNT(DISTINCT {qc}) AS distincts",
+    ]
+    if is_numeric:
+        select_cols.append(f"MIN({qc}) AS min_val")
+        select_cols.append(f"MAX({qc}) AS max_val")
+        select_cols.append(f"AVG({qc}) AS mean_val")
+        if _supports_median(engine):
+            select_cols.append(
+                f"PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY {qc}) AS median_val"
+            )
+    stats = pd.read_sql(f"SELECT {', '.join(select_cols)} FROM {qt}", engine)
+
+    null_count = int(stats.iloc[0]["nulls"])
+    distinct_count = int(stats.iloc[0]["distincts"])
+    distinct_ratio = distinct_count / row_count if row_count > 0 else 0
+
+    profile = {
+        "null_count": null_count,
+        "distinct_count": distinct_count,
+        "distinct_ratio": round(distinct_ratio, 4),
+    }
+
+    if is_numeric:
+        profile["min"] = stats.iloc[0]["min_val"]
+        profile["max"] = stats.iloc[0]["max_val"]
+        profile["mean"] = stats.iloc[0]["mean_val"]
+        if _supports_median(engine):
+            profile["median"] = stats.iloc[0]["median_val"]
+
+    if 0 < distinct_ratio <= TOP_VALUES_THRESHOLD:
+        top_sql = _head_query(
+            engine,
+            select_clause=f"{qc}, COUNT(*) AS cnt",
+            from_clause=f"{qt} GROUP BY {qc}",
+            n=10,
+            order_by="ORDER BY cnt DESC",
+        )
+        top = pd.read_sql(top_sql, engine)
+        profile["top_values"] = list(zip(top[col_name].tolist(), top["cnt"].tolist()))
+
+    sample = pd.read_sql(_head_query(engine, qc, qt, 5), engine)
+    profile["sample_values"] = sample[col_name].tolist()
+
+    return profile
+
+
+def profile_table(engine: Engine, table_name: str, columns: list[dict]) -> list[dict]:
+    """Profile every column in a table. Returns [{col, profile, text}, ...].
+
+    Per-column errors are logged and skipped so one bad column doesn't abort
+    the whole table.
+    """
+    row_count = get_row_count(engine, table_name)
+    if row_count == 0:
+        logger.info("skipping empty table", table=table_name)
+        return []
+
+    results = []
+    for col in columns:
+        try:
+            profile = profile_column(
+                engine, table_name, col["name"], col.get("is_numeric", False), row_count
+            )
+            text = build_text(table_name, row_count, col, profile)
+            results.append({"col": col, "profile": profile, "text": text})
+        except Exception as e:
+            logger.error(
+                "column profiling failed",
+                table=table_name,
+                column=col["name"],
+                error=str(e),
+            )
+            continue
+    return results
+
+
+def build_text(table_name: str, row_count: int, col: dict, profile: dict) -> str:
+    col_name = col["name"]
+    col_type = col["type"]
+
+    key_label = ""
+    if col.get("is_primary_key"):
+        key_label = " [PRIMARY KEY]"
+    elif col.get("foreign_key"):
+        key_label = f" [FK -> {col['foreign_key']}]"
+
+    text = f"Table: {table_name} ({row_count} rows)\n"
+    text += f"Column: {col_name} ({col_type}){key_label}\n"
+    text += f"Null count: {profile['null_count']}\n"
+    text += f"Distinct count: {profile['distinct_count']} ({profile['distinct_ratio']:.1%})\n"
+    if "min" in profile:
+        text += f"Min: {profile['min']}, Max: {profile['max']}\n"
+        text += f"Mean: {profile['mean']}\n"
+        if profile.get("median") is not None:
+            text += f"Median: {profile['median']}\n"
+    if "top_values" in profile:
+        top_str = ", ".join(f"{v} ({c})" for v, c in profile["top_values"])
+        text += f"Top values: {top_str}\n"
+    text += f"Sample values: {profile['sample_values']}"
+    return text

src/pipeline/document_pipeline/document_pipeline.py

@@ -0,0 +1,80 @@
+"""Document upload and processing pipeline."""
+
+from fastapi import HTTPException, UploadFile
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.document.document_service import document_service
+from src.knowledge.processing_service import knowledge_processor
+from src.middlewares.logging import get_logger
+from src.storage.az_blob.az_blob import blob_storage
+
+logger = get_logger("document_pipeline")
+
+SUPPORTED_FILE_TYPES = ["pdf", "docx", "txt", "csv", "xlsx"]
+
+
+class DocumentPipeline:
+    """Orchestrates the full document upload, process, and delete flows."""
+
+    async def upload(self, file: UploadFile, user_id: str, db: AsyncSession) -> dict:
+        """Validate → upload to blob → save to DB."""
+        content = await file.read()
+        file_type = file.filename.split(".")[-1].lower() if "." in file.filename else "txt"
+
+        if file_type not in SUPPORTED_FILE_TYPES:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Unsupported file type. Supported: {SUPPORTED_FILE_TYPES}",
+            )
+
+        blob_name = await blob_storage.upload_file(content, file.filename, user_id)
+        document = await document_service.create_document(
+            db=db,
+            user_id=user_id,
+            filename=file.filename,
+            blob_name=blob_name,
+            file_size=len(content),
+            file_type=file_type,
+        )
+
+        logger.info(f"Uploaded document {document.id} for user {user_id}")
+        return {"id": document.id, "filename": document.filename, "status": document.status}
+
+    async def process(self, document_id: str, user_id: str, db: AsyncSession) -> dict:
+        """Validate ownership → extract text → chunk → ingest to vector store."""
+        document = await document_service.get_document(db, document_id)
+
+        if not document:
+            raise HTTPException(status_code=404, detail="Document not found")
+        if document.user_id != user_id:
+            raise HTTPException(status_code=403, detail="Access denied")
+
+        try:
+            await document_service.update_document_status(db, document_id, "processing")
+            chunks_count = await knowledge_processor.process_document(document, db)
+            await document_service.update_document_status(db, document_id, "completed")
+
+            logger.info(f"Processed document {document_id}: {chunks_count} chunks")
+            return {"document_id": document_id, "chunks_processed": chunks_count}
+
+        except Exception as e:
+            logger.error(f"Processing failed for document {document_id}", error=str(e))
+            await document_service.update_document_status(db, document_id, "failed", str(e))
+            raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}")
+
+    async def delete(self, document_id: str, user_id: str, db: AsyncSession) -> dict:
+        """Validate ownership → delete from blob and DB."""
+        document = await document_service.get_document(db, document_id)
+
+        if not document:
+            raise HTTPException(status_code=404, detail="Document not found")
+        if document.user_id != user_id:
+            raise HTTPException(status_code=403, detail="Access denied")
+
+        await document_service.delete_document(db, document_id)
+
+        logger.info(f"Deleted document {document_id} for user {user_id}")
+        return {"document_id": document_id}
+
+
+document_pipeline = DocumentPipeline()

src/utils/db_credential_encryption.py

@@ -0,0 +1,70 @@
+"""Fernet encryption utilities for user-registered database credentials.
+
+Encryption key is sourced from `dataeyond__db__credential__key` env variable,
+intentionally separate from the user-auth bcrypt salt (`emarcal__bcrypt__salt`).
+
+Usage:
+    from src.utils.db_credential_encryption import encrypt_credentials_dict, decrypt_credentials_dict
+
+    # Before INSERT:
+    safe_creds = encrypt_credentials_dict(raw_credentials)
+
+    # After SELECT:
+    plain_creds = decrypt_credentials_dict(row.credentials)
+"""
+
+from cryptography.fernet import Fernet
+from src.config.settings import settings
+
+# Sensitive credential field names that must be encrypted at rest.
+# Covers all supported DB types:
+#   - password      : postgres, mysql, sqlserver, supabase, snowflake
+#   - service_account_json : bigquery
+SENSITIVE_FIELDS: frozenset[str] = frozenset({"password", "service_account_json"})
+
+
+def _get_cipher() -> Fernet:
+    key = settings.dataeyond_db_credential_key
+    if not key:
+        raise ValueError(
+            "dataeyond__db__credential__key is not set. "
+            "Generate one with: Fernet.generate_key().decode()"
+        )
+    return Fernet(key.encode())
+
+
+def encrypt_credential(value: str) -> str:
+    """Encrypt a single credential string value."""
+    return _get_cipher().encrypt(value.encode()).decode()
+
+
+def decrypt_credential(value: str) -> str:
+    """Decrypt a single Fernet-encrypted credential string."""
+    return _get_cipher().decrypt(value.encode()).decode()
+
+
+def encrypt_credentials_dict(creds: dict) -> dict:
+    """Return a copy of the credentials dict with sensitive fields encrypted.
+
+    Call this before inserting a new DatabaseClient record.
+    """
+    cipher = _get_cipher()
+    result = dict(creds)
+    for field in SENSITIVE_FIELDS:
+        if result.get(field):
+            result[field] = cipher.encrypt(result[field].encode()).decode()
+    return result
+
+
+def decrypt_credentials_dict(creds: dict) -> dict:
+    """Return a copy of the credentials dict with sensitive fields decrypted.
+
+    Call this after fetching a DatabaseClient record from DB.
+    """
+    cipher = _get_cipher()
+    result = dict(creds)
+    for field in SENSITIVE_FIELDS:
+        if result.get(field):
+            result[field] = cipher.decrypt(result[field].encode()).decode()
+    return result
+ 

uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = "==3.12.*"
 resolution-markers = [
     "python_full_version >= '3.12.4'",
@@ -49,6 +49,8 @@ dependencies = [
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "pymongo" },
+    { name = "pymssql" },
+    { name = "pymysql" },
     { name = "pypdf" },
     { name = "python-docx" },
     { name = "python-dotenv" },
@@ -57,8 +59,10 @@ dependencies = [
     { name = "redis" },
     { name = "sentence-transformers" },
     { name = "slowapi" },
+    { name = "snowflake-sqlalchemy" },
     { name = "spacy" },
     { name = "sqlalchemy", extra = ["asyncio"] },
+    { name = "sqlalchemy-bigquery" },
     { name = "sse-starlette" },
     { name = "starlette" },
     { name = "structlog" },
@@ -131,6 +135,8 @@ requires-dist = [
     { name = "pydantic", specifier = "==2.10.3" },
     { name = "pydantic-settings", specifier = "==2.7.0" },
     { name = "pymongo", specifier = ">=4.14.0" },
+    { name = "pymssql", specifier = ">=2.3.0" },
+    { name = "pymysql", specifier = ">=1.1.1" },
     { name = "pypdf", specifier = "==5.1.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = "==8.3.4" },
     { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = "==0.24.0" },
@@ -143,8 +149,10 @@ requires-dist = [
     { name = "ruff", marker = "extra == 'dev'", specifier = "==0.8.4" },
     { name = "sentence-transformers", specifier = "==3.3.1" },
     { name = "slowapi", specifier = "==0.1.9" },
+    { name = "snowflake-sqlalchemy", specifier = ">=1.7.0" },
     { name = "spacy", specifier = "==3.8.3" },
     { name = "sqlalchemy", extras = ["asyncio"], specifier = "==2.0.36" },
+    { name = "sqlalchemy-bigquery", specifier = ">=1.11.0" },
     { name = "sse-starlette", specifier = "==2.1.3" },
     { name = "starlette", specifier = "==0.41.3" },
     { name = "structlog", specifier = "==24.4.0" },
@@ -280,6 +288,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/13/b5/7af0cb920a476dccd612fbc9a21a3745fb29b1fcd74636078db8f7ba294c/APScheduler-3.10.4-py3-none-any.whl", hash = "sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661", size = 59303, upload-time = "2023-08-19T16:44:56.814Z" },
 ]
 
+[[package]]
+name = "asn1crypto"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" },
+]
+
 [[package]]
 name = "asyncpg"
 version = "0.30.0"
@@ -428,6 +445,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/07/fb43edc2ff0a6a367e4a94fc39eb3b85aa1e55e24cc857af2db145ce9f0d/blis-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f20f7ad69aaffd1ce14fe77de557b6df9b61e0c9e582f75a843715d836b5c8af", size = 6192759, upload-time = "2025-11-17T12:27:56.176Z" },
 ]
 
+[[package]]
+name = "boto3"
+version = "1.42.89"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+    { name = "jmespath" },
+    { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bb/0c/f7bccb22b245cabf392816baba20f9e95f78ace7dbc580fd40136e80e732/boto3-1.42.89.tar.gz", hash = "sha256:3e43aacc0801bba9bcd23a8c271c089af297a69565f783fcdd357ae0e330bf1e", size = 113165, upload-time = "2026-04-13T19:36:17.516Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b9/33/55103ba5ef9975ea54b8d39e69b76eb6e9fded3beae5f01065e26951a3a1/boto3-1.42.89-py3-none-any.whl", hash = "sha256:6204b189f4d0c655535f43d7eaa57ff4e8d965b8463c97e45952291211162932", size = 140556, upload-time = "2026-04-13T19:36:13.894Z" },
+]
+
+[[package]]
+name = "botocore"
+version = "1.42.89"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "jmespath" },
+    { name = "python-dateutil" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0f/cc/e6be943efa9051bd15c2ee14077c2b10d6e27c9e9385fc43a03a5c4ed8b5/botocore-1.42.89.tar.gz", hash = "sha256:95ac52f472dad29942f3088b278ab493044516c16dbf9133c975af16527baa99", size = 15206290, upload-time = "2026-04-13T19:36:02.321Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/91/f1/90a7b8eda38b7c3a65ca7ee0075bdf310b6b471cb1b95fab6e8994323a50/botocore-1.42.89-py3-none-any.whl", hash = "sha256:d9b786c8d9db6473063b4cc5be0ba7e6a381082307bd6afb69d4216f9fa95f35", size = 14887287, upload-time = "2026-04-13T19:35:56.677Z" },
+]
+
 [[package]]
 name = "cachetools"
 version = "5.5.0"
@@ -941,6 +986,109 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" },
 ]
 
+[[package]]
+name = "google-api-core"
+version = "2.30.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-auth" },
+    { name = "googleapis-common-protos" },
+    { name = "proto-plus" },
+    { name = "protobuf" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" },
+]
+
+[package.optional-dependencies]
+grpc = [
+    { name = "grpcio" },
+    { name = "grpcio-status" },
+]
+
+[[package]]
+name = "google-auth"
+version = "2.49.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cryptography" },
+    { name = "pyasn1-modules" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" },
+]
+
+[[package]]
+name = "google-cloud-bigquery"
+version = "3.41.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-api-core", extra = ["grpc"] },
+    { name = "google-auth" },
+    { name = "google-cloud-core" },
+    { name = "google-resumable-media" },
+    { name = "packaging" },
+    { name = "python-dateutil" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ce/13/6515c7aab55a4a0cf708ffd309fb9af5bab54c13e32dc22c5acd6497193c/google_cloud_bigquery-3.41.0.tar.gz", hash = "sha256:2217e488b47ed576360c9b2cc07d59d883a54b83167c0ef37f915c26b01a06fe", size = 513434, upload-time = "2026-03-30T22:50:55.347Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/40/33/1d3902efadef9194566d499d61507e1f038454e0b55499d2d7f8ab2a4fee/google_cloud_bigquery-3.41.0-py3-none-any.whl", hash = "sha256:2a5b5a737b401cbd824a6e5eac7554100b878668d908e6548836b5d8aaa4dcaa", size = 262343, upload-time = "2026-03-30T22:48:45.444Z" },
+]
+
+[[package]]
+name = "google-cloud-core"
+version = "2.5.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-api-core" },
+    { name = "google-auth" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dc/24/6ca08b0a03c7b0c620427503ab00353a4ae806b848b93bcea18b6b76fde6/google_cloud_core-2.5.1.tar.gz", hash = "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811", size = 36078, upload-time = "2026-03-30T22:50:08.057Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/73/d9/5bb050cb32826466aa9b25f79e2ca2879fe66cb76782d4ed798dd7506151/google_cloud_core-2.5.1-py3-none-any.whl", hash = "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7", size = 29452, upload-time = "2026-03-30T22:48:31.567Z" },
+]
+
+[[package]]
+name = "google-crc32c"
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" },
+    { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" },
+    { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" },
+    { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" },
+]
+
+[[package]]
+name = "google-resumable-media"
+version = "2.8.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-crc32c" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3f/d1/b1ea14b93b6b78f57fc580125de44e9f593ab88dd2460f1a8a8d18f74754/google_resumable_media-2.8.2.tar.gz", hash = "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70", size = 2164510, upload-time = "2026-03-30T23:34:25.369Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5e/f8/50bfaf4658431ff9de45c5c3935af7ab01157a4903c603cd0eee6e78e087/google_resumable_media-2.8.2-py3-none-any.whl", hash = "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220", size = 81511, upload-time = "2026-03-30T23:34:09.671Z" },
+]
+
+[[package]]
+name = "googleapis-common-protos"
+version = "1.74.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" },
+]
+
 [[package]]
 name = "greenlet"
 version = "3.3.2"
@@ -958,6 +1106,41 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/58/2e/fe7f36ff1982d6b10a60d5e0740c759259a7d6d2e1dc41da6d96de32fff6/greenlet-3.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:d3a62fa76a32b462a97198e4c9e99afb9ab375115e74e9a83ce180e7a496f643", size = 230331, upload-time = "2026-02-20T20:17:23.34Z" },
 ]
 
+[[package]]
+name = "grpcio"
+version = "1.80.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5c/e8/a2b749265eb3415abc94f2e619bbd9e9707bebdda787e61c593004ec927a/grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0", size = 6015616, upload-time = "2026-03-30T08:47:13.428Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/97/b1282161a15d699d1e90c360df18d19165a045ce1c343c7f313f5e8a0b77/grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2", size = 12014204, upload-time = "2026-03-30T08:47:15.873Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/5e/d319c6e997b50c155ac5a8cb12f5173d5b42677510e886d250d50264949d/grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de", size = 6563866, upload-time = "2026-03-30T08:47:18.588Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/f6/fdd975a2cb4d78eb67769a7b3b3830970bfa2e919f1decf724ae4445f42c/grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921", size = 7273060, upload-time = "2026-03-30T08:47:21.113Z" },
+    { url = "https://files.pythonhosted.org/packages/db/f0/a3deb5feba60d9538a962913e37bd2e69a195f1c3376a3dd44fe0427e996/grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411", size = 6782121, upload-time = "2026-03-30T08:47:23.827Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/84/36c6dcfddc093e108141f757c407902a05085e0c328007cb090d56646cdf/grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd", size = 7383811, upload-time = "2026-03-30T08:47:26.517Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/ef/f3a77e3dc5b471a0ec86c564c98d6adfa3510d38f8ee99010410858d591e/grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f", size = 8393860, upload-time = "2026-03-30T08:47:29.439Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/8d/9d4d27ed7f33d109c50d6b5ce578a9914aa68edab75d65869a17e630a8d1/grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f", size = 7830132, upload-time = "2026-03-30T08:47:33.254Z" },
+    { url = "https://files.pythonhosted.org/packages/14/e4/9990b41c6d7a44e1e9dee8ac11d7a9802ba1378b40d77468a7761d1ad288/grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193", size = 4140904, upload-time = "2026-03-30T08:47:35.319Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/2c/296f6138caca1f4b92a31ace4ae1b87dab692fc16a7a3417af3bb3c805bf/grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff", size = 4880944, upload-time = "2026-03-30T08:47:37.831Z" },
+]
+
+[[package]]
+name = "grpcio-status"
+version = "1.80.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "googleapis-common-protos" },
+    { name = "grpcio" },
+    { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -1127,6 +1310,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" },
 ]
 
+[[package]]
+name = "jmespath"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
+]
+
 [[package]]
 name = "joblib"
 version = "1.5.3"
@@ -2121,6 +2313,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" },
 ]
 
+[[package]]
+name = "proto-plus"
+version = "1.27.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/81/0d/94dfe80193e79d55258345901acd2917523d56e8381bc4dee7fd38e3868a/proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24", size = 57204, upload-time = "2026-03-26T22:18:57.174Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/84/f3/1fba73eeffafc998a25d59703b63f8be4fe8a5cb12eaff7386a0ba0f7125/proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", size = 50450, upload-time = "2026-03-26T22:13:42.927Z" },
+]
+
+[[package]]
+name = "protobuf"
+version = "6.33.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" },
+    { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" },
+    { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },
+]
+
 [[package]]
 name = "psycopg"
 version = "3.2.3"
@@ -2181,6 +2400,27 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b5/bf/635fbe5dd10ed200afbbfbe98f8602829252ca1cce81cc48fb25ed8dadc0/psycopg2-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:e03e4a6dbe87ff81540b434f2e5dc2bddad10296db5eea7bdc995bf5f4162938", size = 2713969, upload-time = "2025-10-10T11:10:15.946Z" },
 ]
 
+[[package]]
+name = "pyasn1"
+version = "0.6.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" },
+]
+
+[[package]]
+name = "pyasn1-modules"
+version = "0.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pyasn1" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" },
+]
+
 [[package]]
 name = "pycparser"
 version = "3.0"
@@ -2310,6 +2550,43 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/60/4c/33f75713d50d5247f2258405142c0318ff32c6f8976171c4fcae87a9dbdf/pymongo-4.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:dfc320f08ea9a7ec5b2403dc4e8150636f0d6150f4b9792faaae539c88e7db3b", size = 892971, upload-time = "2026-01-07T18:04:35.594Z" },
 ]
 
+[[package]]
+name = "pymssql"
+version = "2.3.13"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7a/cc/843c044b7f71ee329436b7327c578383e2f2499313899f88ad267cdf1f33/pymssql-2.3.13.tar.gz", hash = "sha256:2137e904b1a65546be4ccb96730a391fcd5a85aab8a0632721feb5d7e39cfbce", size = 203153, upload-time = "2026-02-14T05:00:36.865Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ba/60/a2e8a8a38f7be21d54402e2b3365cd56f1761ce9f2706c97f864e8aa8300/pymssql-2.3.13-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cf4f32b4a05b66f02cb7d55a0f3bcb0574a6f8cf0bee4bea6f7b104038364733", size = 3158689, upload-time = "2026-02-14T04:59:46.982Z" },
+    { url = "https://files.pythonhosted.org/packages/43/9e/0cf0ffb9e2f73238baf766d8e31d7237b5bee3cc1bb29a376b404610994a/pymssql-2.3.13-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:2b056eb175955f7fb715b60dc1c0c624969f4d24dbdcf804b41ab1e640a2b131", size = 2960018, upload-time = "2026-02-14T04:59:48.668Z" },
+    { url = "https://files.pythonhosted.org/packages/93/ea/bc27354feaca717faa4626911f6b19bb62985c87dda28957c63de4de5895/pymssql-2.3.13-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:319810b89aa64b99d9c5c01518752c813938df230496fa2c4c6dda0603f04c4c", size = 3065719, upload-time = "2026-02-14T04:59:50.369Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/7a/8028681c96241fb5fc850b87c8959402c353e4b83c6e049a99ffa67ded54/pymssql-2.3.13-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0ea72641cb0f8bce7ad8565dbdbda4a7437aa58bce045f2a3a788d71af2e4be", size = 3190567, upload-time = "2026-02-14T04:59:52.202Z" },
+    { url = "https://files.pythonhosted.org/packages/aa/f1/ab5b76adbbd6db9ce746d448db34b044683522e7e7b95053f9dd0165297b/pymssql-2.3.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1493f63d213607f708a5722aa230776ada726ccdb94097fab090a1717a2534e0", size = 3710481, upload-time = "2026-02-14T04:59:54.01Z" },
+    { url = "https://files.pythonhosted.org/packages/59/aa/2fa0951475cd0a1829e0b8bfbe334d04ece4bce11546a556b005c4100689/pymssql-2.3.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb3275985c23479e952d6462ae6c8b2b6993ab6b99a92805a9c17942cf3d5b3d", size = 3453789, upload-time = "2026-02-14T04:59:56.841Z" },
+    { url = "https://files.pythonhosted.org/packages/78/08/8cd2af9003f9fc03912b658a64f5a4919dcd68f0dd3bbc822b49a3d14fd9/pymssql-2.3.13-cp312-cp312-win_amd64.whl", hash = "sha256:a930adda87bdd8351a5637cf73d6491936f34e525a5e513068a6eac742f69cdb", size = 1994709, upload-time = "2026-02-14T04:59:58.972Z" },
+]
+
+[[package]]
+name = "pymysql"
+version = "1.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/ae/1fe3fcd9f959efa0ebe200b8de88b5a5ce3e767e38c7ac32fb179f16a388/pymysql-1.1.2.tar.gz", hash = "sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03", size = 48258, upload-time = "2025-08-24T12:55:55.146Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" },
+]
+
+[[package]]
+name = "pyopenssl"
+version = "25.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cryptography" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/04/8c/cd89ad05804f8e3c17dea8f178c3f40eeab5694c30e0c9f5bcd49f576fc3/pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b", size = 179937, upload-time = "2025-05-17T16:28:31.31Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/80/28/2659c02301b9500751f8d42f9a6632e1508aa5120de5e43042b8b30f8d5d/pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", size = 56771, upload-time = "2025-05-17T16:28:29.197Z" },
+]
+
 [[package]]
 name = "pyparsing"
 version = "3.3.2"
@@ -2610,6 +2887,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/13/9f/026e18ca7d7766783d779dae5e9c656746c6ede36ef73c6d934aaf4a6dec/ruff-0.8.4-py3-none-win_arm64.whl", hash = "sha256:9183dd615d8df50defa8b1d9a074053891ba39025cf5ae88e8bcb52edcc4bf08", size = 9074500, upload-time = "2024-12-19T13:36:23.92Z" },
 ]
 
+[[package]]
+name = "s3transfer"
+version = "0.16.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" },
+]
+
 [[package]]
 name = "safetensors"
 version = "0.7.0"
@@ -2764,6 +3053,60 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
 ]
 
+[[package]]
+name = "snowflake-connector-python"
+version = "4.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "asn1crypto" },
+    { name = "boto3" },
+    { name = "botocore" },
+    { name = "certifi" },
+    { name = "charset-normalizer" },
+    { name = "cryptography" },
+    { name = "filelock" },
+    { name = "idna" },
+    { name = "packaging" },
+    { name = "platformdirs" },
+    { name = "pyjwt" },
+    { name = "pyopenssl" },
+    { name = "pytz" },
+    { name = "requests" },
+    { name = "sortedcontainers" },
+    { name = "tomlkit" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/4aff125021a9c5e0183f2f55dd7d04b7256a0e1e10db50d537a7415d9c55/snowflake_connector_python-4.0.0.tar.gz", hash = "sha256:4b10a865c4a5e1fa60c365c7fe41e0433605e6e5edc824e8730a9038f330b3a6", size = 813937, upload-time = "2025-10-09T10:11:34.631Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ea/b0/462c0deee35d6d03d3d729b3f923615bae665beb7f9a94673a23a52080fe/snowflake_connector_python-4.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bfd3b8523d7adc830f99c5c4c635689ceca61700a05368d5bbb34c6811f2ec54", size = 1029568, upload-time = "2025-10-09T10:11:42.125Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/4b/bb3ae3f07e7927c8f16c4c0f1283d3c721978d16e8bf4193fc8e41025c1e/snowflake_connector_python-4.0.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:835161dd46ef8f5fc9d2f135ca654c2f3fbdf57b035d3e1980506aa8eac671dc", size = 1041337, upload-time = "2025-10-09T10:11:43.692Z" },
+    { url = "https://files.pythonhosted.org/packages/9c/75/4bfac89f10c6dbb75e97adf1e217737fc599ebf964031c9298b6cbd807d0/snowflake_connector_python-4.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65e4e36dd1b0c7235d84cddef8a3c97c5ea0dc8fea85e31e45fc485000b77a83", size = 2699730, upload-time = "2025-10-09T10:11:25.295Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/78/0e916416c50909dbae511fe38b1e671a9efa62decdce51b174a0396804e4/snowflake_connector_python-4.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6132986d6965e4005b0167270612fbc7fa4bc4ef42726a40b85a8f57475a78d", size = 2731336, upload-time = "2025-10-09T10:11:27.028Z" },
+    { url = "https://files.pythonhosted.org/packages/83/f0/3db8a2f3f5ee724d309c661af739a70d0643070b9b4597728151ef900f9b/snowflake_connector_python-4.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:a790f06808e4481c23cfed1396d2c9a786060ddd62408b1fda1a63e1e6bc4b07", size = 1176292, upload-time = "2025-10-09T10:11:54.956Z" },
+]
+
+[[package]]
+name = "snowflake-sqlalchemy"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "snowflake-connector-python" },
+    { name = "sqlalchemy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ff/6a/fcc5c00c3a253029a7b7b293a3958ba07d5e97623b643de47be0cc9e5530/snowflake_sqlalchemy-1.9.0.tar.gz", hash = "sha256:fb32baf559f7f933ae8fde2ec535bcea5381bb15188777cd8c006b3226efa3b1", size = 141707, upload-time = "2026-03-04T13:48:17.905Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/28/b7ae8df80847e8157b74669ad7e1b0180e82ac0e3daf950612effd232fea/snowflake_sqlalchemy-1.9.0-py3-none-any.whl", hash = "sha256:f0b1528173e93c8c80bd9ca510985054667e0e514dd90b890271ac1cfae261c1", size = 78953, upload-time = "2026-03-04T13:48:16.393Z" },
+]
+
+[[package]]
+name = "sortedcontainers"
+version = "2.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
+]
+
 [[package]]
 name = "spacy"
 version = "3.8.3"
@@ -2842,6 +3185,22 @@ asyncio = [
     { name = "greenlet" },
 ]
 
+[[package]]
+name = "sqlalchemy-bigquery"
+version = "1.16.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "google-api-core" },
+    { name = "google-auth" },
+    { name = "google-cloud-bigquery" },
+    { name = "packaging" },
+    { name = "sqlalchemy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7e/6a/c49932b3d9c44cab9202b1866c5b36b7f0d0455d4653fbc0af4466aeaa76/sqlalchemy_bigquery-1.16.0.tar.gz", hash = "sha256:fe937a0d1f4cf7219fcf5d4995c6718805b38d4df43e29398dec5dc7b6d1987e", size = 119632, upload-time = "2025-11-06T01:35:40.373Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/87/11e6de00ef7949bb8ea06b55304a1a4911c329fdf0d9882b464db240c2c5/sqlalchemy_bigquery-1.16.0-py3-none-any.whl", hash = "sha256:0fe7634cd954f3e74f5e2db6d159f9e5ee87a47fbe8d52eac3cd3bb3dadb3a77", size = 40615, upload-time = "2025-11-06T01:35:39.358Z" },
+]
+
 [[package]]
 name = "srsly"
 version = "2.5.3"
@@ -3015,6 +3374,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" },
 ]
 
+[[package]]
+name = "tomlkit"
+version = "0.14.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" },
+]
+
 [[package]]
 name = "torch"
 version = "2.11.0"