[KM-437][DB] db pipeline+ metadata

.gitattributes

@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-software/** filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -26,8 +26,6 @@ test/users/user_accounts.csv
 .env.prd
 .env.example
 
-CLAUDE.md
-
 erd/
 playground/
 playground_retriever.py
@@ -35,5 +33,4 @@ playground_chat.py
 playground_flush_cache.py
 playground_create_user.py
 API_CONTRACT.md
-context_engineering/
-sample_file/
+context_engineering/

main.py

@@ -10,7 +10,6 @@ from src.api.v1.chat import router as chat_router
 from src.api.v1.room import router as room_router
 from src.api.v1.users import router as users_router
 from src.api.v1.knowledge import router as knowledge_router
-from src.api.v1.db_client import router as db_client_router
 from src.db.postgres.init_db import init_db
 import uvicorn
 
@@ -36,7 +35,6 @@ app.include_router(document_router)
 app.include_router(knowledge_router)
 app.include_router(room_router)
 app.include_router(chat_router)
-app.include_router(db_client_router)
 
 
 @app.on_event("startup")

pyproject.toml

@@ -79,13 +79,12 @@ dependencies = [
     "jsonpatch>=1.33",
     "pymongo>=4.14.0",
     "psycopg2>=2.9.11",
-    "pdf2image>=1.17.0",
-    "pytesseract>=0.3.13",
-    "pypdf2>=3.0.1",
-    "pdf2image>=1.17.0",
-    "pytesseract>=0.3.13",
-    "pypdf2>=3.0.1",
-    "pdf24.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest==8.3.4",
+    "pytest-asyncio==0.24.0",
     "pytest-cov==6.0.0",
     "httpx==0.28.1",
     "ruff==0.8.4",

src/api/v1/db_client.py

@@ -1,357 +1,5 @@
-"""API endpoints for user-registered database connections.
+from typing import Literal, Dict
 
-Credential schemas (DbType, PostgresCredentials, etc.) live in
-`src/models/credentials.py` — they are imported below (with noqa: F401) so
-FastAPI/Swagger picks them up for OpenAPI schema generation even though they
-are not referenced by name in this file.
-"""
 
-from typing import Any, Dict, List, Literal, Optional
-from datetime import datetime
-
-from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
-from pydantic import BaseModel, Field
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from src.database_client.database_client_service import database_client_service
-from src.db.postgres.connection import get_db
-from src.middlewares.logging import get_logger, log_execution
-from src.middlewares.rate_limit import limiter
-from src.models.credentials import (  # noqa: F401 — re-exported for Swagger schema discovery
-    BigQueryCredentials,
-    CredentialSchemas,
-    DbType,
-    MysqlCredentials,
-    PostgresCredentials,
-    SnowflakeCredentials,
-    SqlServerCredentials,
-    SupabaseCredentials,
-)
-from src.pipeline.db_pipeline import db_pipeline_service
-from src.utils.db_credential_encryption import decrypt_credentials_dict
-
-logger = get_logger("database_client_api")
-
-router = APIRouter(prefix="/api/v1", tags=["Database Clients"])
-
-
-# ---------------------------------------------------------------------------
-# Request / Response schemas
-# ---------------------------------------------------------------------------
-
-
-class DatabaseClientCreate(BaseModel):
-    """
-    Payload to register a new external database connection.
-
-    The `credentials` object shape depends on `db_type`:
-
-    | db_type     | Required fields                                          |
-    |-------------|----------------------------------------------------------|
-    | postgres    | host, port, database, username, password, ssl_mode       |
-    | mysql       | host, port, database, username, password, ssl            |
-    | sqlserver   | host, port, database, username, password, driver?        |
-    | supabase    | host, port, database, username, password, ssl_mode       |
-    | bigquery    | project_id, dataset_id, location?, service_account_json  |
-    | snowflake   | account, warehouse, database, schema?, username, password, role? |
-
-    Sensitive fields (`password`, `service_account_json`) are encrypted
-    at rest using Fernet symmetric encryption.
-    """
-
-    name: str = Field(..., description="Display name for this connection.", examples=["Production DB"])
-    db_type: DbType = Field(..., description="Type of the database engine.", examples=["postgres"])
-    credentials: Dict[str, Any] = Field(
-        ...,
-        description="Connection credentials. Shape depends on db_type. See schema descriptions above.",
-        examples=[
-            {
-                "host": "db.example.com",
-                "port": 5432,
-                "database": "mydb",
-                "username": "admin",
-                "password": "s3cr3t!",
-                "ssl_mode": "require",
-            }
-        ],
-    )
-
-
-class DatabaseClientUpdate(BaseModel):
-    """
-    Payload to update an existing database connection.
-
-    All fields are optional — only provided fields will be updated.
-    If `credentials` is provided, it replaces the entire credentials object
-    and sensitive fields are re-encrypted.
-    """
-
-    name: Optional[str] = Field(None, description="New display name for this connection.", examples=["Staging DB"])
-    credentials: Optional[Dict[str, Any]] = Field(
-        None,
-        description="Updated credentials object. Replaces existing credentials entirely if provided.",
-        examples=[{"host": "new-host.example.com", "port": 5432, "database": "mydb", "username": "admin", "password": "n3wP@ss!", "ssl_mode": "require"}],
-    )
-    status: Optional[Literal["active", "inactive"]] = Field(
-        None,
-        description="Set to 'inactive' to soft-disable the connection without deleting it.",
-        examples=["inactive"],
-    )
-
-
-class DatabaseClientResponse(BaseModel):
-    """
-    Database connection record returned by the API.
-
-    Credentials are **never** included in the response for security reasons.
-    """
-
-    id: str = Field(..., description="Unique identifier of the database connection.")
-    user_id: str = Field(..., description="ID of the user who owns this connection.")
-    name: str = Field(..., description="Display name of the connection.")
-    db_type: str = Field(..., description="Database engine type.")
-    status: str = Field(..., description="Connection status: 'active' or 'inactive'.")
-    created_at: datetime = Field(..., description="Timestamp when the connection was registered.")
-    updated_at: Optional[datetime] = Field(None, description="Timestamp of the last update, if any.")
-
-    model_config = {"from_attributes": True}
-
-
-# ---------------------------------------------------------------------------
-# Endpoints
-# ---------------------------------------------------------------------------
-
-
-@router.post(
-    "/database-clients",
-    response_model=DatabaseClientResponse,
-    status_code=status.HTTP_201_CREATED,
-    summary="Register a new database connection",
-    response_description="The newly created database connection record (credentials excluded).",
-    responses={
-        201: {"description": "Connection registered successfully."},
-        422: {"description": "Validation error — check the credentials shape for the given db_type."},
-        500: {"description": "Internal server error."},
-    },
-)
-@limiter.limit("10/minute")
-@log_execution(logger)
-async def create_database_client(
-    request: Request,
-    payload: DatabaseClientCreate,
-    user_id: str = Query(..., description="ID of the user registering the connection."),
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Register a new external database connection for a user.
-
-    The `credentials` object must match the shape for the chosen `db_type`
-    (see **CredentialSchemas** in the schema section below for exact fields).
-    Sensitive fields (`password`, `service_account_json`) are encrypted
-    before being persisted — they are never returned in any response.
-    """
-    try:
-        client = await database_client_service.create(
-            db=db,
-            user_id=user_id,
-            name=payload.name,
-            db_type=payload.db_type,
-            credentials=payload.credentials,
-        )
-        return DatabaseClientResponse.model_validate(client)
-    except Exception as e:
-        logger.error(f"Failed to create database client for user {user_id}", error=str(e))
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to create database client: {str(e)}",
-        )
-
-
-@router.get(
-    "/database-clients/{user_id}",
-    response_model=List[DatabaseClientResponse],
-    summary="List all database connections for a user",
-    response_description="List of database connections (credentials excluded).",
-    responses={
-        200: {"description": "Returns an empty list if the user has no connections."},
-    },
-)
-@log_execution(logger)
-async def list_database_clients(
-    user_id: str,
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Return all database connections registered by the specified user,
-    ordered by creation date (newest first).
-
-    Credentials are never included in the response.
-    """
-    clients = await database_client_service.get_user_clients(db, user_id)
-    return [DatabaseClientResponse.model_validate(c) for c in clients]
-
-
-@router.get(
-    "/database-clients/{user_id}/{client_id}",
-    response_model=DatabaseClientResponse,
-    summary="Get a single database connection",
-    response_description="Database connection detail (credentials excluded).",
-    responses={
-        404: {"description": "Connection not found."},
-        403: {"description": "Access denied — user_id does not own this connection."},
-    },
-)
-@log_execution(logger)
-async def get_database_client(
-    user_id: str,
-    client_id: str,
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Return the detail of a single database connection.
-
-    Returns **403** if the `user_id` in the path does not match the owner
-    of the requested connection.
-    """
-    client = await database_client_service.get(db, client_id)
-
-    if not client:
-        raise HTTPException(status_code=404, detail="Database client not found")
-
-    if client.user_id != user_id:
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    return DatabaseClientResponse.model_validate(client)
-
-
-@router.put(
-    "/database-clients/{client_id}",
-    response_model=DatabaseClientResponse,
-    summary="Update a database connection",
-    response_description="Updated database connection record (credentials excluded).",
-    responses={
-        404: {"description": "Connection not found."},
-        403: {"description": "Access denied — user_id does not own this connection."},
-    },
-)
-@log_execution(logger)
-async def update_database_client(
-    client_id: str,
-    payload: DatabaseClientUpdate,
-    user_id: str = Query(..., description="ID of the user who owns the connection."),
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Update an existing database connection.
-
-    Only fields present in the request body are updated.
-    If `credentials` is provided it **replaces** the entire credentials object
-    and sensitive fields are re-encrypted automatically.
-    """
-    client = await database_client_service.get(db, client_id)
-
-    if not client:
-        raise HTTPException(status_code=404, detail="Database client not found")
-
-    if client.user_id != user_id:
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    updated = await database_client_service.update(
-        db=db,
-        client_id=client_id,
-        name=payload.name,
-        credentials=payload.credentials,
-        status=payload.status,
-    )
-    return DatabaseClientResponse.model_validate(updated)
-
-
-@router.delete(
-    "/database-clients/{client_id}",
-    status_code=status.HTTP_200_OK,
-    summary="Delete a database connection",
-    responses={
-        200: {"description": "Connection deleted successfully."},
-        404: {"description": "Connection not found."},
-        403: {"description": "Access denied — user_id does not own this connection."},
-    },
-)
-@log_execution(logger)
-async def delete_database_client(
-    client_id: str,
-    user_id: str = Query(..., description="ID of the user who owns the connection."),
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Permanently delete a database connection.
-
-    This action is irreversible. The stored credentials are also removed.
-    """
-    client = await database_client_service.get(db, client_id)
-
-    if not client:
-        raise HTTPException(status_code=404, detail="Database client not found")
-
-    if client.user_id != user_id:
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    await database_client_service.delete(db, client_id)
-    return {"status": "success", "message": "Database client deleted successfully"}
-
-
-@router.post(
-    "/database-clients/{client_id}/ingest",
-    status_code=status.HTTP_200_OK,
-    summary="Ingest schema from a registered database into the vector store",
-    response_description="Count of chunks ingested.",
-    responses={
-        200: {"description": "Ingestion completed successfully."},
-        403: {"description": "Access denied — user_id does not own this connection."},
-        404: {"description": "Connection not found."},
-        501: {"description": "The connection's db_type is not yet supported by the pipeline."},
-        500: {"description": "Ingestion failed (connection error, profiling error, etc.)."},
-    },
-)
-@limiter.limit("5/minute")
-@log_execution(logger)
-async def ingest_database_client(
-    request: Request,
-    client_id: str,
-    user_id: str = Query(..., description="ID of the user who owns the connection."),
-    db: AsyncSession = Depends(get_db),
-):
-    """
-    Decrypt the stored credentials, connect to the user's database, introspect
-    its schema, profile each column, embed the descriptions, and store them in
-    the shared PGVector collection tagged with `source_type="database"`.
-
-    Chunks become retrievable via the same retriever used for document chunks.
-    """
-    client = await database_client_service.get(db, client_id)
-
-    if not client:
-        raise HTTPException(status_code=404, detail="Database client not found")
-
-    if client.user_id != user_id:
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    creds = decrypt_credentials_dict(client.credentials)
-
-    try:
-        with db_pipeline_service.engine_scope(
-            db_type=client.db_type,
-            credentials=creds,
-        ) as engine:
-            total = await db_pipeline_service.run(user_id=user_id, engine=engine)
-    except NotImplementedError as e:
-        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(e))
-    except Exception as e:
-        logger.error(
-            f"Ingestion failed for client {client_id}", user_id=user_id, error=str(e)
-        )
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Ingestion failed: {e}",
-        )
-
-    return {"status": "success", "client_id": client_id, "chunks_ingested": total}
+dbtypes: Literal["postgresql", "mysql", "sqlite"] = Literal["postgresql", "mysql", "sqlite"]
+creds: Dict[str, str]

src/api/v1/document.py

@@ -1,20 +1,21 @@
 """Document management API endpoints."""
- 
-from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File
+
+from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, status
 from sqlalchemy.ext.asyncio import AsyncSession
 from src.db.postgres.connection import get_db
 from src.document.document_service import document_service
+from src.knowledge.processing_service import knowledge_processor
+from src.storage.az_blob.az_blob import blob_storage
 from src.middlewares.logging import get_logger, log_execution
 from src.middlewares.rate_limit import limiter
-from src.pipeline.document_pipeline.document_pipeline import document_pipeline
 from pydantic import BaseModel
 from typing import List
- 
+
 logger = get_logger("document_api")
- 
+
 router = APIRouter(prefix="/api/v1", tags=["Documents"])
- 
- 
+
+
 class DocumentResponse(BaseModel):
     id: str
     filename: str
@@ -22,8 +23,8 @@ class DocumentResponse(BaseModel):
     file_size: int
     file_type: str
     created_at: str
- 
- 
+
+
 @router.get("/documents/{user_id}", response_model=List[DocumentResponse])
 @log_execution(logger)
 async def list_documents(
@@ -43,8 +44,8 @@ async def list_documents(
         )
         for doc in documents
     ]
- 
- 
+
+
 @router.post("/document/upload")
 @limiter.limit("10/minute")
 @log_execution(logger)
@@ -56,12 +57,57 @@ async def upload_document(
 ):
     """Upload a document."""
     if not user_id:
-        raise HTTPException(status_code=400, detail="user_id is required")
- 
-    data = await document_pipeline.upload(file, user_id, db)
-    return {"status": "success", "message": "Document uploaded successfully", "data": data}
- 
- 
+        raise HTTPException(
+            status_code=400,
+            detail="user_id is required"
+        )
+
+    try:
+        # Read file content
+        content = await file.read()
+        file_size = len(content)
+
+        # Get file type
+        filename = file.filename
+        file_type = filename.split('.')[-1].lower() if '.' in filename else 'txt'
+
+        if file_type not in ['pdf', 'docx', 'txt']:
+            raise HTTPException(
+                status_code=400,
+                detail="Unsupported file type. Supported: pdf, docx, txt"
+            )
+
+        # Upload to blob storage
+        blob_name = await blob_storage.upload_file(content, filename, user_id)
+
+        # Create document record
+        document = await document_service.create_document(
+            db=db,
+            user_id=user_id,
+            filename=filename,
+            blob_name=blob_name,
+            file_size=file_size,
+            file_type=file_type
+        )
+
+        return {
+            "status": "success",
+            "message": "Document uploaded successfully",
+            "data": {
+                "id": document.id,
+                "filename": document.filename,
+                "status": document.status
+            }
+        }
+
+    except Exception as e:
+        logger.error(f"Upload failed for user {user_id}", error=str(e))
+        raise HTTPException(
+            status_code=500,
+            detail=f"Upload failed: {str(e)}"
+        )
+
+
 @router.delete("/document/delete")
 @log_execution(logger)
 async def delete_document(
@@ -70,10 +116,31 @@ async def delete_document(
     db: AsyncSession = Depends(get_db)
 ):
     """Delete a document."""
-    await document_pipeline.delete(document_id, user_id, db)
-    return {"status": "success", "message": "Document deleted successfully"}
- 
- 
+    document = await document_service.get_document(db, document_id)
+
+    if not document:
+        raise HTTPException(
+            status_code=404,
+            detail="Document not found"
+        )
+
+    if document.user_id != user_id:
+        raise HTTPException(
+            status_code=403,
+            detail="Access denied"
+        )
+
+    success = await document_service.delete_document(db, document_id)
+
+    if success:
+        return {"status": "success", "message": "Document deleted successfully"}
+    else:
+        raise HTTPException(
+            status_code=500,
+            detail="Failed to delete document"
+        )
+
+
 @router.post("/document/process")
 @log_execution(logger)
 async def process_document(
@@ -82,6 +149,45 @@ async def process_document(
     db: AsyncSession = Depends(get_db)
 ):
     """Process document and ingest to vector index."""
-    data = await document_pipeline.process(document_id, user_id, db)
-    return {"status": "success", "message": "Document processed successfully", "data": data}
- 
+    document = await document_service.get_document(db, document_id)
+
+    if not document:
+        raise HTTPException(
+            status_code=404,
+            detail="Document not found"
+        )
+
+    if document.user_id != user_id:
+        raise HTTPException(
+            status_code=403,
+            detail="Access denied"
+        )
+
+    try:
+        # Update status to processing
+        await document_service.update_document_status(db, document_id, "processing")
+
+        # Process document
+        chunks_count = await knowledge_processor.process_document(document, db)
+
+        # Update status to completed
+        await document_service.update_document_status(db, document_id, "completed")
+
+        return {
+            "status": "success",
+            "message": "Document processed successfully",
+            "data": {
+                "document_id": document_id,
+                "chunks_processed": chunks_count
+            }
+        }
+
+    except Exception as e:
+        logger.error(f"Processing failed for document {document_id}", error=str(e))
+        await document_service.update_document_status(
+            db, document_id, "failed", str(e)
+        )
+        raise HTTPException(
+            status_code=500,
+            detail=f"Processing failed: {str(e)}"
+        )

src/config/settings.py

@@ -61,11 +61,6 @@ class Settings(BaseSettings):
     # Bcrypt salt (for users - existing)
     emarcal_bcrypt_salt: str = Field(alias="emarcal__bcrypt__salt", default="")
 
-    # DB credential encryption (Fernet key for user-registered database creds)
-    dataeyond_db_credential_key: str = Field(
-        alias="dataeyond__db__credential__key", default=""
-    )
-
 
 # Singleton instance
 settings = Settings()

src/database_client/database_client_service.py

@@ -1,118 +0,0 @@
-"""Service for managing user-registered external database connections."""
-
-import uuid
-from typing import List, Optional
-
-from sqlalchemy import delete, select
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from src.db.postgres.models import DatabaseClient
-from src.middlewares.logging import get_logger
-from src.utils.db_credential_encryption import (
-    decrypt_credentials_dict,
-    encrypt_credentials_dict,
-)
-
-logger = get_logger("database_client_service")
-
-
-class DatabaseClientService:
-    """Service for managing user-registered external database connections."""
-
-    async def create(
-        self,
-        db: AsyncSession,
-        user_id: str,
-        name: str,
-        db_type: str,
-        credentials: dict,
-    ) -> DatabaseClient:
-        """Register a new database client connection.
-
-        Credentials are encrypted before being stored.
-        """
-        client = DatabaseClient(
-            id=str(uuid.uuid4()),
-            user_id=user_id,
-            name=name,
-            db_type=db_type,
-            credentials=encrypt_credentials_dict(credentials),
-            status="active",
-        )
-        db.add(client)
-        await db.commit()
-        await db.refresh(client)
-        logger.info(f"Created database client {client.id} for user {user_id}")
-        return client
-
-    async def get_user_clients(
-        self,
-        db: AsyncSession,
-        user_id: str,
-    ) -> List[DatabaseClient]:
-        """Return all active and inactive database clients for a user."""
-        result = await db.execute(
-            select(DatabaseClient)
-            .where(DatabaseClient.user_id == user_id)
-            .order_by(DatabaseClient.created_at.desc())
-        )
-        return result.scalars().all()
-
-    async def get(
-        self,
-        db: AsyncSession,
-        client_id: str,
-    ) -> Optional[DatabaseClient]:
-        """Return a single database client by its ID."""
-        result = await db.execute(
-            select(DatabaseClient).where(DatabaseClient.id == client_id)
-        )
-        return result.scalars().first()
-
-    async def update(
-        self,
-        db: AsyncSession,
-        client_id: str,
-        name: Optional[str] = None,
-        credentials: Optional[dict] = None,
-        status: Optional[str] = None,
-    ) -> Optional[DatabaseClient]:
-        """Update an existing database client connection.
-
-        Only non-None fields are updated.
-        Credentials are re-encrypted if provided.
-        """
-        client = await self.get(db, client_id)
-        if not client:
-            return None
-
-        if name is not None:
-            client.name = name
-        if credentials is not None:
-            client.credentials = encrypt_credentials_dict(credentials)
-        if status is not None:
-            client.status = status
-
-        await db.commit()
-        await db.refresh(client)
-        logger.info(f"Updated database client {client_id}")
-        return client
-
-    async def delete(
-        self,
-        db: AsyncSession,
-        client_id: str,
-    ) -> bool:
-        """Permanently delete a database client connection."""
-        result = await db.execute(
-            delete(DatabaseClient).where(DatabaseClient.id == client_id)
-        )
-        await db.commit()
-        deleted = result.rowcount > 0
-        if deleted:
-            logger.info(f"Deleted database client {client_id}")
-        return deleted
-
-
-database_client_service = DatabaseClientService()
- 

src/db/postgres/init_db.py

@@ -2,14 +2,7 @@
 
 from sqlalchemy import text
 from src.db.postgres.connection import engine, Base
-from src.db.postgres.models import (
-    ChatMessage,
-    DatabaseClient,
-    Document,
-    MessageSource,
-    Room,
-    User,
-)
+from src.db.postgres.models import Document, Room, ChatMessage, User, MessageSource
 
 
 async def init_db():

src/db/postgres/models.py

@@ -4,7 +4,6 @@ from uuid import uuid4
 from sqlalchemy import Column, String, DateTime, Text, Integer, ForeignKey
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
-from sqlalchemy.dialects.postgresql import JSONB
 from src.db.postgres.connection import Base
 
 
@@ -82,18 +81,3 @@ class MessageSource(Base):
     created_at = Column(DateTime(timezone=True), server_default=func.now())
 
     message = relationship("ChatMessage", back_populates="sources")
-
-
-class DatabaseClient(Base):
-    """User-registered external database connections."""
-    __tablename__ = "databases"
-
-    id = Column(String, primary_key=True, default=lambda: str(uuid4()))
-    user_id = Column(String, nullable=False, index=True)
-    name = Column(String, nullable=False)       # display name, e.g. "Prod DB"
-    db_type = Column(String, nullable=False)    # postgres|mysql|sqlserver|supabase|bigquery|snowflake
-    credentials = Column(JSONB, nullable=False) # per-type JSON; sensitive fields Fernet-encrypted
-    status = Column(String, nullable=False, default="active")  # active | inactive
-    created_at = Column(DateTime(timezone=True), server_default=func.now())
-    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
- 

src/knowledge/processing_service.py

@@ -5,14 +5,14 @@ from langchain_core.documents import Document as LangChainDocument
 from src.db.postgres.vector_store import get_vector_store
 from src.storage.az_blob.az_blob import blob_storage
 from src.db.postgres.models import Document as DBDocument
+from src.config.settings import settings
 from sqlalchemy.ext.asyncio import AsyncSession
 from src.middlewares.logging import get_logger
+from azure.ai.documentintelligence.aio import DocumentIntelligenceClient
+from azure.core.credentials import AzureKeyCredential
 from typing import List
-import sys
+import pypdf
 import docx
-import pandas as pd
-import pytesseract
-from pdf2image import convert_from_bytes
 from io import BytesIO
 
 logger = get_logger("knowledge_processing")
@@ -40,10 +40,6 @@ class KnowledgeProcessingService:
 
             if db_doc.file_type == "pdf":
                 documents = await self._build_pdf_documents(content, db_doc)
-            elif db_doc.file_type == "csv":
-                documents = self._build_csv_documents(content, db_doc)
-            elif db_doc.file_type == "xlsx":
-                documents = self._build_excel_documents(content, db_doc)
             else:
                 text = self._extract_text(content, db_doc.file_type)
                 if not text.strip():
@@ -53,14 +49,10 @@ class KnowledgeProcessingService:
                     LangChainDocument(
                         page_content=chunk,
                         metadata={
+                            "document_id": db_doc.id,
                             "user_id": db_doc.user_id,
-                            "source_type": "document",
-                            "data": {
-                                "document_id": db_doc.id,
-                                "filename": db_doc.filename,
-                                "file_type": db_doc.file_type,
-                                "chunk_index": i,
-                            },
+                            "filename": db_doc.filename,
+                            "chunk_index": i,
                         }
                     )
                     for i, chunk in enumerate(chunks)
@@ -82,98 +74,62 @@ class KnowledgeProcessingService:
     async def _build_pdf_documents(
         self, content: bytes, db_doc: DBDocument
     ) -> List[LangChainDocument]:
-        """Build LangChain documents from PDF with page_label metadata using Tesseract OCR."""
+        """Build LangChain documents from PDF with page_label metadata.
+
+        Uses Azure Document Intelligence (per-page) when credentials are present,
+        falls back to pypdf (also per-page) otherwise.
+        """
         documents: List[LangChainDocument] = []
 
-        poppler_path = None
-        if sys.platform == "win32":
-            pytesseract.pytesseract.tesseract_cmd = r"./software/Tesseract-OCR/tesseract.exe"
-            poppler_path = "./software/poppler-24.08.0/Library/bin"
-
-        images = convert_from_bytes(content, poppler_path=poppler_path)
-        logger.info(f"Tesseract OCR: converting {len(images)} pages")
-
-        for page_num, image in enumerate(images, start=1):
-            page_text = pytesseract.image_to_string(image)
-            if not page_text.strip():
-                continue
-            for chunk in self.text_splitter.split_text(page_text):
-                documents.append(LangChainDocument(
-                    page_content=chunk,
-                    metadata={
-                        "user_id": db_doc.user_id,
-                        "source_type": "document",
-                        "data": {
+        if settings.azureai_docintel_endpoint and settings.azureai_docintel_key:
+            async with DocumentIntelligenceClient(
+                endpoint=settings.azureai_docintel_endpoint,
+                credential=AzureKeyCredential(settings.azureai_docintel_key),
+            ) as client:
+                poller = await client.begin_analyze_document(
+                    model_id="prebuilt-read",
+                    body=BytesIO(content),
+                    content_type="application/pdf",
+                )
+                result = await poller.result()
+                logger.info(f"Azure DI extracted {len(result.pages or [])} pages")
+
+                for page in result.pages or []:
+                    page_text = "\n".join(
+                        line.content for line in (page.lines or [])
+                    )
+                    if not page_text.strip():
+                        continue
+                    for chunk in self.text_splitter.split_text(page_text):
+                        documents.append(LangChainDocument(
+                            page_content=chunk,
+                            metadata={
+                                "document_id": db_doc.id,
+                                "user_id": db_doc.user_id,
+                                "filename": db_doc.filename,
+                                "chunk_index": len(documents),
+                                "page_label": page.page_number,
+                            }
+                        ))
+        else:
+            logger.warning("Azure DI not configured, using pypdf")
+            pdf_reader = pypdf.PdfReader(BytesIO(content))
+            for page_num, page in enumerate(pdf_reader.pages, start=1):
+                page_text = page.extract_text() or ""
+                if not page_text.strip():
+                    continue
+                for chunk in self.text_splitter.split_text(page_text):
+                    documents.append(LangChainDocument(
+                        page_content=chunk,
+                        metadata={
                             "document_id": db_doc.id,
+                            "user_id": db_doc.user_id,
                             "filename": db_doc.filename,
-                            "file_type": db_doc.file_type,
                             "chunk_index": len(documents),
                             "page_label": page_num,
-                        },
-                    }
-                ))
-
-        return documents
-
-    def _profile_dataframe(
-        self, df: pd.DataFrame, source_name: str, db_doc: DBDocument
-    ) -> List[LangChainDocument]:
-        """Profile each column of a dataframe → one chunk per column."""
-        documents = []
-        row_count = len(df)
-
-        for col_name in df.columns:
-            col = df[col_name]
-            is_numeric = pd.api.types.is_numeric_dtype(col)
-            null_count = int(col.isnull().sum())
-            distinct_count = int(col.nunique())
-            distinct_ratio = distinct_count / row_count if row_count > 0 else 0
-
-            text = f"Source: {source_name} ({row_count} rows)\n"
-            text += f"Column: {col_name} ({col.dtype})\n"
-            text += f"Null count: {null_count}\n"
-            text += f"Distinct count: {distinct_count} ({distinct_ratio:.1%})\n"
-
-            if is_numeric:
-                text += f"Min: {col.min()}, Max: {col.max()}\n"
-                text += f"Mean: {col.mean():.4f}, Median: {col.median():.4f}\n"
-
-            if 0 < distinct_ratio <= 0.05:
-                top_values = col.value_counts().head(10)
-                top_str = ", ".join(f"{v} ({c})" for v, c in top_values.items())
-                text += f"Top values: {top_str}\n"
-
-            text += f"Sample values: {col.dropna().head(5).tolist()}"
-
-            documents.append(LangChainDocument(
-                page_content=text,
-                metadata={
-                    "user_id": db_doc.user_id,
-                    "source_type": "document",
-                    "data": {
-                        "document_id": db_doc.id,
-                        "filename": db_doc.filename,
-                        "file_type": db_doc.file_type,
-                        "source": source_name,
-                        "column_name": col_name,
-                        "column_type": str(col.dtype),
-                    }
-                }
-            ))
-        return documents
+                        }
+                    ))
 
-    def _build_csv_documents(self, content: bytes, db_doc: DBDocument) -> List[LangChainDocument]:
-        """Profile each column of a CSV file."""
-        df = pd.read_csv(BytesIO(content))
-        return self._profile_dataframe(df, db_doc.filename, db_doc)
-
-    def _build_excel_documents(self, content: bytes, db_doc: DBDocument) -> List[LangChainDocument]:
-        """Profile each column of every sheet in an Excel file."""
-        sheets = pd.read_excel(BytesIO(content), sheet_name=None)
-        documents = []
-        for sheet_name, df in sheets.items():
-            source_name = f"{db_doc.filename} / sheet: {sheet_name}"
-            documents.extend(self._profile_dataframe(df, source_name, db_doc))
         return documents
 
     def _extract_text(self, content: bytes, file_type: str) -> str:

src/models/credentials.py

@@ -1,164 +0,0 @@
-"""Pydantic credential schemas for user-registered external databases.
-
-Imported by the `/database-clients` API router (`src/api/v1/db_client.py`) and,
-via `DbType`, by the db pipeline connector (`src/pipeline/db_pipeline/connector.py`).
-
-Sensitive fields (`password`, `service_account_json`) are Fernet-encrypted by
-the database_client service before being stored in the JSONB column; these
-schemas describe the plaintext wire format, not the stored shape.
-"""
-
-from typing import Literal, Optional, Union
-
-from pydantic import BaseModel, Field
-
-# ---------------------------------------------------------------------------
-# Supported DB types
-# ---------------------------------------------------------------------------
-
-DbType = Literal["postgres", "mysql", "sqlserver", "supabase", "bigquery", "snowflake"]
-
-
-# ---------------------------------------------------------------------------
-# Typed credential schemas per DB type
-# ---------------------------------------------------------------------------
-
-
-class PostgresCredentials(BaseModel):
-    """Connection credentials for PostgreSQL."""
-
-    host: str = Field(..., description="Hostname or IP address of the PostgreSQL server.", examples=["db.example.com"])
-    port: int = Field(5432, description="Port number (default: 5432).", examples=[5432])
-    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
-    username: str = Field(..., description="Database username.", examples=["admin"])
-    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
-    ssl_mode: Literal["disable", "require", "verify-ca", "verify-full"] = Field(
-        "require",
-        description="SSL mode for the connection.",
-        examples=["require"],
-    )
-
-
-class MysqlCredentials(BaseModel):
-    """Connection credentials for MySQL."""
-
-    host: str = Field(..., description="Hostname or IP address of the MySQL server.", examples=["db.example.com"])
-    port: int = Field(3306, description="Port number (default: 3306).", examples=[3306])
-    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
-    username: str = Field(..., description="Database username.", examples=["admin"])
-    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
-    ssl: bool = Field(True, description="Enable SSL for the connection.", examples=[True])
-
-
-class SqlServerCredentials(BaseModel):
-    """Connection credentials for Microsoft SQL Server."""
-
-    host: str = Field(..., description="Hostname or IP address of the SQL Server.", examples=["sqlserver.example.com"])
-    port: int = Field(1433, description="Port number (default: 1433).", examples=[1433])
-    database: str = Field(..., description="Name of the target database.", examples=["mydb"])
-    username: str = Field(..., description="Database username.", examples=["sa"])
-    password: str = Field(..., description="Database password. Will be encrypted at rest.", examples=["s3cr3t!"])
-    driver: Optional[str] = Field(
-        None,
-        description="ODBC driver name. Leave empty to use the default driver.",
-        examples=["ODBC Driver 17 for SQL Server"],
-    )
-
-
-class SupabaseCredentials(BaseModel):
-    """Connection credentials for Supabase (PostgreSQL-based).
-
-    Use the connection string details from your Supabase project dashboard
-    under Settings > Database.
-    """
-
-    host: str = Field(
-        ...,
-        description="Supabase database host (e.g. db.<project-ref>.supabase.co, or the pooler host).",
-        examples=["db.xxxx.supabase.co"],
-    )
-    port: int = Field(
-        5432,
-        description="Port number. Use 5432 for direct connection, 6543 for the connection pooler.",
-        examples=[5432],
-    )
-    database: str = Field("postgres", description="Database name (always 'postgres' for Supabase).", examples=["postgres"])
-    username: str = Field(
-        ...,
-        description="Database user. Use 'postgres' for direct connection, or 'postgres.<project-ref>' for the pooler.",
-        examples=["postgres"],
-    )
-    password: str = Field(..., description="Database password (set in Supabase dashboard). Will be encrypted at rest.", examples=["s3cr3t!"])
-    ssl_mode: Literal["require", "verify-ca", "verify-full"] = Field(
-        "require",
-        description="SSL mode. Supabase always requires SSL.",
-        examples=["require"],
-    )
-
-
-class BigQueryCredentials(BaseModel):
-    """Connection credentials for Google BigQuery.
-
-    Requires a GCP Service Account with at least BigQuery Data Viewer
-    and BigQuery Job User roles.
-    """
-
-    project_id: str = Field(..., description="GCP project ID where the BigQuery dataset resides.", examples=["my-gcp-project"])
-    dataset_id: str = Field(..., description="BigQuery dataset name to connect to.", examples=["my_dataset"])
-    location: Optional[str] = Field(
-        "US",
-        description="Dataset location/region (default: US).",
-        examples=["US", "EU", "asia-southeast1"],
-    )
-    service_account_json: str = Field(
-        ...,
-        description=(
-            "Full content of the GCP Service Account key JSON file as a string. "
-            "Will be encrypted at rest."
-        ),
-        examples=['{"type":"service_account","project_id":"my-gcp-project","private_key_id":"..."}'],
-    )
-
-
-class SnowflakeCredentials(BaseModel):
-    """Connection credentials for Snowflake."""
-
-    account: str = Field(
-        ...,
-        description="Snowflake account identifier, including region if applicable (e.g. myaccount.us-east-1).",
-        examples=["myaccount.us-east-1"],
-    )
-    warehouse: str = Field(..., description="Name of the virtual warehouse to use for queries.", examples=["COMPUTE_WH"])
-    database: str = Field(..., description="Name of the target Snowflake database.", examples=["MY_DB"])
-    db_schema: Optional[str] = Field("PUBLIC", alias="schema", description="Schema name (default: PUBLIC).", examples=["PUBLIC"])
-    username: str = Field(..., description="Snowflake username.", examples=["admin"])
-    password: str = Field(..., description="Snowflake password. Will be encrypted at rest.", examples=["s3cr3t!"])
-    role: Optional[str] = Field(None, description="Snowflake role to assume for the session.", examples=["SYSADMIN"])
-
-
-# Union of all credential shapes — reserved for future typed validation on
-# DatabaseClientCreate.credentials (currently Dict[str, Any]). Kept exported
-# so downstream code can reference it without re-declaring.
-CredentialsUnion = Union[
-    PostgresCredentials,
-    MysqlCredentials,
-    SqlServerCredentials,
-    SupabaseCredentials,
-    BigQueryCredentials,
-    SnowflakeCredentials,
-]
-
-
-# Doc-only helper: surfaces per-type credential shapes in the Swagger "Schemas"
-# panel so API consumers can discover the exact field set for each db_type.
-# Not referenced by any endpoint — importing it in db_client.py is enough for
-# FastAPI's OpenAPI generator to pick it up.
-class CredentialSchemas(BaseModel):
-    """Reference schemas for `credentials` per `db_type` (Swagger-only, not used by endpoints)."""
-
-    postgres: PostgresCredentials
-    mysql: MysqlCredentials
-    sqlserver: SqlServerCredentials
-    supabase: SupabaseCredentials
-    bigquery: BigQueryCredentials
-    snowflake: SnowflakeCredentials

src/pipeline/db_pipeline/__init__.py

@@ -1,3 +0,0 @@
-from src.pipeline.db_pipeline.db_pipeline_service import DbPipelineService, db_pipeline_service
-
-__all__ = ["DbPipelineService", "db_pipeline_service"]

src/pipeline/db_pipeline/db_pipeline_service.py

@@ -1,197 +0,0 @@
-"""Service for ingesting a user's external database into the vector store.
-
-End-to-end flow: connect -> introspect schema -> profile columns -> build text
--> embed + store in the shared PGVector collection (tagged with
-`source_type="database"`, retrievable via the same retriever used for docs).
-
-Sync DB work (SQLAlchemy inspect, pandas read_sql) runs in a threadpool;
-async vector writes stay on the event loop.
-"""
-
-import asyncio
-from contextlib import contextmanager
-from typing import Any, Iterator, Optional
-
-from langchain_core.documents import Document as LangChainDocument
-from sqlalchemy import URL, create_engine
-from sqlalchemy.engine import Engine
-
-from src.db.postgres.vector_store import get_vector_store
-from src.middlewares.logging import get_logger
-from src.models.credentials import DbType
-from src.pipeline.db_pipeline.extractor import get_schema, profile_table
-
-logger = get_logger("db_pipeline")
-
-
-class DbPipelineService:
-    """End-to-end DB ingestion: connect -> introspect -> profile -> embed -> store."""
-
-    def connect(self, db_type: DbType, credentials: dict[str, Any]) -> Engine:
-        """Build a SQLAlchemy engine for the user's database.
-
-        `credentials` is the plaintext dict matching the per-type schema in
-        `src/models/credentials.py`. BigQuery/Snowflake auth models differ
-        from host/port/user/pass, so every shape flows through one dict.
-
-        Optional driver imports (snowflake-sqlalchemy, json for BigQuery) are
-        done lazily so an env missing one driver doesn't break module import.
-        """
-        logger.info("connecting to user db", db_type=db_type)
-
-        if db_type in ("postgres", "supabase"):
-            query = (
-                {"sslmode": credentials["ssl_mode"]} if credentials.get("ssl_mode") else {}
-            )
-            url = URL.create(
-                drivername="postgresql+psycopg2",
-                username=credentials["username"],
-                password=credentials["password"],
-                host=credentials["host"],
-                port=credentials["port"],
-                database=credentials["database"],
-                query=query,
-            )
-            return create_engine(url)
-
-        if db_type == "mysql":
-            url = URL.create(
-                drivername="mysql+pymysql",
-                username=credentials["username"],
-                password=credentials["password"],
-                host=credentials["host"],
-                port=credentials["port"],
-                database=credentials["database"],
-            )
-            # pymysql only activates TLS when the `ssl` dict is truthy
-            # (empty dict is falsy and silently disables TLS). Use system-
-            # default CAs via certifi + hostname verification — required by
-            # managed MySQL providers like TiDB Cloud / PlanetScale / Aiven.
-            if credentials.get("ssl", True):
-                import certifi
-
-                connect_args = {
-                    "ssl": {
-                        "ca": certifi.where(),
-                        "check_hostname": True,
-                    }
-                }
-            else:
-                connect_args = {}
-            return create_engine(url, connect_args=connect_args)
-
-        if db_type == "sqlserver":
-            # `driver` applies to pyodbc only; we ship pymssql. Accept-and-ignore
-            # keeps the credential schema stable.
-            if credentials.get("driver"):
-                logger.info(
-                    "sqlserver driver hint ignored (using pymssql)",
-                    driver=credentials["driver"],
-                )
-            url = URL.create(
-                drivername="mssql+pymssql",
-                username=credentials["username"],
-                password=credentials["password"],
-                host=credentials["host"],
-                port=credentials["port"],
-                database=credentials["database"],
-            )
-            return create_engine(url)
-
-        if db_type == "bigquery":
-            import json
-
-            sa_info = json.loads(credentials["service_account_json"])
-            # sqlalchemy-bigquery URL shape: bigquery://<project>/<dataset>
-            url = f"bigquery://{credentials['project_id']}/{credentials['dataset_id']}"
-            return create_engine(
-                url,
-                credentials_info=sa_info,
-                location=credentials.get("location", "US"),
-            )
-
-        if db_type == "snowflake":
-            from snowflake.sqlalchemy import URL as SnowflakeURL
-
-            url = SnowflakeURL(
-                account=credentials["account"],
-                user=credentials["username"],
-                password=credentials["password"],
-                database=credentials["database"],
-                schema=(
-                    credentials.get("db_schema")
-                    or credentials.get("schema")
-                    or "PUBLIC"
-                ),
-                warehouse=credentials["warehouse"],
-                role=credentials.get("role") or "",
-            )
-            return create_engine(url)
-
-        raise ValueError(f"Unsupported db_type: {db_type}")
-
-    @contextmanager
-    def engine_scope(
-        self, db_type: DbType, credentials: dict[str, Any]
-    ) -> Iterator[Engine]:
-        """Yield a connected Engine and dispose its pool on exit.
-
-        API callers should prefer this over raw `connect(...)` so user DB
-        connection pools do not leak between pipeline runs.
-        """
-        engine = self.connect(db_type, credentials)
-        try:
-            yield engine
-        finally:
-            engine.dispose()
-
-    def _to_document(
-        self, user_id: str, table_name: str, entry: dict
-    ) -> LangChainDocument:
-        col = entry["col"]
-        return LangChainDocument(
-            page_content=entry["text"],
-            metadata={
-                "user_id": user_id,
-                "source_type": "database",
-                "data": {
-                    "table_name": table_name,
-                    "column_name": col["name"],
-                    "column_type": col["type"],
-                    "is_primary_key": col.get("is_primary_key", False),
-                    "foreign_key": col.get("foreign_key"),
-                },
-            },
-        )
-
-    async def run(
-        self,
-        user_id: str,
-        engine: Engine,
-        exclude_tables: Optional[frozenset[str]] = None,
-    ) -> int:
-        """Introspect the user's DB, profile columns, embed descriptions, store in PGVector.
-
-        Returns:
-            Total number of chunks ingested.
-        """
-        vector_store = get_vector_store()
-        logger.info("db pipeline start", user_id=user_id)
-
-        schema = await asyncio.to_thread(get_schema, engine, exclude_tables)
-
-        total = 0
-        for table_name, columns in schema.items():
-            logger.info("profiling table", table=table_name, columns=len(columns))
-            entries = await asyncio.to_thread(profile_table, engine, table_name, columns)
-            docs = [self._to_document(user_id, table_name, e) for e in entries]
-            if docs:
-                await vector_store.aadd_documents(docs)
-                total += len(docs)
-                logger.info("ingested chunks", table=table_name, count=len(docs))
-
-        logger.info("db pipeline complete", user_id=user_id, total=total)
-        return total
-
-
-db_pipeline_service = DbPipelineService()

src/pipeline/db_pipeline/extractor.py

@@ -1,213 +0,0 @@
-"""Schema introspection and per-column profiling for a user's database.
-
-Identifiers (table/column names) are quoted via the engine's dialect preparer,
-which handles reserved words, mixed case, and embedded quotes correctly across
-dialects. Values used in SQL come from SQLAlchemy inspection of the DB itself,
-not user input.
-"""
-
-from typing import Optional
-
-import pandas as pd
-from sqlalchemy import Float, Integer, Numeric, inspect
-from sqlalchemy.engine import Engine
-
-from src.middlewares.logging import get_logger
-
-logger = get_logger("db_extractor")
-
-TOP_VALUES_THRESHOLD = 0.05  # show top values if distinct_ratio <= 5%
-
-# Dialects where PERCENTILE_CONT(...) WITHIN GROUP is supported as an aggregate.
-# MySQL has no percentile aggregate; BigQuery has PERCENTILE_CONT only as an
-# analytic (window) function — both drop median and keep min/max/mean.
-_MEDIAN_DIALECTS = frozenset({"postgresql", "mssql", "snowflake"})
-
-
-def _supports_median(engine: Engine) -> bool:
-    return engine.dialect.name in _MEDIAN_DIALECTS
-
-
-def _head_query(
-    engine: Engine,
-    select_clause: str,
-    from_clause: str,
-    n: int,
-    order_by: str = "",
-) -> str:
-    """LIMIT/TOP-equivalent head query for the engine's dialect."""
-    if engine.dialect.name == "mssql":
-        return f"SELECT TOP {n} {select_clause} FROM {from_clause} {order_by}".strip()
-    return f"SELECT {select_clause} FROM {from_clause} {order_by} LIMIT {n}".strip()
-
-
-def _qi(engine: Engine, name: str) -> str:
-    """Dialect-correct identifier quoting (schema.table also handled if dotted)."""
-    preparer = engine.dialect.identifier_preparer
-    if "." in name:
-        schema, _, table = name.partition(".")
-        return f"{preparer.quote(schema)}.{preparer.quote(table)}"
-    return preparer.quote(name)
-
-
-def get_schema(
-    engine: Engine, exclude_tables: Optional[frozenset[str]] = None
-) -> dict[str, list[dict]]:
-    """Returns {table_name: [{name, type, is_numeric, is_primary_key, foreign_key}, ...]}."""
-    exclude = exclude_tables or frozenset()
-    inspector = inspect(engine)
-    schema = {}
-    for table_name in inspector.get_table_names():
-        if table_name in exclude:
-            continue
-
-        pk = inspector.get_pk_constraint(table_name)
-        pk_cols = set(pk["constrained_columns"]) if pk else set()
-
-        fk_map = {}
-        for fk in inspector.get_foreign_keys(table_name):
-            for col, ref_col in zip(fk["constrained_columns"], fk["referred_columns"]):
-                fk_map[col] = f"{fk['referred_table']}.{ref_col}"
-
-        cols = inspector.get_columns(table_name)
-        schema[table_name] = [
-            {
-                "name": c["name"],
-                "type": str(c["type"]),
-                "is_numeric": isinstance(c["type"], (Integer, Numeric, Float)),
-                "is_primary_key": c["name"] in pk_cols,
-                "foreign_key": fk_map.get(c["name"]),
-            }
-            for c in cols
-        ]
-    logger.info("extracted schema", table_count=len(schema))
-    return schema
-
-
-def get_row_count(engine: Engine, table_name: str) -> int:
-    return pd.read_sql(f"SELECT COUNT(*) FROM {_qi(engine, table_name)}", engine).iloc[0, 0]
-
-
-def profile_column(
-    engine: Engine,
-    table_name: str,
-    col_name: str,
-    is_numeric: bool,
-    row_count: int,
-) -> dict:
-    """Returns null_count, distinct_count, min/max, top values, and sample values."""
-    if row_count == 0:
-        return {
-            "null_count": 0,
-            "distinct_count": 0,
-            "distinct_ratio": 0.0,
-            "sample_values": [],
-        }
-
-    qt = _qi(engine, table_name)
-    qc = _qi(engine, col_name)
-
-    # Combined stats query: null_count, distinct_count, and min/max (if numeric).
-    # One round-trip instead of two.
-    select_cols = [
-        f"COUNT(*) - COUNT({qc}) AS nulls",
-        f"COUNT(DISTINCT {qc}) AS distincts",
-    ]
-    if is_numeric:
-        select_cols.append(f"MIN({qc}) AS min_val")
-        select_cols.append(f"MAX({qc}) AS max_val")
-        select_cols.append(f"AVG({qc}) AS mean_val")
-        if _supports_median(engine):
-            select_cols.append(
-                f"PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY {qc}) AS median_val"
-            )
-    stats = pd.read_sql(f"SELECT {', '.join(select_cols)} FROM {qt}", engine)
-
-    null_count = int(stats.iloc[0]["nulls"])
-    distinct_count = int(stats.iloc[0]["distincts"])
-    distinct_ratio = distinct_count / row_count if row_count > 0 else 0
-
-    profile = {
-        "null_count": null_count,
-        "distinct_count": distinct_count,
-        "distinct_ratio": round(distinct_ratio, 4),
-    }
-
-    if is_numeric:
-        profile["min"] = stats.iloc[0]["min_val"]
-        profile["max"] = stats.iloc[0]["max_val"]
-        profile["mean"] = stats.iloc[0]["mean_val"]
-        if _supports_median(engine):
-            profile["median"] = stats.iloc[0]["median_val"]
-
-    if 0 < distinct_ratio <= TOP_VALUES_THRESHOLD:
-        top_sql = _head_query(
-            engine,
-            select_clause=f"{qc}, COUNT(*) AS cnt",
-            from_clause=f"{qt} GROUP BY {qc}",
-            n=10,
-            order_by="ORDER BY cnt DESC",
-        )
-        top = pd.read_sql(top_sql, engine)
-        profile["top_values"] = list(zip(top[col_name].tolist(), top["cnt"].tolist()))
-
-    sample = pd.read_sql(_head_query(engine, qc, qt, 5), engine)
-    profile["sample_values"] = sample[col_name].tolist()
-
-    return profile
-
-
-def profile_table(engine: Engine, table_name: str, columns: list[dict]) -> list[dict]:
-    """Profile every column in a table. Returns [{col, profile, text}, ...].
-
-    Per-column errors are logged and skipped so one bad column doesn't abort
-    the whole table.
-    """
-    row_count = get_row_count(engine, table_name)
-    if row_count == 0:
-        logger.info("skipping empty table", table=table_name)
-        return []
-
-    results = []
-    for col in columns:
-        try:
-            profile = profile_column(
-                engine, table_name, col["name"], col.get("is_numeric", False), row_count
-            )
-            text = build_text(table_name, row_count, col, profile)
-            results.append({"col": col, "profile": profile, "text": text})
-        except Exception as e:
-            logger.error(
-                "column profiling failed",
-                table=table_name,
-                column=col["name"],
-                error=str(e),
-            )
-            continue
-    return results
-
-
-def build_text(table_name: str, row_count: int, col: dict, profile: dict) -> str:
-    col_name = col["name"]
-    col_type = col["type"]
-
-    key_label = ""
-    if col.get("is_primary_key"):
-        key_label = " [PRIMARY KEY]"
-    elif col.get("foreign_key"):
-        key_label = f" [FK -> {col['foreign_key']}]"
-
-    text = f"Table: {table_name} ({row_count} rows)\n"
-    text += f"Column: {col_name} ({col_type}){key_label}\n"
-    text += f"Null count: {profile['null_count']}\n"
-    text += f"Distinct count: {profile['distinct_count']} ({profile['distinct_ratio']:.1%})\n"
-    if "min" in profile:
-        text += f"Min: {profile['min']}, Max: {profile['max']}\n"
-        text += f"Mean: {profile['mean']}\n"
-        if profile.get("median") is not None:
-            text += f"Median: {profile['median']}\n"
-    if "top_values" in profile:
-        top_str = ", ".join(f"{v} ({c})" for v, c in profile["top_values"])
-        text += f"Top values: {top_str}\n"
-    text += f"Sample values: {profile['sample_values']}"
-    return text

src/pipeline/document_pipeline/document_pipeline.py

@@ -1,80 +0,0 @@
-"""Document upload and processing pipeline."""
-
-from fastapi import HTTPException, UploadFile
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from src.document.document_service import document_service
-from src.knowledge.processing_service import knowledge_processor
-from src.middlewares.logging import get_logger
-from src.storage.az_blob.az_blob import blob_storage
-
-logger = get_logger("document_pipeline")
-
-SUPPORTED_FILE_TYPES = ["pdf", "docx", "txt", "csv", "xlsx"]
-
-
-class DocumentPipeline:
-    """Orchestrates the full document upload, process, and delete flows."""
-
-    async def upload(self, file: UploadFile, user_id: str, db: AsyncSession) -> dict:
-        """Validate → upload to blob → save to DB."""
-        content = await file.read()
-        file_type = file.filename.split(".")[-1].lower() if "." in file.filename else "txt"
-
-        if file_type not in SUPPORTED_FILE_TYPES:
-            raise HTTPException(
-                status_code=400,
-                detail=f"Unsupported file type. Supported: {SUPPORTED_FILE_TYPES}",
-            )
-
-        blob_name = await blob_storage.upload_file(content, file.filename, user_id)
-        document = await document_service.create_document(
-            db=db,
-            user_id=user_id,
-            filename=file.filename,
-            blob_name=blob_name,
-            file_size=len(content),
-            file_type=file_type,
-        )
-
-        logger.info(f"Uploaded document {document.id} for user {user_id}")
-        return {"id": document.id, "filename": document.filename, "status": document.status}
-
-    async def process(self, document_id: str, user_id: str, db: AsyncSession) -> dict:
-        """Validate ownership → extract text → chunk → ingest to vector store."""
-        document = await document_service.get_document(db, document_id)
-
-        if not document:
-            raise HTTPException(status_code=404, detail="Document not found")
-        if document.user_id != user_id:
-            raise HTTPException(status_code=403, detail="Access denied")
-
-        try:
-            await document_service.update_document_status(db, document_id, "processing")
-            chunks_count = await knowledge_processor.process_document(document, db)
-            await document_service.update_document_status(db, document_id, "completed")
-
-            logger.info(f"Processed document {document_id}: {chunks_count} chunks")
-            return {"document_id": document_id, "chunks_processed": chunks_count}
-
-        except Exception as e:
-            logger.error(f"Processing failed for document {document_id}", error=str(e))
-            await document_service.update_document_status(db, document_id, "failed", str(e))
-            raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}")
-
-    async def delete(self, document_id: str, user_id: str, db: AsyncSession) -> dict:
-        """Validate ownership → delete from blob and DB."""
-        document = await document_service.get_document(db, document_id)
-
-        if not document:
-            raise HTTPException(status_code=404, detail="Document not found")
-        if document.user_id != user_id:
-            raise HTTPException(status_code=403, detail="Access denied")
-
-        await document_service.delete_document(db, document_id)
-
-        logger.info(f"Deleted document {document_id} for user {user_id}")
-        return {"document_id": document_id}
-
-
-document_pipeline = DocumentPipeline()

src/utils/db_credential_encryption.py

@@ -1,70 +0,0 @@
-"""Fernet encryption utilities for user-registered database credentials.
-
-Encryption key is sourced from `dataeyond__db__credential__key` env variable,
-intentionally separate from the user-auth bcrypt salt (`emarcal__bcrypt__salt`).
-
-Usage:
-    from src.utils.db_credential_encryption import encrypt_credentials_dict, decrypt_credentials_dict
-
-    # Before INSERT:
-    safe_creds = encrypt_credentials_dict(raw_credentials)
-
-    # After SELECT:
-    plain_creds = decrypt_credentials_dict(row.credentials)
-"""
-
-from cryptography.fernet import Fernet
-from src.config.settings import settings
-
-# Sensitive credential field names that must be encrypted at rest.
-# Covers all supported DB types:
-#   - password      : postgres, mysql, sqlserver, supabase, snowflake
-#   - service_account_json : bigquery
-SENSITIVE_FIELDS: frozenset[str] = frozenset({"password", "service_account_json"})
-
-
-def _get_cipher() -> Fernet:
-    key = settings.dataeyond_db_credential_key
-    if not key:
-        raise ValueError(
-            "dataeyond__db__credential__key is not set. "
-            "Generate one with: Fernet.generate_key().decode()"
-        )
-    return Fernet(key.encode())
-
-
-def encrypt_credential(value: str) -> str:
-    """Encrypt a single credential string value."""
-    return _get_cipher().encrypt(value.encode()).decode()
-
-
-def decrypt_credential(value: str) -> str:
-    """Decrypt a single Fernet-encrypted credential string."""
-    return _get_cipher().decrypt(value.encode()).decode()
-
-
-def encrypt_credentials_dict(creds: dict) -> dict:
-    """Return a copy of the credentials dict with sensitive fields encrypted.
-
-    Call this before inserting a new DatabaseClient record.
-    """
-    cipher = _get_cipher()
-    result = dict(creds)
-    for field in SENSITIVE_FIELDS:
-        if result.get(field):
-            result[field] = cipher.encrypt(result[field].encode()).decode()
-    return result
-
-
-def decrypt_credentials_dict(creds: dict) -> dict:
-    """Return a copy of the credentials dict with sensitive fields decrypted.
-
-    Call this after fetching a DatabaseClient record from DB.
-    """
-    cipher = _get_cipher()
-    result = dict(creds)
-    for field in SENSITIVE_FIELDS:
-        if result.get(field):
-            result[field] = cipher.decrypt(result[field].encode()).decode()
-    return result
- 

uv.lock

@@ -1,5 +1,5 @@
 version = 1
-revision = 3
+revision = 2
 requires-python = "==3.12.*"
 resolution-markers = [
     "python_full_version >= '3.12.4'",
@@ -49,8 +49,6 @@ dependencies = [
     { name = "pydantic" },
     { name = "pydantic-settings" },
     { name = "pymongo" },
-    { name = "pymssql" },
-    { name = "pymysql" },
     { name = "pypdf" },
     { name = "python-docx" },
     { name = "python-dotenv" },
@@ -59,10 +57,8 @@ dependencies = [
     { name = "redis" },
     { name = "sentence-transformers" },
     { name = "slowapi" },
-    { name = "snowflake-sqlalchemy" },
     { name = "spacy" },
     { name = "sqlalchemy", extra = ["asyncio"] },
-    { name = "sqlalchemy-bigquery" },
     { name = "sse-starlette" },
     { name = "starlette" },
     { name = "structlog" },
@@ -135,8 +131,6 @@ requires-dist = [
     { name = "pydantic", specifier = "==2.10.3" },
     { name = "pydantic-settings", specifier = "==2.7.0" },
     { name = "pymongo", specifier = ">=4.14.0" },
-    { name = "pymssql", specifier = ">=2.3.0" },
-    { name = "pymysql", specifier = ">=1.1.1" },
     { name = "pypdf", specifier = "==5.1.0" },
     { name = "pytest", marker = "extra == 'dev'", specifier = "==8.3.4" },
     { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = "==0.24.0" },
@@ -149,10 +143,8 @@ requires-dist = [
     { name = "ruff", marker = "extra == 'dev'", specifier = "==0.8.4" },
     { name = "sentence-transformers", specifier = "==3.3.1" },
     { name = "slowapi", specifier = "==0.1.9" },
-    { name = "snowflake-sqlalchemy", specifier = ">=1.7.0" },
     { name = "spacy", specifier = "==3.8.3" },
     { name = "sqlalchemy", extras = ["asyncio"], specifier = "==2.0.36" },
-    { name = "sqlalchemy-bigquery", specifier = ">=1.11.0" },
     { name = "sse-starlette", specifier = "==2.1.3" },
     { name = "starlette", specifier = "==0.41.3" },
     { name = "structlog", specifier = "==24.4.0" },
@@ -288,15 +280,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/13/b5/7af0cb920a476dccd612fbc9a21a3745fb29b1fcd74636078db8f7ba294c/APScheduler-3.10.4-py3-none-any.whl", hash = "sha256:fb91e8a768632a4756a585f79ec834e0e27aad5860bac7eaa523d9ccefd87661", size = 59303, upload-time = "2023-08-19T16:44:56.814Z" },
 ]
 
-[[package]]
-name = "asn1crypto"
-version = "1.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" },
-]
-
 [[package]]
 name = "asyncpg"
 version = "0.30.0"
@@ -445,34 +428,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/07/fb43edc2ff0a6a367e4a94fc39eb3b85aa1e55e24cc857af2db145ce9f0d/blis-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f20f7ad69aaffd1ce14fe77de557b6df9b61e0c9e582f75a843715d836b5c8af", size = 6192759, upload-time = "2025-11-17T12:27:56.176Z" },
 ]
 
-[[package]]
-name = "boto3"
-version = "1.42.89"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "botocore" },
-    { name = "jmespath" },
-    { name = "s3transfer" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/0c/f7bccb22b245cabf392816baba20f9e95f78ace7dbc580fd40136e80e732/boto3-1.42.89.tar.gz", hash = "sha256:3e43aacc0801bba9bcd23a8c271c089af297a69565f783fcdd357ae0e330bf1e", size = 113165, upload-time = "2026-04-13T19:36:17.516Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b9/33/55103ba5ef9975ea54b8d39e69b76eb6e9fded3beae5f01065e26951a3a1/boto3-1.42.89-py3-none-any.whl", hash = "sha256:6204b189f4d0c655535f43d7eaa57ff4e8d965b8463c97e45952291211162932", size = 140556, upload-time = "2026-04-13T19:36:13.894Z" },
-]
-
-[[package]]
-name = "botocore"
-version = "1.42.89"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "jmespath" },
-    { name = "python-dateutil" },
-    { name = "urllib3" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/0f/cc/e6be943efa9051bd15c2ee14077c2b10d6e27c9e9385fc43a03a5c4ed8b5/botocore-1.42.89.tar.gz", hash = "sha256:95ac52f472dad29942f3088b278ab493044516c16dbf9133c975af16527baa99", size = 15206290, upload-time = "2026-04-13T19:36:02.321Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/91/f1/90a7b8eda38b7c3a65ca7ee0075bdf310b6b471cb1b95fab6e8994323a50/botocore-1.42.89-py3-none-any.whl", hash = "sha256:d9b786c8d9db6473063b4cc5be0ba7e6a381082307bd6afb69d4216f9fa95f35", size = 14887287, upload-time = "2026-04-13T19:35:56.677Z" },
-]
-
 [[package]]
 name = "cachetools"
 version = "5.5.0"
@@ -986,109 +941,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" },
 ]
 
-[[package]]
-name = "google-api-core"
-version = "2.30.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "google-auth" },
-    { name = "googleapis-common-protos" },
-    { name = "proto-plus" },
-    { name = "protobuf" },
-    { name = "requests" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" },
-]
-
-[package.optional-dependencies]
-grpc = [
-    { name = "grpcio" },
-    { name = "grpcio-status" },
-]
-
-[[package]]
-name = "google-auth"
-version = "2.49.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "cryptography" },
-    { name = "pyasn1-modules" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" },
-]
-
-[[package]]
-name = "google-cloud-bigquery"
-version = "3.41.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "google-api-core", extra = ["grpc"] },
-    { name = "google-auth" },
-    { name = "google-cloud-core" },
-    { name = "google-resumable-media" },
-    { name = "packaging" },
-    { name = "python-dateutil" },
-    { name = "requests" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ce/13/6515c7aab55a4a0cf708ffd309fb9af5bab54c13e32dc22c5acd6497193c/google_cloud_bigquery-3.41.0.tar.gz", hash = "sha256:2217e488b47ed576360c9b2cc07d59d883a54b83167c0ef37f915c26b01a06fe", size = 513434, upload-time = "2026-03-30T22:50:55.347Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/40/33/1d3902efadef9194566d499d61507e1f038454e0b55499d2d7f8ab2a4fee/google_cloud_bigquery-3.41.0-py3-none-any.whl", hash = "sha256:2a5b5a737b401cbd824a6e5eac7554100b878668d908e6548836b5d8aaa4dcaa", size = 262343, upload-time = "2026-03-30T22:48:45.444Z" },
-]
-
-[[package]]
-name = "google-cloud-core"
-version = "2.5.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "google-api-core" },
-    { name = "google-auth" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/dc/24/6ca08b0a03c7b0c620427503ab00353a4ae806b848b93bcea18b6b76fde6/google_cloud_core-2.5.1.tar.gz", hash = "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811", size = 36078, upload-time = "2026-03-30T22:50:08.057Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/73/d9/5bb050cb32826466aa9b25f79e2ca2879fe66cb76782d4ed798dd7506151/google_cloud_core-2.5.1-py3-none-any.whl", hash = "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7", size = 29452, upload-time = "2026-03-30T22:48:31.567Z" },
-]
-
-[[package]]
-name = "google-crc32c"
-version = "1.8.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" },
-    { url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" },
-    { url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" },
-    { url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" },
-    { url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" },
-]
-
-[[package]]
-name = "google-resumable-media"
-version = "2.8.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "google-crc32c" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3f/d1/b1ea14b93b6b78f57fc580125de44e9f593ab88dd2460f1a8a8d18f74754/google_resumable_media-2.8.2.tar.gz", hash = "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70", size = 2164510, upload-time = "2026-03-30T23:34:25.369Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5e/f8/50bfaf4658431ff9de45c5c3935af7ab01157a4903c603cd0eee6e78e087/google_resumable_media-2.8.2-py3-none-any.whl", hash = "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220", size = 81511, upload-time = "2026-03-30T23:34:09.671Z" },
-]
-
-[[package]]
-name = "googleapis-common-protos"
-version = "1.74.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "protobuf" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" },
-]
-
 [[package]]
 name = "greenlet"
 version = "3.3.2"
@@ -1106,41 +958,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/58/2e/fe7f36ff1982d6b10a60d5e0740c759259a7d6d2e1dc41da6d96de32fff6/greenlet-3.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:d3a62fa76a32b462a97198e4c9e99afb9ab375115e74e9a83ce180e7a496f643", size = 230331, upload-time = "2026-02-20T20:17:23.34Z" },
 ]
 
-[[package]]
-name = "grpcio"
-version = "1.80.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/e8/a2b749265eb3415abc94f2e619bbd9e9707bebdda787e61c593004ec927a/grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0", size = 6015616, upload-time = "2026-03-30T08:47:13.428Z" },
-    { url = "https://files.pythonhosted.org/packages/3e/97/b1282161a15d699d1e90c360df18d19165a045ce1c343c7f313f5e8a0b77/grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2", size = 12014204, upload-time = "2026-03-30T08:47:15.873Z" },
-    { url = "https://files.pythonhosted.org/packages/6e/5e/d319c6e997b50c155ac5a8cb12f5173d5b42677510e886d250d50264949d/grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de", size = 6563866, upload-time = "2026-03-30T08:47:18.588Z" },
-    { url = "https://files.pythonhosted.org/packages/ae/f6/fdd975a2cb4d78eb67769a7b3b3830970bfa2e919f1decf724ae4445f42c/grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921", size = 7273060, upload-time = "2026-03-30T08:47:21.113Z" },
-    { url = "https://files.pythonhosted.org/packages/db/f0/a3deb5feba60d9538a962913e37bd2e69a195f1c3376a3dd44fe0427e996/grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411", size = 6782121, upload-time = "2026-03-30T08:47:23.827Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/84/36c6dcfddc093e108141f757c407902a05085e0c328007cb090d56646cdf/grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd", size = 7383811, upload-time = "2026-03-30T08:47:26.517Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/ef/f3a77e3dc5b471a0ec86c564c98d6adfa3510d38f8ee99010410858d591e/grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f", size = 8393860, upload-time = "2026-03-30T08:47:29.439Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/8d/9d4d27ed7f33d109c50d6b5ce578a9914aa68edab75d65869a17e630a8d1/grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f", size = 7830132, upload-time = "2026-03-30T08:47:33.254Z" },
-    { url = "https://files.pythonhosted.org/packages/14/e4/9990b41c6d7a44e1e9dee8ac11d7a9802ba1378b40d77468a7761d1ad288/grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193", size = 4140904, upload-time = "2026-03-30T08:47:35.319Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/2c/296f6138caca1f4b92a31ace4ae1b87dab692fc16a7a3417af3bb3c805bf/grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff", size = 4880944, upload-time = "2026-03-30T08:47:37.831Z" },
-]
-
-[[package]]
-name = "grpcio-status"
-version = "1.80.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "googleapis-common-protos" },
-    { name = "grpcio" },
-    { name = "protobuf" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/b1/ed/105f619bdd00cb47a49aa2feea6232ea2bbb04199d52a22cc6a7d603b5cb/grpcio_status-1.80.0.tar.gz", hash = "sha256:df73802a4c89a3ea88aa2aff971e886fccce162bc2e6511408b3d67a144381cd", size = 13901, upload-time = "2026-03-30T08:54:34.784Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/76/80/58cd2dfc19a07d022abe44bde7c365627f6c7cb6f692ada6c65ca437d09a/grpcio_status-1.80.0-py3-none-any.whl", hash = "sha256:4b56990363af50dbf2c2ebb80f1967185c07d87aa25aa2bea45ddb75fc181dbe", size = 14638, upload-time = "2026-03-30T08:54:01.569Z" },
-]
-
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -1310,15 +1127,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" },
 ]
 
-[[package]]
-name = "jmespath"
-version = "1.1.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
-]
-
 [[package]]
 name = "joblib"
 version = "1.5.3"
@@ -2313,33 +2121,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" },
 ]
 
-[[package]]
-name = "proto-plus"
-version = "1.27.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "protobuf" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/81/0d/94dfe80193e79d55258345901acd2917523d56e8381bc4dee7fd38e3868a/proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24", size = 57204, upload-time = "2026-03-26T22:18:57.174Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/84/f3/1fba73eeffafc998a25d59703b63f8be4fe8a5cb12eaff7386a0ba0f7125/proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", size = 50450, upload-time = "2026-03-26T22:13:42.927Z" },
-]
-
-[[package]]
-name = "protobuf"
-version = "6.33.6"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" },
-    { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" },
-    { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" },
-]
-
 [[package]]
 name = "psycopg"
 version = "3.2.3"
@@ -2400,27 +2181,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b5/bf/635fbe5dd10ed200afbbfbe98f8602829252ca1cce81cc48fb25ed8dadc0/psycopg2-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:e03e4a6dbe87ff81540b434f2e5dc2bddad10296db5eea7bdc995bf5f4162938", size = 2713969, upload-time = "2025-10-10T11:10:15.946Z" },
 ]
 
-[[package]]
-name = "pyasn1"
-version = "0.6.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" },
-]
-
-[[package]]
-name = "pyasn1-modules"
-version = "0.4.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "pyasn1" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" },
-]
-
 [[package]]
 name = "pycparser"
 version = "3.0"
@@ -2550,43 +2310,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/60/4c/33f75713d50d5247f2258405142c0318ff32c6f8976171c4fcae87a9dbdf/pymongo-4.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:dfc320f08ea9a7ec5b2403dc4e8150636f0d6150f4b9792faaae539c88e7db3b", size = 892971, upload-time = "2026-01-07T18:04:35.594Z" },
 ]
 
-[[package]]
-name = "pymssql"
-version = "2.3.13"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/7a/cc/843c044b7f71ee329436b7327c578383e2f2499313899f88ad267cdf1f33/pymssql-2.3.13.tar.gz", hash = "sha256:2137e904b1a65546be4ccb96730a391fcd5a85aab8a0632721feb5d7e39cfbce", size = 203153, upload-time = "2026-02-14T05:00:36.865Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ba/60/a2e8a8a38f7be21d54402e2b3365cd56f1761ce9f2706c97f864e8aa8300/pymssql-2.3.13-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cf4f32b4a05b66f02cb7d55a0f3bcb0574a6f8cf0bee4bea6f7b104038364733", size = 3158689, upload-time = "2026-02-14T04:59:46.982Z" },
-    { url = "https://files.pythonhosted.org/packages/43/9e/0cf0ffb9e2f73238baf766d8e31d7237b5bee3cc1bb29a376b404610994a/pymssql-2.3.13-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:2b056eb175955f7fb715b60dc1c0c624969f4d24dbdcf804b41ab1e640a2b131", size = 2960018, upload-time = "2026-02-14T04:59:48.668Z" },
-    { url = "https://files.pythonhosted.org/packages/93/ea/bc27354feaca717faa4626911f6b19bb62985c87dda28957c63de4de5895/pymssql-2.3.13-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:319810b89aa64b99d9c5c01518752c813938df230496fa2c4c6dda0603f04c4c", size = 3065719, upload-time = "2026-02-14T04:59:50.369Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/7a/8028681c96241fb5fc850b87c8959402c353e4b83c6e049a99ffa67ded54/pymssql-2.3.13-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0ea72641cb0f8bce7ad8565dbdbda4a7437aa58bce045f2a3a788d71af2e4be", size = 3190567, upload-time = "2026-02-14T04:59:52.202Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/f1/ab5b76adbbd6db9ce746d448db34b044683522e7e7b95053f9dd0165297b/pymssql-2.3.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1493f63d213607f708a5722aa230776ada726ccdb94097fab090a1717a2534e0", size = 3710481, upload-time = "2026-02-14T04:59:54.01Z" },
-    { url = "https://files.pythonhosted.org/packages/59/aa/2fa0951475cd0a1829e0b8bfbe334d04ece4bce11546a556b005c4100689/pymssql-2.3.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb3275985c23479e952d6462ae6c8b2b6993ab6b99a92805a9c17942cf3d5b3d", size = 3453789, upload-time = "2026-02-14T04:59:56.841Z" },
-    { url = "https://files.pythonhosted.org/packages/78/08/8cd2af9003f9fc03912b658a64f5a4919dcd68f0dd3bbc822b49a3d14fd9/pymssql-2.3.13-cp312-cp312-win_amd64.whl", hash = "sha256:a930adda87bdd8351a5637cf73d6491936f34e525a5e513068a6eac742f69cdb", size = 1994709, upload-time = "2026-02-14T04:59:58.972Z" },
-]
-
-[[package]]
-name = "pymysql"
-version = "1.1.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f5/ae/1fe3fcd9f959efa0ebe200b8de88b5a5ce3e767e38c7ac32fb179f16a388/pymysql-1.1.2.tar.gz", hash = "sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03", size = 48258, upload-time = "2025-08-24T12:55:55.146Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" },
-]
-
-[[package]]
-name = "pyopenssl"
-version = "25.1.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "cryptography" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/04/8c/cd89ad05804f8e3c17dea8f178c3f40eeab5694c30e0c9f5bcd49f576fc3/pyopenssl-25.1.0.tar.gz", hash = "sha256:8d031884482e0c67ee92bf9a4d8cceb08d92aba7136432ffb0703c5280fc205b", size = 179937, upload-time = "2025-05-17T16:28:31.31Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/80/28/2659c02301b9500751f8d42f9a6632e1508aa5120de5e43042b8b30f8d5d/pyopenssl-25.1.0-py3-none-any.whl", hash = "sha256:2b11f239acc47ac2e5aca04fd7fa829800aeee22a2eb30d744572a157bd8a1ab", size = 56771, upload-time = "2025-05-17T16:28:29.197Z" },
-]
-
 [[package]]
 name = "pyparsing"
 version = "3.3.2"
@@ -2887,18 +2610,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/13/9f/026e18ca7d7766783d779dae5e9c656746c6ede36ef73c6d934aaf4a6dec/ruff-0.8.4-py3-none-win_arm64.whl", hash = "sha256:9183dd615d8df50defa8b1d9a074053891ba39025cf5ae88e8bcb52edcc4bf08", size = 9074500, upload-time = "2024-12-19T13:36:23.92Z" },
 ]
 
-[[package]]
-name = "s3transfer"
-version = "0.16.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "botocore" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" },
-]
-
 [[package]]
 name = "safetensors"
 version = "0.7.0"
@@ -3053,60 +2764,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
 ]
 
-[[package]]
-name = "snowflake-connector-python"
-version = "4.0.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "asn1crypto" },
-    { name = "boto3" },
-    { name = "botocore" },
-    { name = "certifi" },
-    { name = "charset-normalizer" },
-    { name = "cryptography" },
-    { name = "filelock" },
-    { name = "idna" },
-    { name = "packaging" },
-    { name = "platformdirs" },
-    { name = "pyjwt" },
-    { name = "pyopenssl" },
-    { name = "pytz" },
-    { name = "requests" },
-    { name = "sortedcontainers" },
-    { name = "tomlkit" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/1d/f1/4aff125021a9c5e0183f2f55dd7d04b7256a0e1e10db50d537a7415d9c55/snowflake_connector_python-4.0.0.tar.gz", hash = "sha256:4b10a865c4a5e1fa60c365c7fe41e0433605e6e5edc824e8730a9038f330b3a6", size = 813937, upload-time = "2025-10-09T10:11:34.631Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ea/b0/462c0deee35d6d03d3d729b3f923615bae665beb7f9a94673a23a52080fe/snowflake_connector_python-4.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bfd3b8523d7adc830f99c5c4c635689ceca61700a05368d5bbb34c6811f2ec54", size = 1029568, upload-time = "2025-10-09T10:11:42.125Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/4b/bb3ae3f07e7927c8f16c4c0f1283d3c721978d16e8bf4193fc8e41025c1e/snowflake_connector_python-4.0.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:835161dd46ef8f5fc9d2f135ca654c2f3fbdf57b035d3e1980506aa8eac671dc", size = 1041337, upload-time = "2025-10-09T10:11:43.692Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/75/4bfac89f10c6dbb75e97adf1e217737fc599ebf964031c9298b6cbd807d0/snowflake_connector_python-4.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65e4e36dd1b0c7235d84cddef8a3c97c5ea0dc8fea85e31e45fc485000b77a83", size = 2699730, upload-time = "2025-10-09T10:11:25.295Z" },
-    { url = "https://files.pythonhosted.org/packages/cd/78/0e916416c50909dbae511fe38b1e671a9efa62decdce51b174a0396804e4/snowflake_connector_python-4.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6132986d6965e4005b0167270612fbc7fa4bc4ef42726a40b85a8f57475a78d", size = 2731336, upload-time = "2025-10-09T10:11:27.028Z" },
-    { url = "https://files.pythonhosted.org/packages/83/f0/3db8a2f3f5ee724d309c661af739a70d0643070b9b4597728151ef900f9b/snowflake_connector_python-4.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:a790f06808e4481c23cfed1396d2c9a786060ddd62408b1fda1a63e1e6bc4b07", size = 1176292, upload-time = "2025-10-09T10:11:54.956Z" },
-]
-
-[[package]]
-name = "snowflake-sqlalchemy"
-version = "1.9.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "snowflake-connector-python" },
-    { name = "sqlalchemy" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ff/6a/fcc5c00c3a253029a7b7b293a3958ba07d5e97623b643de47be0cc9e5530/snowflake_sqlalchemy-1.9.0.tar.gz", hash = "sha256:fb32baf559f7f933ae8fde2ec535bcea5381bb15188777cd8c006b3226efa3b1", size = 141707, upload-time = "2026-03-04T13:48:17.905Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/88/28/b7ae8df80847e8157b74669ad7e1b0180e82ac0e3daf950612effd232fea/snowflake_sqlalchemy-1.9.0-py3-none-any.whl", hash = "sha256:f0b1528173e93c8c80bd9ca510985054667e0e514dd90b890271ac1cfae261c1", size = 78953, upload-time = "2026-03-04T13:48:16.393Z" },
-]
-
-[[package]]
-name = "sortedcontainers"
-version = "2.4.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
-]
-
 [[package]]
 name = "spacy"
 version = "3.8.3"
@@ -3185,22 +2842,6 @@ asyncio = [
     { name = "greenlet" },
 ]
 
-[[package]]
-name = "sqlalchemy-bigquery"
-version = "1.16.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "google-api-core" },
-    { name = "google-auth" },
-    { name = "google-cloud-bigquery" },
-    { name = "packaging" },
-    { name = "sqlalchemy" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/7e/6a/c49932b3d9c44cab9202b1866c5b36b7f0d0455d4653fbc0af4466aeaa76/sqlalchemy_bigquery-1.16.0.tar.gz", hash = "sha256:fe937a0d1f4cf7219fcf5d4995c6718805b38d4df43e29398dec5dc7b6d1987e", size = 119632, upload-time = "2025-11-06T01:35:40.373Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c0/87/11e6de00ef7949bb8ea06b55304a1a4911c329fdf0d9882b464db240c2c5/sqlalchemy_bigquery-1.16.0-py3-none-any.whl", hash = "sha256:0fe7634cd954f3e74f5e2db6d159f9e5ee87a47fbe8d52eac3cd3bb3dadb3a77", size = 40615, upload-time = "2025-11-06T01:35:39.358Z" },
-]
-
 [[package]]
 name = "srsly"
 version = "2.5.3"
@@ -3374,15 +3015,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" },
 ]
 
-[[package]]
-name = "tomlkit"
-version = "0.14.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" },
-]
-
 [[package]]
 name = "torch"
 version = "2.11.0"