Rename app to Autonomous Data Analyst and improve user-doc indexing

DEPLOY.md

@@ -1,6 +1,6 @@
 # Deploy for free (Hugging Face Spaces)
 
-Host the **Autonomous Data Scientist** Gradio app for free on [Hugging Face Spaces](https://huggingface.co/spaces). The app will use **Groq** (free LLM API) and **HuggingFace embeddings** in the cloud; no Ollama needed.
+Host the **Autonomous Data Analyst** Gradio app for free on [Hugging Face Spaces](https://huggingface.co/spaces). The app will use **Groq** (free LLM API) and **HuggingFace embeddings** in the cloud; no Ollama needed.
 
 ## 1. Get a free Groq API key
 

README.md

@@ -1,5 +1,5 @@
 ---
-title: Autonomous Data Scientist
+title: Autonomous Data Analyst
 emoji: 💻
 colorFrom: purple
 colorTo: gray

app_gradio.py

@@ -27,6 +27,7 @@ from agent_runner import (
     resume_agent,
     run_agent,
 )
+from src.vector_store import add_user_documents, get_user_docs_count
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", stream=sys.stdout)
 logger = logging.getLogger(__name__)
@@ -52,11 +53,12 @@ SAMPLE_QUESTIONS = [
     "What is the expense policy?",
     "How many PTO days do we get?",
     "What's the code review process?",
+    "What's in my uploaded documents?",
 ]
 
 
 def _normalize_chat_history(history):
-    """Convert tuple format [(u,a),...] to Gradio 6 format [{role, content}, ...]."""
+    """Convert tuple format [(u,a),...] to internal format [{role, content}, ...]."""
     if not history:
         return []
     out = []
@@ -69,6 +71,32 @@ def _normalize_chat_history(history):
     return out
 
 
+def _history_to_chatbot_value(history):
+    """Return history in the format expected by the Chatbot (Gradio 3 = list of tuples, Gradio 6 = list of dicts)."""
+    history = _normalize_chat_history(history)
+    if not history:
+        return []
+    try:
+        if gr.__version__.startswith("3."):
+            # Gradio 3: [(user, assistant), ...]
+            out = []
+            i = 0
+            while i < len(history):
+                if i + 1 < len(history) and history[i].get("role") == "user" and history[i + 1].get("role") == "assistant":
+                    out.append((history[i]["content"], history[i + 1]["content"]))
+                    i += 2
+                elif history[i].get("role") == "user":
+                    out.append((history[i]["content"], ""))
+                    i += 1
+                else:
+                    out.append(("", history[i]["content"]))
+                    i += 1
+            return out
+    except Exception:
+        pass
+    return history  # Gradio 6: list of {role, content}
+
+
 def chat_turn(
     message,
     history,
@@ -196,9 +224,9 @@ def run_tests(db_url, ollama_model):
 
 
 def build_ui():
-    with gr.Blocks(title="Autonomous Data Scientist") as demo:
+    with gr.Blocks(title="Autonomous Data Analyst") as demo:
         gr.HTML("""
-        <div class="main-title">📊 Autonomous Data Scientist</div>
+        <div class="main-title">📊 Autonomous Data Analyst</div>
         <div class="subtitle">Ask questions in plain English — get answers, tables, and charts from your data or company policies.</div>
         """)
 
@@ -240,6 +268,17 @@ def build_ui():
                 b = gr.Button(q, size="sm")
                 sample_btns.append((b, q))
 
+        gr.Markdown("### 📄 Your documents")
+        gr.Markdown("Add your own documents (resume, notes, etc.). You can search from **your documents** and **company policies** together — ask questions and answers use both. Upload PDF, TXT, or MD (PDFs need selectable text).")
+        with gr.Row():
+            user_files = gr.File(
+                label="Your documents (PDF, TXT, MD)",
+                file_count="multiple",
+                file_types=[".pdf", ".txt", ".md"],
+            )
+            index_btn = gr.Button("Add to search index", variant="secondary")
+        user_docs_status = gr.Markdown(value="", visible=True)
+
         gr.Markdown("### 2️⃣ Conversation")
         gr.Markdown("Answers and follow-up details appear here.")
         # Gradio 6: Chatbot expects list of {role, content} dicts; initialize with empty list
@@ -301,7 +340,7 @@ def build_ui():
                 table_val = df if (df is not None and not df.empty) else pd.DataFrame()
                 chart_val = viz_path if viz_path else None
                 return (
-                    new_hist,
+                    _history_to_chatbot_value(new_hist),
                     fresh_tid,
                     new_awaiting,
                     new_pending,
@@ -326,7 +365,7 @@ def build_ui():
                 else:
                     err_msg = f"Something went wrong: **{err_msg}**. Check that the database URL is set (Settings) and that the Space has finished loading, then try again."
                 return (
-                    add_turn(hist, message, err_msg),
+                    _history_to_chatbot_value(add_turn(hist, message, err_msg)),
                     fresh_tid,
                     False,
                     None,
@@ -345,7 +384,7 @@ def build_ui():
                 table_val = df if (df is not None and not df.empty) else pd.DataFrame()
                 chart_val = viz_path if viz_path else None
                 return (
-                    new_hist,
+                    _history_to_chatbot_value(new_hist),
                     new_awaiting,
                     new_pending,
                     gr.update(visible=False),
@@ -356,7 +395,7 @@ def build_ui():
                 logger.exception("Error in update_after_approve")
                 err_msg = f"Approval failed: **{str(e)}**. Please try again."
                 return (
-                    add_turn(hist, "", err_msg),
+                    _history_to_chatbot_value(add_turn(hist, "", err_msg)),
                     False,
                     None,
                     gr.update(visible=False),
@@ -371,7 +410,7 @@ def build_ui():
             try:
                 new_hist, new_awaiting, new_pending, _, _ = reject_click(hist, pending, tid, url, model)
                 return (
-                    new_hist,
+                    _history_to_chatbot_value(new_hist),
                     new_awaiting,
                     new_pending,
                     gr.update(visible=False),
@@ -382,7 +421,7 @@ def build_ui():
                 logger.exception("Error in update_after_reject")
                 err_msg = f"Reject failed: **{str(e)}**. Please try again."
                 return (
-                    add_turn(hist, "", err_msg),
+                    _history_to_chatbot_value(add_turn(hist, "", err_msg)),
                     False,
                     None,
                     gr.update(visible=False),
@@ -437,6 +476,44 @@ def build_ui():
             outputs=[chatbot, thread_id, awaiting_approval, pending_result, msg, approval_row, table_out, chart_out],
         )
 
+        def index_user_documents(files):
+            """Add uploaded files to the search index (user_documents collection). Does not affect DB."""
+            if not files:
+                return "Upload one or more PDF, TXT, or MD files, then click **Add to search index**."
+            # Gradio 3 can return: single path str, list of path strs, or list of dicts with "name" key
+            paths = []
+            for f in [files] if isinstance(files, (str, Path)) else (files or []):
+                p = None
+                if isinstance(f, dict) and f.get("name"):
+                    p = f["name"]
+                elif isinstance(f, (str, Path)):
+                    p = f
+                elif hasattr(f, "name"):
+                    p = getattr(f, "name", None)
+                if p and str(p).strip():
+                    paths.append(str(Path(p).resolve()))
+            paths = list(dict.fromkeys(paths))  # dedupe, keep order
+            if not paths:
+                return "No valid file paths from upload. Try uploading again (PDF, TXT, or MD)."
+            try:
+                n, err_detail = add_user_documents(paths)
+                if n == 0:
+                    return (
+                        f"❌ No chunks indexed. {err_detail or 'No text could be extracted.'}\n\n"
+                        "**Tip:** Save as Plain Text (.txt) or .md and upload that file."
+                    )
+                total = get_user_docs_count()
+                return f"✅ Indexed **{n}** chunks from {len(paths)} file(s). Your documents: **{total}** chunks in search index. Ask a question and answers can use your documents."
+            except Exception as e:
+                logger.exception("Error indexing user documents")
+                return f"❌ Indexing failed: **{str(e)}**"
+
+        index_btn.click(
+            fn=index_user_documents,
+            inputs=[user_files],
+            outputs=[user_docs_status],
+        )
+
         def toggle_guide():
             content = load_guide()
             return gr.update(value=content, visible=True)
@@ -446,9 +523,42 @@ def build_ui():
         for b, q in sample_btns:
             b.click(fn=lambda q=q: q, inputs=[], outputs=[msg])
 
+        def user_docs_status_on_load():
+            n = get_user_docs_count()
+            if n > 0:
+                return "Your documents: **%d** chunks in search index. Ask questions to search them." % n
+            return ""
+
+        demo.load(fn=user_docs_status_on_load, inputs=[], outputs=[user_docs_status])
+
     return demo
 
 
 if __name__ == "__main__":
     demo = build_ui()
-    demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False, theme=THEME, css=CSS)
+    # Gradio 3.x has no theme/css in launch(); Gradio 6 does. Try ports 7860-7870 if one is in use.
+    use_theme = hasattr(gr, "__version__") and not str(getattr(gr, "__version__", "")).startswith("3.")
+    last_err = None
+    for port in range(7860, 7871):
+        try:
+            if use_theme:
+                demo.launch(server_name="0.0.0.0", server_port=port, show_api=False, theme=THEME, css=CSS)
+            else:
+                demo.launch(server_name="0.0.0.0", server_port=port, show_api=False)
+            break  # launch() blocks; if we get here it exited normally
+        except TypeError:
+            use_theme = False
+            try:
+                demo.launch(server_name="0.0.0.0", server_port=port, show_api=False)
+            except OSError as e:
+                last_err = e
+                if "address already in use" in str(e).lower() or "48" in str(e) or "empty port" in str(e).lower():
+                    continue
+                raise
+        except OSError as e:
+            last_err = e
+            if "address already in use" in str(e).lower() or "48" in str(e) or "empty port" in str(e).lower():
+                continue
+            raise
+    if last_err is not None:
+        raise OSError(f"Ports 7860-7870 are in use. Stop the other Gradio process or use a different port.") from last_err

docs/DATA_AND_POLICIES_GUIDE.md

@@ -1,6 +1,6 @@
 # Data & Policy Guide
 
-Use this guide to understand what data and policies you can ask about. Ask questions in plain English; the Autonomous Data Scientist will use this information to answer you.
+Use this guide to understand what data and policies you can ask about. Ask questions in plain English; the Autonomous Data Analyst will use this information to answer you.
 
 ---
 
@@ -154,4 +154,4 @@ Policy content is taken from company documents (e.g. PDFs) stored in the system.
 
 ---
 
-*This guide is used by the Autonomous Data Scientist to answer your questions. You don’t need to write SQL or know table names — just ask in plain English.*
+*This guide is used by the Autonomous Data Analyst to answer your questions. You don’t need to write SQL or know table names — just ask in plain English.*

requirements.txt

@@ -13,6 +13,12 @@ python-dotenv>=1.0.0
 Faker>=30.0.0
 chromadb>=0.4.0
 langchain-chroma>=0.1.0
+langchain-text-splitters>=0.2.0
+pypdf>=4.0.0
+# Optional: for image-only (scanned) PDFs — also install poppler and tesseract on your system
+# pdf2image>=1.16.0
+# pytesseract>=0.3.10
+# Pillow>=10.0.0
 streamlit>=1.28.0
 # Gradio 4.x has gradio_client bug (TypeError in json_schema_to_python_type); use 3.x
 gradio>=3.50,<4

src/agent.py

@@ -179,7 +179,11 @@ def create_agent(
             prompt = ChatPromptTemplate.from_messages([
                 (
                     "system",
-                    "Answer the user's question based only on the provided context. Be concise. Use the exact policy details from the context. Do not say 'unknown'.",
+                    "You answer only from the provided context. The context is plain text — never say it is 'encoded', 'unreadable', or 'not available'. "
+                    "If you see a section 'From your documents:' use it to answer questions about the user (where they worked, experience, dates, resume). "
+                    "If you see 'From company policies:' use it for policy questions. Use exact wording from the context. "
+                    "If 'From your documents:' is missing entirely, say: 'No uploaded documents in the context. Add your document and click Add to search index, then try again.' "
+                    "If it is present but does not contain the answer, say what you can from it or that the specific detail was not found. Be concise.",
                 ),
                 ("human", "Context:\n{context}\n\nQuestion: {question}"),
             ])

src/vector_store.py

@@ -1,17 +1,26 @@
 """
-ChromaDB vector store for company policies/documentation.
+ChromaDB vector store for company policies/documentation and user-uploaded documents.
 Uses Ollama embeddings locally; uses HuggingFace embeddings when USE_HF_EMBEDDINGS or GROQ_API_KEY is set (cloud).
+User documents are stored in a separate collection and do not affect DB or policy data.
 """
 
+import logging
 import os
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional, Tuple, Union
+
+logger = logging.getLogger(__name__)
 
 from langchain_chroma import Chroma
 from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
 
 COLLECTION_NAME = "company_policies"
-PERSIST_DIR = Path(__file__).resolve().parent.parent / "data" / "chroma_db"
+USER_DOCUMENTS_COLLECTION = "user_documents"
+# Use absolute path so index and retrieve always use the same store (same process or restarts)
+PERSIST_DIR = (Path(__file__).resolve().parent.parent / "data" / "chroma_db").resolve()
+# HuggingFace/sentence-transformers have a max batch token limit (~5461); add docs in small batches
+EMBED_BATCH_SIZE = 50
 
 # Policy documents for auto-seed when running in cloud (no pre-seeded Chroma)
 SAMPLE_POLICIES = [
@@ -58,9 +67,10 @@ def _get_embeddings():
 def get_vector_store(
     collection_name: str = COLLECTION_NAME,
     persist_directory: Optional[Path] = None,
+    seed_if_empty: bool = True,
 ) -> Chroma:
     """Return Chroma vector store. Auto-seeds with policy docs when empty (e.g. on first cloud run)."""
-    persist = persist_directory or PERSIST_DIR
+    persist = (persist_directory or PERSIST_DIR).resolve()
     persist.mkdir(parents=True, exist_ok=True)
     embeddings = _get_embeddings()
     store = Chroma(
@@ -68,21 +78,223 @@ def get_vector_store(
         embedding_function=embeddings,
         persist_directory=str(persist),
     )
-    # On cloud, Chroma is often empty; seed from SAMPLE_POLICIES once
+    # On cloud, Chroma is often empty; seed from SAMPLE_POLICIES once (only for company_policies)
+    if seed_if_empty and collection_name == COLLECTION_NAME:
+        try:
+            if store._collection.count() == 0:
+                store.add_documents(SAMPLE_POLICIES)
+        except Exception:
+            pass
+    return store
+
+
+def _sanitize_text(text: str) -> str:
+    """Strip control chars and normalize whitespace so stored text is readable (not 'encoded')."""
+    if not text:
+        return ""
+    # Remove control characters except newline and tab
+    cleaned = "".join(c for c in text if c == "\n" or c == "\t" or (ord(c) >= 32 and ord(c) != 127))
+    # Normalize line breaks and collapse multiple spaces
+    lines = [line.strip() for line in cleaned.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+def _extract_text_from_file(file_path: Union[str, Path]) -> str:
+    """Extract raw text from a file. Supports .txt, .md, .pdf. For image-only PDFs, tries OCR if available."""
+    path = Path(file_path)
+    if not path.exists():
+        logger.warning("File does not exist: %s", path)
+        return ""
+    suffix = path.suffix.lower()
+    if suffix == ".pdf":
+        try:
+            from pypdf import PdfReader
+            reader = PdfReader(str(path))
+            text = "\n".join(page.extract_text() or "" for page in reader.pages)
+            if text.strip():
+                return _sanitize_text(text)
+            # No text from pypdf: try OCR for image-only (scanned) PDFs
+            logger.info("PDF has no selectable text; attempting OCR (requires pdf2image, pytesseract, Pillow).")
+            try:
+                from pdf2image import convert_from_path
+                import pytesseract
+                images = convert_from_path(str(path), dpi=200)
+                text = "\n".join(pytesseract.image_to_string(img) for img in images)
+                if text.strip():
+                    logger.info("OCR extracted text from %s", path.name)
+                    return _sanitize_text(text)
+            except ImportError as ie:
+                logger.warning("OCR dependencies not installed (pdf2image, pytesseract, Pillow). For image PDFs, install them or upload TXT/MD. %s", ie)
+            except Exception as ocr_err:
+                logger.warning("OCR failed for %s: %s", path.name, ocr_err)
+            return ""
+        except Exception as e:
+            logger.warning("PDF extraction failed for %s: %s", path.name, e)
+            return ""
+    if suffix in (".txt", ".md", ".markdown"):
+        raw = path.read_text(encoding="utf-8", errors="replace")
+        return _sanitize_text(raw)
+    # Fallback: try reading as text
+    raw = path.read_text(encoding="utf-8", errors="replace")
+    return _sanitize_text(raw)
+
+
+def add_user_documents(
+    files: List[Union[str, Path]],
+    chunk_size: int = 600,
+    chunk_overlap: int = 100,
+) -> Tuple[int, Optional[str]]:
+    """
+    Add user-uploaded files to the user_documents collection.
+    Extracts text, chunks, embeds, and stores. Does not affect DB or company_policies.
+    Returns (number of chunks added, error_detail or None).
+    """
+    if not files:
+        return 0, "No files provided."
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+    docs: List[Document] = []
+    seen_paths: List[str] = []
+    skipped_no_exist: List[str] = []
+    skipped_no_text: List[str] = []
+    for fp in files:
+        path = Path(fp)
+        pstr = str(path.resolve())
+        if pstr in seen_paths:
+            continue
+        seen_paths.append(pstr)
+        if not path.exists():
+            skipped_no_exist.append(path.name or pstr)
+            logger.warning("Skipping missing file: %s", pstr)
+            continue
+        text = _extract_text_from_file(path)
+        if not text.strip():
+            skipped_no_text.append(path.name or pstr)
+            continue
+        chunks = splitter.split_text(text)
+        for i, chunk in enumerate(chunks):
+            docs.append(Document(
+                page_content=chunk.strip(),
+                metadata={"source": path.name, "chunk": i, "type": "user_upload"},
+            ))
+    if not docs:
+        reasons = []
+        if skipped_no_exist:
+            reasons.append(f"{len(skipped_no_exist)} file(s) not found on disk (temp file may have been removed)")
+        if skipped_no_text:
+            reasons.append(f"{len(skipped_no_text)} file(s) had no extractable text (empty or image-only PDF)")
+        if not skipped_no_exist and not skipped_no_text and seen_paths:
+            reasons.append("No text could be extracted from the given paths.")
+        return 0, " ".join(reasons) if reasons else "No valid content to index."
+    persist_path = str(PERSIST_DIR.resolve())
+    store = get_vector_store(collection_name=USER_DOCUMENTS_COLLECTION, seed_if_empty=False)
+    for i in range(0, len(docs), EMBED_BATCH_SIZE):
+        batch = docs[i : i + EMBED_BATCH_SIZE]
+        store.add_documents(batch)
+    total = store._collection.count()
+    logger.info("Added %d chunks from user docs; user_documents collection now has %d total (persist: %s)", len(docs), total, persist_path)
+    return len(docs), None
+
+
+def add_user_text(
+    text: str,
+    source_name: str = "pasted",
+    chunk_size: int = 600,
+    chunk_overlap: int = 100,
+) -> Tuple[int, Optional[str]]:
+    """
+    Add pasted text (e.g. resume) to the user_documents collection. No file needed.
+    Returns (chunks added, error_detail or None).
+    """
+    cleaned = _sanitize_text(text or "")
+    if not cleaned.strip():
+        return 0, "No text to index. Paste your resume or document text and try again."
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+        separators=["\n\n", "\n", ". ", " ", ""],
+    )
+    chunks = splitter.split_text(cleaned)
+    docs = [
+        Document(page_content=c.strip(), metadata={"source": source_name, "chunk": i, "type": "user_upload"})
+        for i, c in enumerate(chunks) if c.strip()
+    ]
+    if not docs:
+        return 0, "No chunks produced from the text."
+    store = get_vector_store(collection_name=USER_DOCUMENTS_COLLECTION, seed_if_empty=False)
+    for i in range(0, len(docs), EMBED_BATCH_SIZE):
+        batch = docs[i : i + EMBED_BATCH_SIZE]
+        store.add_documents(batch)
+    total = store._collection.count()
+    logger.info("Added %d chunks from pasted text; user_documents has %d total", len(docs), total)
+    return len(docs), None
+
+
+def get_user_docs_count() -> int:
+    """Return number of chunks in the user_documents collection (for UI status)."""
     try:
-        if store._collection.count() == 0:
-            store.add_documents(SAMPLE_POLICIES)
+        store = get_vector_store(collection_name=USER_DOCUMENTS_COLLECTION, seed_if_empty=False)
+        return store._collection.count()
     except Exception:
-        pass
-    return store
+        return 0
 
 
 def retrieve_context(
     query: str,
-    k: int = 4,
+    k: int = 6,
     collection_name: str = COLLECTION_NAME,
+    include_user_docs: bool = True,
 ) -> str:
-    """Retrieve relevant docs from vector store and return as concatenated string."""
-    store = get_vector_store(collection_name=collection_name)
-    docs = store.similarity_search(query, k=k)
-    return "\n\n---\n\n".join(doc.page_content for doc in docs) if docs else ""
+    """
+    Retrieve relevant docs from vector store(s) and return as concatenated string with clear labels.
+    User documents are returned FIRST and with more chunks so resume/personal doc questions are answered from uploads.
+    """
+    user_parts: List[str] = []
+    policy_parts: List[str] = []
+    k_user = 8   # more chunks from resume/your docs so "where did X work" finds the right section
+    k_policy = 2 # fewer policy chunks when user has uploads so your doc content isn't drowned out
+    # User documents FIRST (so model prioritizes your resume over company policies)
+    if include_user_docs:
+        try:
+            store_user = get_vector_store(collection_name=USER_DOCUMENTS_COLLECTION, seed_if_empty=False)
+            n_user = store_user._collection.count()
+            if n_user > 0:
+                docs_u = store_user.similarity_search(query, k=min(k_user, n_user))
+                fallback_query = "work experience employment job history company role position"
+                docs_u2 = store_user.similarity_search(fallback_query, k=min(4, n_user))
+                seen: dict = {}
+                for doc in (docs_u or []) + (docs_u2 or []):
+                    clean = _sanitize_text(doc.page_content or "").strip()
+                    if clean:
+                        seen[clean] = clean
+                # If still nothing (e.g. all chunks sanitized away or poor match), get any chunks so "From your documents" is never empty
+                if not seen:
+                    docs_u3 = store_user.similarity_search("document text content", k=min(6, n_user))
+                    for doc in (docs_u3 or []):
+                        clean = _sanitize_text(doc.page_content or "").strip()
+                        if clean:
+                            seen[clean] = clean
+                all_user_content = list(seen.values()) if seen else []
+                if all_user_content:
+                    user_text = "\n\n".join(all_user_content)
+                    user_parts.append("**From your documents:**\n\n" + user_text)
+                    logger.info("User docs: %d in collection, %d chunks in context (path=%s)", n_user, len(all_user_content), str(PERSIST_DIR))
+                else:
+                    logger.warning("User docs: %d in collection but no chunks retrieved for query (path=%s)", n_user, str(PERSIST_DIR))
+        except Exception as e:
+            logger.warning("Could not retrieve from user_documents: %s", e)
+    # Company policies second
+    store_policies = get_vector_store(collection_name=COLLECTION_NAME)
+    docs_p = store_policies.similarity_search(query, k=k_policy)
+    if docs_p:
+        policy_text = "\n\n".join(_sanitize_text(doc.page_content or "").strip() for doc in docs_p if _sanitize_text(doc.page_content or "").strip())
+        if policy_text:
+            policy_parts.append("**From company policies:**\n\n" + policy_text)
+    # Your documents first, then policies
+    parts = user_parts + policy_parts
+    return "\n\n---\n\n".join(parts) if parts else ""