Persist state across restarts and avoid readability js crash loop

README.md

@@ -192,6 +192,19 @@ For dynamic websites, DeerFlow also supports the `agent_browser` tool (based on
   # agent-browser install --with-deps
   ```
 
+#### Persistent Storage (Space / External DB)
+
+By default, local `langgraph dev` state is in-memory and will be lost after restart.
+
+This Space startup now supports persistent state with:
+- `DEER_FLOW_HOME` (defaults to `/data/deer-flow`): stores thread files, artifacts, and memory
+- `DEER_FLOW_EXTENSIONS_CONFIG_PATH`: stores MCP/skills enable-state JSON
+- LangGraph checkpointer:
+  - `LANGGRAPH_CHECKPOINT_POSTGRES_URI` (recommended external DB, PostgreSQL)
+  - fallback: `LANGGRAPH_CHECKPOINT_SQLITE_PATH` (SQLite file, defaults to `${DEER_FLOW_HOME}/checkpoints.sqlite`)
+
+If you enable HF persistent storage, `/data` survives restarts, so memory/skills/thread artifacts and SQLite checkpoints persist.
+
 ## From Deep Research to Super Agent Harness
 
 DeerFlow started as a Deep Research framework — and the community ran with it. Since launch, developers have pushed it far beyond research: building data pipelines, generating slide decks, spinning up dashboards, automating content workflows. Things we never anticipated.

backend/checkpointer.py

@@ -0,0 +1,58 @@
+"""LangGraph checkpointer bootstrap for persistent thread state.
+
+Resolution order:
+1. `LANGGRAPH_CHECKPOINT_POSTGRES_URI` -> Postgres checkpointer
+2. `LANGGRAPH_CHECKPOINT_SQLITE_PATH`  -> SQLite checkpointer
+3. Fallback to `${DEER_FLOW_HOME}/checkpoints.sqlite`
+"""
+
+import atexit
+import os
+from pathlib import Path
+
+from langgraph.checkpoint.postgres import PostgresSaver
+from langgraph.checkpoint.sqlite import SqliteSaver
+
+_ctx = None
+
+
+def _default_sqlite_path() -> str:
+    deer_flow_home = os.getenv("DEER_FLOW_HOME")
+    if deer_flow_home:
+        base = Path(deer_flow_home)
+    else:
+        base = Path.cwd() / ".deer-flow"
+    base.mkdir(parents=True, exist_ok=True)
+    return str((base / "checkpoints.sqlite").resolve())
+
+
+def _init_checkpointer():
+    global _ctx
+
+    postgres_uri = os.getenv("LANGGRAPH_CHECKPOINT_POSTGRES_URI", "").strip()
+    if postgres_uri:
+        _ctx = PostgresSaver.from_conn_string(postgres_uri)
+        saver = _ctx.__enter__()
+        saver.setup()
+        return saver
+
+    sqlite_path = os.getenv("LANGGRAPH_CHECKPOINT_SQLITE_PATH", "").strip() or _default_sqlite_path()
+    sqlite_file = Path(sqlite_path).expanduser().resolve()
+    sqlite_file.parent.mkdir(parents=True, exist_ok=True)
+    _ctx = SqliteSaver.from_conn_string(str(sqlite_file))
+    saver = _ctx.__enter__()
+    saver.setup()
+    return saver
+
+
+def _close_context() -> None:
+    global _ctx
+    if _ctx is not None:
+        try:
+            _ctx.__exit__(None, None, None)
+        finally:
+            _ctx = None
+
+
+checkpointer = _init_checkpointer()
+atexit.register(_close_context)

backend/docs/CONFIGURATION.md

@@ -198,6 +198,10 @@ models:
 - `NOVITA_API_KEY` - Novita API key (OpenAI-compatible endpoint)
 - `TAVILY_API_KEY` - Tavily search API key
 - `DEER_FLOW_CONFIG_PATH` - Custom config file path
+- `DEER_FLOW_HOME` - Base directory for memory/thread files (default path root for persistence)
+- `DEER_FLOW_EXTENSIONS_CONFIG_PATH` - Path to `extensions_config.json` (MCP + skills state)
+- `LANGGRAPH_CHECKPOINT_POSTGRES_URI` - PostgreSQL URI for LangGraph checkpointer
+- `LANGGRAPH_CHECKPOINT_SQLITE_PATH` - SQLite file path for LangGraph checkpointer
 
 ## Configuration Location
 

backend/langgraph.json

@@ -4,7 +4,8 @@
     "."
   ],
   "env": ".env",
+  "checkpointer": "checkpointer:checkpointer",
   "graphs": {
     "lead_agent": "src.agents:make_lead_agent"
   }
-}
+}

backend/pyproject.toml

@@ -15,6 +15,8 @@ dependencies = [
     "langchain-mcp-adapters>=0.1.0",
     "langchain-openai>=1.1.7",
     "langgraph>=1.0.6",
+    "langgraph-checkpoint-postgres>=3.0.4",
+    "langgraph-checkpoint-sqlite>=3.0.3",
     "langgraph-cli[inmem]>=0.4.11",
     "markdownify>=1.2.2",
     "markitdown[all,xlsx]>=0.0.1a2",

backend/src/utils/readability.py

@@ -1,6 +1,7 @@
 import logging
 import re
 import subprocess
+from functools import lru_cache
 from urllib.parse import urljoin
 
 from markdownify import markdownify as md
@@ -9,6 +10,36 @@ from readabilipy import simple_json_from_html_string
 logger = logging.getLogger(__name__)
 
 
+@lru_cache(maxsize=1)
+def _is_readability_js_available() -> bool:
+    """Check whether Node + @mozilla/readability is available in runtime."""
+    try:
+        subprocess.run(
+            ["node", "-e", "require.resolve('@mozilla/readability')"],
+            capture_output=True,
+            text=True,
+            check=True,
+            timeout=5,
+        )
+        return True
+    except FileNotFoundError:
+        logger.warning("Node.js is not available; readability extraction will use pure-Python mode")
+        return False
+    except subprocess.TimeoutExpired:
+        logger.warning("Node.js readability check timed out; readability extraction will use pure-Python mode")
+        return False
+    except subprocess.CalledProcessError as exc:
+        stderr_info = (exc.stderr or "").strip()
+        if stderr_info:
+            logger.warning(
+                "Node readability module is unavailable (%s); readability extraction will use pure-Python mode",
+                stderr_info,
+            )
+        else:
+            logger.warning("Node readability module is unavailable; readability extraction will use pure-Python mode")
+        return False
+
+
 class Article:
     url: str
 
@@ -57,8 +88,9 @@ class Article:
 
 class ReadabilityExtractor:
     def extract_article(self, html: str) -> Article:
+        use_readability_js = _is_readability_js_available()
         try:
-            article = simple_json_from_html_string(html, use_readability=True)
+            article = simple_json_from_html_string(html, use_readability=use_readability_js)
         except (subprocess.CalledProcessError, FileNotFoundError) as exc:
             stderr = getattr(exc, "stderr", None)
             if isinstance(stderr, bytes):

backend/tests/test_readability.py

@@ -11,6 +11,7 @@ def test_extract_article_falls_back_when_readability_js_fails(monkeypatch):
     """When Node-based readability fails, extraction should fall back to Python mode."""
 
     calls: list[bool] = []
+    monkeypatch.setattr("src.utils.readability._is_readability_js_available", lambda: True)
 
     def _fake_simple_json_from_html_string(html: str, use_readability: bool = False):
         calls.append(use_readability)
@@ -38,6 +39,7 @@ def test_extract_article_re_raises_unexpected_exception(monkeypatch):
     """Unexpected errors should be surfaced instead of silently falling back."""
 
     calls: list[bool] = []
+    monkeypatch.setattr("src.utils.readability._is_readability_js_available", lambda: True)
 
     def _fake_simple_json_from_html_string(html: str, use_readability: bool = False):
         calls.append(use_readability)
@@ -53,3 +55,23 @@ def test_extract_article_re_raises_unexpected_exception(monkeypatch):
     with pytest.raises(RuntimeError, match="unexpected parser failure"):
         ReadabilityExtractor().extract_article("<html><body>test</body></html>")
     assert calls == [True]
+
+
+def test_extract_article_uses_python_path_when_js_unavailable(monkeypatch):
+    """If JS readability is unavailable, extractor should directly use Python mode."""
+
+    calls: list[bool] = []
+    monkeypatch.setattr("src.utils.readability._is_readability_js_available", lambda: False)
+
+    def _fake_simple_json_from_html_string(html: str, use_readability: bool = False):
+        calls.append(use_readability)
+        return {"title": "Python Mode", "content": "<p>OK</p>"}
+
+    monkeypatch.setattr(
+        "src.utils.readability.simple_json_from_html_string",
+        _fake_simple_json_from_html_string,
+    )
+
+    article = ReadabilityExtractor().extract_article("<html><body>test</body></html>")
+    assert calls == [False]
+    assert article.title == "Python Mode"

backend/uv.lock

@@ -131,6 +131,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" },
 ]
 
+[[package]]
+name = "aiosqlite"
+version = "0.22.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4e/8a/64761f4005f17809769d23e518d915db74e6310474e733e3593cfc854ef1/aiosqlite-0.22.1.tar.gz", hash = "sha256:043e0bd78d32888c0a9ca90fc788b38796843360c855a7262a532813133a0650", size = 14821, upload-time = "2025-12-23T19:25:43.997Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" },
+]
+
 [[package]]
 name = "annotated-doc"
 version = "0.0.4"
@@ -610,6 +619,8 @@ dependencies = [
     { name = "langchain-mcp-adapters" },
     { name = "langchain-openai" },
     { name = "langgraph" },
+    { name = "langgraph-checkpoint-postgres" },
+    { name = "langgraph-checkpoint-sqlite" },
     { name = "langgraph-cli", extra = ["inmem"] },
     { name = "markdownify" },
     { name = "markitdown", extra = ["all", "xlsx"] },
@@ -644,6 +655,8 @@ requires-dist = [
     { name = "langchain-mcp-adapters", specifier = ">=0.1.0" },
     { name = "langchain-openai", specifier = ">=1.1.7" },
     { name = "langgraph", specifier = ">=1.0.6" },
+    { name = "langgraph-checkpoint-postgres", specifier = ">=3.0.4" },
+    { name = "langgraph-checkpoint-sqlite", specifier = ">=3.0.3" },
     { name = "langgraph-cli", extras = ["inmem"], specifier = ">=0.4.11" },
     { name = "markdownify", specifier = ">=1.2.2" },
     { name = "markitdown", extras = ["all", "xlsx"], specifier = ">=0.0.1a2" },
@@ -1475,6 +1488,35 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4a/de/ddd53b7032e623f3c7bcdab2b44e8bf635e468f62e10e5ff1946f62c9356/langgraph_checkpoint-4.0.0-py3-none-any.whl", hash = "sha256:3fa9b2635a7c5ac28b338f631abf6a030c3b508b7b9ce17c22611513b589c784", size = 46329, upload-time = "2026-01-12T20:30:25.2Z" },
 ]
 
+[[package]]
+name = "langgraph-checkpoint-postgres"
+version = "3.0.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "langgraph-checkpoint" },
+    { name = "orjson" },
+    { name = "psycopg" },
+    { name = "psycopg-pool" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f5/39/6a409958bd1e4e0804bbe4f9351e620f6087d5346e452c59824298a2a330/langgraph_checkpoint_postgres-3.0.4.tar.gz", hash = "sha256:83e6a1097563369173442de2a66e6d712d60a1a6de07c98c5130d476bb2b76ae", size = 127627, upload-time = "2026-01-31T00:44:16.478Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/14/56/7466f596add278798ab42697a56e992adde6866664afff6a5e4432540f29/langgraph_checkpoint_postgres-3.0.4-py3-none-any.whl", hash = "sha256:12cd5661da2a374882770deb9008a4eb16641c3fd38d7595e312030080390c6e", size = 42834, upload-time = "2026-01-31T00:44:15.118Z" },
+]
+
+[[package]]
+name = "langgraph-checkpoint-sqlite"
+version = "3.0.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiosqlite" },
+    { name = "langgraph-checkpoint" },
+    { name = "sqlite-vec" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/04/61/40b7f8f29d6de92406e668c35265f409f57064907e31eae84ab3f2a3e3e1/langgraph_checkpoint_sqlite-3.0.3.tar.gz", hash = "sha256:438c234d37dabda979218954c9c6eb1db73bee6492c2f1d3a00552fe23fa34ed", size = 123876, upload-time = "2026-01-19T00:38:44.473Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a3/d8/84ef22ee1cc485c4910df450108fd5e246497379522b3c6cfba896f71bf6/langgraph_checkpoint_sqlite-3.0.3-py3-none-any.whl", hash = "sha256:02eb683a79aa6fcda7cd4de43861062a5d160dbbb990ef8a9fd76c979998a952", size = 33593, upload-time = "2026-01-19T00:38:43.288Z" },
+]
+
 [[package]]
 name = "langgraph-cli"
 version = "0.4.11"
@@ -2485,6 +2527,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/57/bf/2086963c69bdac3d7cff1cc7ff79b8ce5ea0bec6797a017e1be338a46248/protobuf-6.33.5-py3-none-any.whl", hash = "sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02", size = 170687, upload-time = "2026-01-29T21:51:32.557Z" },
 ]
 
+[[package]]
+name = "psycopg"
+version = "3.3.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d3/b6/379d0a960f8f435ec78720462fd94c4863e7a31237cf81bf76d0af5883bf/psycopg-3.3.3.tar.gz", hash = "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9", size = 165624, upload-time = "2026-02-18T16:52:16.546Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c8/5b/181e2e3becb7672b502f0ed7f16ed7352aca7c109cfb94cf3878a9186db9/psycopg-3.3.3-py3-none-any.whl", hash = "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698", size = 212768, upload-time = "2026-02-18T16:46:27.365Z" },
+]
+
+[[package]]
+name = "psycopg-pool"
+version = "3.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/56/9a/9470d013d0d50af0da9c4251614aeb3c1823635cab3edc211e3839db0bcf/psycopg_pool-3.3.0.tar.gz", hash = "sha256:fa115eb2860bd88fce1717d75611f41490dec6135efb619611142b24da3f6db5", size = 31606, upload-time = "2025-12-01T11:34:33.11Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e7/c3/26b8a0908a9db249de3b4169692e1c7c19048a9bc41a4d3209cee7dbb758/psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063", size = 39995, upload-time = "2025-12-01T11:34:29.761Z" },
+]
+
 [[package]]
 name = "pycparser"
 version = "2.23"
@@ -3100,6 +3167,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b8/a7/903429719d39ac2c42aa37086c90e816d883560f13c87d51f09a2962e021/speechrecognition-3.14.5-py3-none-any.whl", hash = "sha256:0c496d74e9f29b1daadb0d96f5660f47563e42bf09316dacdd57094c5095977e", size = 32856308, upload-time = "2025-12-31T11:25:41.161Z" },
 ]
 
+[[package]]
+name = "sqlite-vec"
+version = "0.1.6"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/88/ed/aabc328f29ee6814033d008ec43e44f2c595447d9cccd5f2aabe60df2933/sqlite_vec-0.1.6-py3-none-macosx_10_6_x86_64.whl", hash = "sha256:77491bcaa6d496f2acb5cc0d0ff0b8964434f141523c121e313f9a7d8088dee3", size = 164075, upload-time = "2024-11-20T16:40:29.847Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/57/05604e509a129b22e303758bfa062c19afb020557d5e19b008c64016704e/sqlite_vec-0.1.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fdca35f7ee3243668a055255d4dee4dea7eed5a06da8cad409f89facf4595361", size = 165242, upload-time = "2024-11-20T16:40:31.206Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/48/dbb2cc4e5bad88c89c7bb296e2d0a8df58aab9edc75853728c361eefc24f/sqlite_vec-0.1.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b0519d9cd96164cd2e08e8eed225197f9cd2f0be82cb04567692a0a4be02da3", size = 103704, upload-time = "2024-11-20T16:40:33.729Z" },
+    { url = "https://files.pythonhosted.org/packages/80/76/97f33b1a2446f6ae55e59b33869bed4eafaf59b7f4c662c8d9491b6a714a/sqlite_vec-0.1.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl", hash = "sha256:823b0493add80d7fe82ab0fe25df7c0703f4752941aee1c7b2b02cec9656cb24", size = 151556, upload-time = "2024-11-20T16:40:35.387Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/98/e8bc58b178266eae2fcf4c9c7a8303a8d41164d781b32d71097924a6bebe/sqlite_vec-0.1.6-py3-none-win_amd64.whl", hash = "sha256:c65bcfd90fa2f41f9000052bcb8bb75d38240b2dae49225389eca6c3136d3f0c", size = 281540, upload-time = "2024-11-20T16:40:37.296Z" },
+]
+
 [[package]]
 name = "sse-starlette"
 version = "2.1.3"

start-hf.sh

@@ -14,7 +14,26 @@ fi
 
 export OPENAI_API_KEY
 
-mkdir -p /app/logs /app/backend/.deer-flow/threads /app/backend/.deer-flow/artifacts
+PERSIST_ROOT="${DEER_FLOW_HOME:-/data/deer-flow}"
+if ! mkdir -p "${PERSIST_ROOT}" 2>/dev/null; then
+    PERSIST_ROOT="/app/backend/.deer-flow"
+    mkdir -p "${PERSIST_ROOT}"
+fi
+
+export DEER_FLOW_HOME="${PERSIST_ROOT}"
+export DEER_FLOW_EXTENSIONS_CONFIG_PATH="${DEER_FLOW_EXTENSIONS_CONFIG_PATH:-${DEER_FLOW_HOME}/extensions_config.json}"
+export LANGGRAPH_CHECKPOINT_SQLITE_PATH="${LANGGRAPH_CHECKPOINT_SQLITE_PATH:-${DEER_FLOW_HOME}/checkpoints.sqlite}"
+export DEER_FLOW_MEMORY_PATH="${DEER_FLOW_MEMORY_PATH:-${DEER_FLOW_HOME}/memory.json}"
+export DEER_FLOW_SKILLS_PATH="${DEER_FLOW_SKILLS_PATH:-${DEER_FLOW_HOME}/skills}"
+
+mkdir -p /app/logs "${DEER_FLOW_HOME}"
+if [ ! -f "${DEER_FLOW_EXTENSIONS_CONFIG_PATH}" ]; then
+    cp /app/extensions_config.example.json "${DEER_FLOW_EXTENSIONS_CONFIG_PATH}"
+fi
+if [ ! -d "${DEER_FLOW_SKILLS_PATH}" ]; then
+    cp -R /app/skills "${DEER_FLOW_SKILLS_PATH}"
+fi
+
 touch /app/.env /app/backend/.env /app/frontend/.env
 
 if [ -n "${SPACE_PASSWORD}" ]; then
@@ -86,6 +105,14 @@ tools:
     group: bash
     use: src.sandbox.tools:bash_tool
 
+skills:
+  path: ${DEER_FLOW_SKILLS_PATH}
+  container_path: /mnt/skills
+
+memory:
+  enabled: true
+  storage_path: ${DEER_FLOW_MEMORY_PATH}
+
 sandbox:
   use: src.sandbox.local:LocalSandboxProvider
 EOF