sync_data.sh

#!/bin/bash
# OpenWebUI Data Sync Script
# Syncs webui.db, uploads/, and vector_db/ to GitHub with optional WebDAV backup

set -e

# ============================================
# CONFIGURATION
# ============================================

# Set timezone
export TZ="${SYNC_TIMEZONE:-Asia/Kolkata}"

# Absolute paths (critical for reliability)
APP_DIR="/app/backend"
DATA_DIR="${APP_DIR}/data"
GITHUB_DATA_DIR="${DATA_DIR}/github_data"

# Configurable variables with defaults
SYNC_INTERVAL=${SYNC_INTERVAL:-3600}
GIT_USER_NAME=${GIT_USER_NAME:-"AutoSync Bot"}
GIT_USER_EMAIL=${GIT_USER_EMAIL:-"autosync@bot.com"}

# ============================================
# VALIDATION
# ============================================

echo "=========================================="
echo "OpenWebUI Data Sync Starting..."
echo "Timezone: $TZ"
echo "Current time: $(date '+%Y-%m-%d %H:%M:%S')"
echo "=========================================="

if [ -z "$G_NAME" ] || [ -z "$G_TOKEN" ]; then
    echo "[ERROR] Missing required environment variables G_NAME or G_TOKEN"
    echo "[WARN] Sync disabled. OpenWebUI will start without data sync."
    exit 0  # Don't exit 1, let OpenWebUI start anyway
fi

echo "[INFO] GitHub Repo: $G_NAME"
echo "[INFO] Sync Interval: ${SYNC_INTERVAL}s"

# ============================================
# INITIAL CLONE & RESTORE
# ============================================

REPO_URL="https://${G_TOKEN}@github.com/${G_NAME}.git"
mkdir -p "$GITHUB_DATA_DIR"

echo "[INFO] Cloning repository..."
if git clone "$REPO_URL" "$GITHUB_DATA_DIR" 2>/dev/null; then
    echo "[SUCCESS] Repository cloned"
    
    # Configure Git LFS
    cd "$GITHUB_DATA_DIR"
    git lfs install --local 2>/dev/null || true
    git lfs pull 2>/dev/null || true
    cd "$APP_DIR"
else
    echo "[WARN] Clone failed - initializing empty sync directory"
    mkdir -p "$GITHUB_DATA_DIR"
    cd "$GITHUB_DATA_DIR"
    git init
    git remote add origin "$REPO_URL" 2>/dev/null || true
    cd "$APP_DIR"
fi

# Restore data from repository
echo "[INFO] Restoring data from repository..."

if [ -f "$GITHUB_DATA_DIR/webui.db" ]; then
    cp "$GITHUB_DATA_DIR/webui.db" "$DATA_DIR/webui.db"
    echo "[SUCCESS] Restored webui.db"
else
    echo "[INFO] No webui.db found in repo, will create on first sync"
fi

if [ -d "$GITHUB_DATA_DIR/uploads" ]; then
    mkdir -p "$DATA_DIR/uploads"
    cp -r "$GITHUB_DATA_DIR/uploads/"* "$DATA_DIR/uploads/" 2>/dev/null || true
    echo "[SUCCESS] Restored uploads directory"
fi

if [ -d "$GITHUB_DATA_DIR/vector_db" ]; then
    mkdir -p "$DATA_DIR/vector_db"
    cp -r "$GITHUB_DATA_DIR/vector_db/"* "$DATA_DIR/vector_db/" 2>/dev/null || true
    echo "[SUCCESS] Restored vector_db directory"
fi

# ============================================
# SYNC FUNCTION
# ============================================

sync_data() {
    while true; do
        echo "[INFO] Next sync in ${SYNC_INTERVAL} seconds..."
        sleep "$SYNC_INTERVAL"
        
        CURRENT_TIME=$(date '+%Y-%m-%d %H:%M:%S')
        echo ""
        echo "=========================================="
        echo "[SYNC] Starting sync at $CURRENT_TIME"
        echo "=========================================="

        # Navigate to GitHub data directory
        cd "$GITHUB_DATA_DIR"
        
        # Configure Git
        git config user.name "$GIT_USER_NAME"
        git config user.email "$GIT_USER_EMAIL"

        # Ensure on correct branch
        git checkout main 2>/dev/null || git checkout master 2>/dev/null || git checkout -b main

        # Pull latest changes
        git pull origin HEAD --rebase 2>/dev/null || echo "[WARN] Pull failed (might be empty repo)"

        # ----------------------------------------
        # 1. Safe Database Backup
        # ----------------------------------------
        if [ -f "$DATA_DIR/webui.db" ]; then
            echo "[INFO] Creating safe backup of webui.db..."
            if command -v sqlite3 &> /dev/null; then
                sqlite3 "$DATA_DIR/webui.db" ".backup '$GITHUB_DATA_DIR/webui.db'"
                echo "[SUCCESS] SQLite hot backup created"
            else
                cp "$DATA_DIR/webui.db" "$GITHUB_DATA_DIR/webui.db"
                echo "[WARN] sqlite3 not found, used cp (potentially unsafe)"
            fi
        else
            echo "[INFO] No webui.db found yet"
        fi

        # ----------------------------------------
        # 2. Sync Uploads Directory
        # ----------------------------------------
        if [ -d "$DATA_DIR/uploads" ]; then
            echo "[INFO] Syncing uploads directory..."
            mkdir -p "$GITHUB_DATA_DIR/uploads"
            cp -r "$DATA_DIR/uploads/"* "$GITHUB_DATA_DIR/uploads/" 2>/dev/null || true
        fi

        # ----------------------------------------
        # 3. Sync Vector DB
        # ----------------------------------------
        if [ -d "$DATA_DIR/vector_db" ]; then
            echo "[INFO] Syncing vector_db directory..."
            mkdir -p "$GITHUB_DATA_DIR/vector_db"
            cp -r "$DATA_DIR/vector_db/"* "$GITHUB_DATA_DIR/vector_db/" 2>/dev/null || true
        fi

        # ----------------------------------------
        # 4. Git Commit & Push
        # ----------------------------------------
        if [[ -n $(git status -s) ]]; then
            echo "[INFO] Changes detected, committing..."
            git add .
            git commit -m "Auto sync: $CURRENT_TIME"
            
            if git push origin HEAD; then
                echo "[SUCCESS] GitHub push successful"
            else
                echo "[WARN] Push failed, retrying..."
                sleep 5
                git pull origin HEAD --rebase 2>/dev/null || true
                git push origin HEAD || echo "[ERROR] Retry failed"
            fi
        else
            echo "[INFO] No changes detected"
        fi

        # Return to app directory
        cd "$APP_DIR"

        # ----------------------------------------
        # 5. WebDAV Sync (Optional)
        # ----------------------------------------
        if [ -n "$WEBDAV_URL" ] && [ -n "$WEBDAV_USERNAME" ] && [ -n "$WEBDAV_PASSWORD" ]; then
            echo "[INFO] Starting WebDAV sync..."
            FILENAME="webui_$(date +'%Y%m%d_%H%M%S').db"
            
            if [ -f "$DATA_DIR/webui.db" ]; then
                if curl -s -T "$DATA_DIR/webui.db" \
                    --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" \
                    "$WEBDAV_URL/$FILENAME"; then
                    echo "[SUCCESS] WebDAV upload successful: $FILENAME"
                else
                    echo "[ERROR] WebDAV upload failed"
                fi
            fi
        fi

        echo "[SYNC] Completed at $(date '+%Y-%m-%d %H:%M:%S')"
    done
}

# ============================================
# START BACKGROUND SYNC
# ============================================

echo "[INFO] Starting background sync process..."
sync_data &
SYNC_PID=$!
echo "[INFO] Sync process started with PID: $SYNC_PID"