scraper.py

import requests
import json
import re
from bs4 import BeautifulSoup
from typing import List, Dict, Any, Tuple
from utils import clean_time

def scrape_workshops_from_squarespace(url: str) -> List[Dict[str, str]]:
    """
    Extract workshops using our robust Squarespace JSON + HTML parsing system
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    
    try:
        # First try the Squarespace JSON API
        json_url = f"{url}?format=json"
        print(f"🔍 Trying Squarespace JSON API: {json_url}")
        
        response = requests.get(json_url, headers=headers, timeout=10)
        if response.status_code == 200:
            try:
                json_data = response.json()
                workshops = extract_workshops_from_json(json_data, json_url)
                if workshops:
                    print(f"✅ Extracted {len(workshops)} workshops from JSON API")
                    return workshops
                else:
                    print("❌ No workshops found in JSON, falling back to HTML")
            except json.JSONDecodeError:
                print("❌ Invalid JSON response, falling back to HTML")
        
        # Fallback to HTML scraping if JSON fails
        print(f"📄 Falling back to HTML scraping for {url}")
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.content, 'html.parser')
        workshops = parse_workshops_from_html(soup, url)
        
        if workshops:
            print(f"✅ Extracted {len(workshops)} workshops from HTML parsing")
            return workshops
        else:
            print("❌ No workshops found in HTML")
            return []
            
    except Exception as e:
        print(f"❌ Error scraping workshops from {url}: {e}")
        return []

def extract_workshops_from_json(data: Any, source_url: str) -> List[Dict[str, str]]:
    """Extract workshop information from Squarespace JSON data"""
    workshops = []
    
    # Check if there's mainContent HTML to parse
    if isinstance(data, dict) and 'mainContent' in data:
        main_content_html = data['mainContent']
        if isinstance(main_content_html, str):
            print(f"🎯 Found mainContent HTML! Length: {len(main_content_html)} characters")
            
            soup = BeautifulSoup(main_content_html, 'html.parser')
            workshops = parse_workshops_from_html(soup, source_url)
            
            if workshops:
                return workshops
    
    return workshops

def parse_workshops_from_html(soup, source_url: str) -> List[Dict[str, str]]:
    """Enhanced HTML parsing specifically for workshop content"""
    workshops = []
    workshop_texts = set()
    
    print(f"🔍 ENHANCED HTML PARSING:")
    
    # Method 1: Find individual workshop containers
    potential_containers = soup.find_all(['div', 'section', 'article'], 
                                       attrs={'class': re.compile(r'(item|card|product|workshop|class)', re.I)})
    
    print(f"   Found {len(potential_containers)} potential workshop containers")
    
    for container in potential_containers:
        workshop_text = container.get_text(strip=True)
        
        if len(workshop_text) < 30 or workshop_text in workshop_texts:
            continue
            
        if any(keyword in workshop_text.lower() for keyword in ['with', 'casting', 'director', 'agent', 'perfect submission', 'crush the callback', 'get scene']):
            workshop = extract_single_workshop_from_text(workshop_text, source_url)
            if workshop and not is_duplicate_workshop(workshop, workshops):
                workshops.append(workshop)
                workshop_texts.add(workshop_text)
    
    # Method 2: Pattern-based extraction from full text
    all_text = soup.get_text()
    
    workshop_patterns = [
        # Pattern 1: "Workshop Title with Professional Title Name on Date @ Time"
        r'((?:The\s+)?(?:Perfect\s+Submission|Crush\s+the\s+Callback|Get\s+Scene\s+360?))\s+with\s+((?:Casting\s+Director|DDO\s+Agent|Manager|Director|Producer|Agent|Acting\s+Coach|Talent\s+Agent|Executive\s+Casting\s+Producer)\s+[A-Za-z\s]+?)\s+on\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern 2: "Professional Title Name, Workshop Title on Date @ Time"  
        r'((?:Atlanta\s+Models\s+&\s+Talent\s+President|Talent\s+Agent|Casting\s+Director|Manager|Director|Producer|Agent)\s+[A-Za-z\s]+?),\s+((?:The\s+)?(?:Perfect\s+Submission|Crush\s+the\s+Callback|Get\s+Scene\s+360?))\s+on\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern 3: "Casting Director Name, Date @ Time"
        r'(Casting\s+Director)\s+([A-Za-z\s\-]+?),\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*(?:at\s+)?([0-9:]+\s*(?:AM|PM))?',
    ]
    
    for i, pattern in enumerate(workshop_patterns):
        matches = re.findall(pattern, all_text, re.IGNORECASE)
        for match in matches:
            workshop = parse_refined_workshop_match(match, i+1, source_url)
            if workshop and not is_duplicate_workshop(workshop, workshops):
                workshops.append(workshop)
    
    print(f"🎯 TOTAL UNIQUE WORKSHOPS FOUND: {len(workshops)}")
    return workshops

def extract_single_workshop_from_text(text: str, source_url: str) -> Dict[str, str]:
    """Extract workshop info from a single text block"""
    
    # Clean up the text
    text = re.sub(r'\$[0-9,]+\.00', '', text)
    text = re.sub(r'Featured|Sold Out', '', text, flags=re.IGNORECASE)
    text = re.sub(r'\s+', ' ', text).strip()
    text = re.sub(r'\n+', ' ', text)
    
    patterns = [
        # Pattern A: "Title with Professional Name on Date @ Time"
        r'((?:The\s+)?(?:Perfect\s+Submission|Crush\s+the\s+Callback|Get\s+Scene\s+360?))\s+with\s+((?:Casting\s+Director|CD|DDO\s+Agent|Manager|Director|Producer|Agent|Acting\s+Coach|Talent\s+Agent|Executive\s+Casting\s+Producer|Atlanta\s+Models\s+&\s+Talent\s+President)\s+[A-Za-z\s\-]+?)\s+on\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern B: "Professional Name, Title on Date @ Time"
        r'((?:Atlanta\s+Models\s+&\s+Talent\s+President|Talent\s+Agent|Casting\s+Director|Casting\s+Associate|Manager|Director|Producer|Agent|Executive\s+Casting\s+Producer)\s+[A-Za-z\s\-]+?),\s+((?:The\s+)?(?:Perfect\s+Submission|Crush\s+the\s+Callback|Get\s+Scene\s+360?))\s+on\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern C: "Casting Director Name, Date at Time"
        r'(Casting\s+Director|Casting\s+Associate)\s+([A-Za-z\s\-]+?),\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*(?:at\s+)?([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern D: "Company Executive Producer Name on Date"
        r"([A-Za-z']+\s+(?:Executive\s+Casting\s+Producer|Studios\s+Casting\s+Associate))\s+([A-Za-z\s]+?)\s+(?:on\s+)?(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?",
        
        # Pattern E: "Company Agent Name Date" (fixed "on" issue)
        r'([A-Za-z\s]+)\s+(Agent|Talent)\s+([A-Za-z\s]+?)\s+(?:on\s+)?(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern F: "Company, Person, Title on Date"
        r'([A-Za-z\s]+\s+Talent),\s+([A-Za-z\s\.]+?),\s+((?:The\s+)?(?:Perfect\s+Submission|Crush\s+the\s+Callback|Get\s+Scene\s+360?))\s+on\s+(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?',
        
        # Pattern G: Flexible fallback
        r'^([A-Za-z\s&\']{3,25}(?:Director|Agent|Manager|Producer|President|Coach))\s+([A-Za-z\s\-]{3,30}?)\s+(?:on\s+)?(\w+\s+\d+(?:st|nd|rd|th)?)\s*[@\s]*([0-9:]+\s*(?:AM|PM))?$'
    ]
    
    for i, pattern in enumerate(patterns):
        match = re.search(pattern, text, re.IGNORECASE)
        if match:
            return parse_pattern_match(match, i, source_url)
    
    return None

def parse_pattern_match(match, pattern_index: int, source_url: str) -> Dict[str, str]:
    """Parse a regex match or tuple based on pattern type"""
    # Use a helper to get group content whether it's a match object or tuple
    def get_grp(m, idx):
        val = ""
        if hasattr(m, 'group'):
            try:
                val = m.group(idx)
            except IndexError:
                val = ""
        # If it's a tuple (from findall), idx is 1-based in standard regex terminology
        # but 0-indexed in the tuple.
        elif isinstance(m, (tuple, list)):
            if 0 <= idx-1 < len(m):
                val = m[idx-1]
        
        return val if val is not None else ""

    # Initialize variables
    workshop_title = ""
    instructor_title = ""
    instructor_name = ""
    date_str = ""
    time_str = ""

    try:
        if pattern_index == 0:  # Pattern A/1
            workshop_title = get_grp(match, 1).strip()
            professional_full = get_grp(match, 2).strip()
            date_str = get_grp(match, 3).strip()
            time_str = get_grp(match, 4).strip()
            
            if professional_full.startswith('CD '):
                professional_full = 'Casting Director ' + professional_full[3:]
                
            instructor_title, instructor_name = parse_professional_info(professional_full)
            
        elif pattern_index == 1:  # Pattern B/2
            professional_full = get_grp(match, 1).strip()
            workshop_title = get_grp(match, 2).strip()
            date_str = get_grp(match, 3).strip()
            time_str = get_grp(match, 4).strip()
            
            instructor_title, instructor_name = parse_professional_info(professional_full)
            
        elif pattern_index == 2:  # Pattern C/3
            instructor_title = get_grp(match, 1).strip()
            instructor_name = get_grp(match, 2).strip()
            date_str = get_grp(match, 3).strip()
            time_str = get_grp(match, 4).strip()
            workshop_title = "Casting Workshop"
            
        elif pattern_index == 3:  # Pattern D
            instructor_title = get_grp(match, 1).strip()
            instructor_name = get_grp(match, 2).strip()
            date_str = get_grp(match, 3).strip()
            time_str = get_grp(match, 4).strip()
            workshop_title = "Industry Workshop"
            
        elif pattern_index == 4:  # Pattern E
            company_name = get_grp(match, 1).strip()
            agent_type = get_grp(match, 2).strip()
            instructor_name = get_grp(match, 3).strip()
            date_str = get_grp(match, 4).strip()
            time_str = get_grp(match, 5).strip()
            
            instructor_title = f"{company_name} {agent_type}"
            workshop_title = "Industry Workshop"
            
        elif pattern_index == 5:  # Pattern F
            company_name = get_grp(match, 1).strip()
            instructor_name = get_grp(match, 2).strip()
            workshop_title = get_grp(match, 3).strip()
            date_str = get_grp(match, 4).strip()
            time_str = get_grp(match, 5).strip()
            
            instructor_title = company_name
            
        else:  # Pattern G
            professional_full = get_grp(match, 1).strip() + " " + get_grp(match, 2).strip()
            date_str = get_grp(match, 3).strip()
            time_str = get_grp(match, 4).strip()
            workshop_title = "Industry Workshop"
            
            if len(professional_full) > 50 or '\n' in professional_full:
                return None
                
            instructor_title, instructor_name = parse_professional_info(professional_full)
        
        if instructor_name and date_str:
            # Create full_text for embedding (required by existing Flask API)
            full_text = f"{workshop_title} with {instructor_title} {instructor_name}"
            if date_str:
                full_text += f" on {date_str}"
            if time_str:
                full_text += f" at {clean_time(time_str)}"
            
            return {
                'title': workshop_title,
                'instructor_name': instructor_name,
                'instructor_title': instructor_title,
                'date': date_str,
                'time': clean_time(time_str),
                'full_text': full_text,  # Required for existing embedding system
                'source_url': source_url
            }
            
    except Exception as e:
        print(f"Error parsing pattern match: {e}")
        
    return None

def parse_professional_info(professional_full: str) -> tuple:
    """Parse professional title and name from full string"""
    
    professional_full = re.sub(r'\s+', ' ', professional_full).strip()
    
    # Handle specific multi-word titles
    specific_titles = [
        'Atlanta Models & Talent President',
        'Executive Casting Producer', 
        'Casting Director',
        'Casting Associate',
        'DDO Agent',
        'Talent Agent',
        'Acting Coach'
    ]
    
    for title in specific_titles:
        if title in professional_full:
            title_pos = professional_full.find(title)
            
            if title_pos == 0:
                name_part = professional_full[len(title):].strip()
                return title, name_part
            else:
                name_part = professional_full[:title_pos].strip().rstrip(',')
                return title, name_part
    
    # Fallback for single-word titles
    single_word_titles = ['Manager', 'Director', 'Producer', 'Agent', 'Coach', 'President']
    
    words = professional_full.split()
    for i, word in enumerate(words):
        if word in single_word_titles:
            if i > 0 and words[i-1] in ['Casting', 'Talent', 'Executive', 'DDO', 'Acting']:
                title = f"{words[i-1]} {word}"
                name_parts = words[:i-1] + words[i+1:]
            else:
                title = word
                name_parts = words[:i] + words[i+1:]
            
            name = ' '.join(name_parts).strip()
            return title, name
    
    # Final fallback
    if len(words) >= 2:
        return words[0], ' '.join(words[1:])
    
    return '', professional_full

def parse_refined_workshop_match(match, pattern_num: int, source_url: str) -> Dict[str, str]:
    """Parse a regex match into a clean workshop dictionary"""
    return parse_pattern_match(match, pattern_num-1, source_url)  # Adjust for 0-based indexing

def is_duplicate_workshop(new_workshop: Dict, existing_workshops: List[Dict]) -> bool:
    """Enhanced duplicate detection"""
    for existing in existing_workshops:
        if (existing.get('instructor_name', '').strip().lower() == new_workshop.get('instructor_name', '').strip().lower() and 
            existing.get('date', '').strip().lower() == new_workshop.get('date', '').strip().lower()):
            
            existing_title = existing.get('title', '').strip().lower()
            new_title = new_workshop.get('title', '').strip().lower()
            
            if (existing_title == new_title or 
                'workshop' in existing_title and 'workshop' in new_title or
                existing_title in new_title or new_title in existing_title):
                return True
    return False

def calculate_workshop_confidence(w: Dict) -> float:
    """Calculate confidence score of retrieved workshop data"""
    score = 0.0
    if w.get('title'): score += 0.3
    if w.get('instructor_name'): score += 0.3
    if w.get('date'): score += 0.2
    if w.get('time'): score += 0.1
    if w.get('source_url'): score += 0.1
    return round(score, 2)