utility/utils.py

# libraries
import os
import base64
import json
import re
import logging
from datetime import datetime

import cv2
import numpy as np
import requests
from dotenv import load_dotenv
from PIL import Image, ImageEnhance

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    handlers=[logging.StreamHandler()]
)

# Load environment variables from .env file
load_dotenv()

# Groq config
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
GROQ_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"

RESULT_FOLDER = "static/results/"
JSON_FOLDER = "static/json/"

os.makedirs(RESULT_FOLDER, exist_ok=True)
os.makedirs(JSON_FOLDER, exist_ok=True)

# PaddleOCR home directory is no longer needed for the main path,
# but keeping this does not hurt if something else imports it.
os.environ["PADDLEOCR_HOME"] = "/tmp/.paddleocr"
os.makedirs(os.environ["PADDLEOCR_HOME"], exist_ok=True)


def load_image(image_path):
    ext = os.path.splitext(image_path)[1].lower()
    if ext in [".png", ".jpg", ".jpeg", ".webp", ".tiff", ".bmp"]:
        image = cv2.imread(image_path)
        if image is None:
            raise ValueError(f"Failed to load image from {image_path}")
        return image
    raise ValueError(f"Unsupported image format: {ext}")


def upscale_image(image, scale=2):
    height, width = image.shape[:2]
    return cv2.resize(image, (width * scale, height * scale), interpolation=cv2.INTER_CUBIC)


def reduce_noise(image):
    return cv2.fastNlMeansDenoisingColored(image, None, 10, 10, 7, 21)


def sharpen_image(image):
    kernel = np.array([
        [0, -1, 0],
        [-1, 5, -1],
        [0, -1, 0]
    ])
    return cv2.filter2D(image, -1, kernel)


def enhance_image(image):
    pil_img = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    enhancer = ImageEnhance.Contrast(pil_img)
    enhanced_image = enhancer.enhance(1.5)
    return cv2.cvtColor(np.array(enhanced_image), cv2.COLOR_RGB2BGR)


def process_image(image_path, scale=2):
    image = load_image(image_path)
    upscaled_image = upscale_image(image, scale)
    denoised_image = reduce_noise(upscaled_image)
    sharpened_image = sharpen_image(denoised_image)
    final_image = enhance_image(sharpened_image)
    return final_image


def image_to_base64(image):
    """
    image: OpenCV BGR numpy array
    returns: base64 string of JPEG bytes
    """
    ok, buffer = cv2.imencode(".jpg", image)
    if not ok:
        raise ValueError("Failed to encode image to JPEG.")
    return base64.b64encode(buffer).decode("utf-8")


def _empty_schema():
    return {
        "Name": [],
        "Designation": [],
        "Company": [],
        "Contact": [],
        "Address": [],
        "Email": [],
        "Link": []
    }


def _coerce_list(value):
    if value is None:
        return []
    if isinstance(value, list):
        return [v for v in value if v is not None and str(v).strip() != ""]
    if isinstance(value, tuple):
        return [v for v in value if v is not None and str(v).strip() != ""]
    if isinstance(value, str):
        s = value.strip()
        return [] if s == "" else [s]
    return [value]


def _strip_code_fences(text):
    if not isinstance(text, str):
        return text
    text = text.strip()
    if text.startswith("```"):
        text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.IGNORECASE)
        text = re.sub(r"\s*```$", "", text)
    return text.strip()


def _parse_json_content(content):
    """
    Parses Groq response content into dict.
    Handles:
    - plain JSON string
    - fenced JSON
    - accidental text around JSON
    """
    if isinstance(content, dict):
        return content

    if content is None:
        return {}

    content = _strip_code_fences(str(content))

    try:
        return json.loads(content)
    except json.JSONDecodeError:
        # Try to recover a JSON object embedded in text
        match = re.search(r"\{.*\}", content, flags=re.DOTALL)
        if match:
            return json.loads(match.group(0))
        raise


def normalize_llm_schema(data):
    """
    Normalizes model output to:
    {
        "Name": [],
        "Designation": [],
        "Company": [],
        "Contact": [],
        "Address": [],
        "Email": [],
        "Link": []
    }
    Accepts a dict that may have nulls, strings, or alternate key spellings.
    """
    data = data or {}

    # Common alternate keys seen in model outputs
    key_aliases = {
        "Name": ["Name", "name", "FullName", "full_name", "person_name"],
        "Designation": ["Designation", "designation", "Title", "title", "Role", "role"],
        "Company": ["Company", "company", "Organization", "organization", "Org", "org"],
        "Contact": ["Contact", "contact", "Phone", "phone", "Mobile", "mobile", "PhoneNumber", "phone_number"],
        "Address": ["Address", "address", "Location", "location"],
        "Email": ["Email", "email", "E-mail", "e_mail"],
        "Link": ["Link", "link", "URL", "url", "Website", "website", "Portfolio", "portfolio"]
    }

    normalized = _empty_schema()

    for canonical_key, aliases in key_aliases.items():
        chosen = []
        for alias in aliases:
            if alias in data and data[alias] is not None:
                chosen = _coerce_list(data[alias])
                break
        normalized[canonical_key] = chosen

    return normalized


def call_groq_vlm(image_bgr, prompt, timeout=120, retries=2):
    if not GROQ_API_KEY:
        raise ValueError("GROQ_API_KEY is missing from environment variables.")

    base64_image = image_to_base64(image_bgr)

    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {GROQ_API_KEY}"
    }

    payload = {
        "model": GROQ_MODEL,
        "messages": [
            {
                "role": "system",
                "content": (
                    "You are a strict information extraction engine. "
                    "Return only valid JSON and no markdown."
                )
            },
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{base64_image}"
                        }
                    }
                ]
            }
        ],
        "temperature": 0.1,
        "top_p": 1,
        "max_completion_tokens": 1024,
        "stream": False,
        "response_format": {"type": "json_object"}
    }

    last_error = None
    for attempt in range(retries + 1):
        try:
            resp = requests.post(GROQ_URL, headers=headers, json=payload, timeout=timeout)
            resp.raise_for_status()
            data = resp.json()

            content = data["choices"][0]["message"]["content"]
            parsed = _parse_json_content(content)
            return normalize_llm_schema(parsed)

        except Exception as e:
            last_error = e
            logging.exception(f"Groq VLM request failed on attempt {attempt + 1}")
            if attempt < retries:
                continue

    raise last_error


def build_vlm_prompt():
    return """
Extract structured text from this image and return ONLY valid JSON.

Schema:
{
  "Name": [],
  "Designation": [],
  "Company": [],
  "Contact": [],
  "Address": [],
  "Email": [],
  "Link": []
}

Rules:
- Always return all keys.
- Every value must be a JSON array.
- If a field is not found, return [].
- Do not return null.
- Do not add explanations or markdown.
- Extract all visible text from the image, including business card text, printed labels, logos, URLs, and contact details.
"""


from paddleocr import PaddleOCR

# Global PaddleOCR instance (lazy initialized)
_PADDLE_OCR = None

def get_paddle_ocr():
    global _PADDLE_OCR
    if _PADDLE_OCR is None:
        try:
            _PADDLE_OCR = PaddleOCR(use_angle_cls=False, lang='en')
        except Exception as e:
            logging.error(f"Failed to initialize PaddleOCR: {e}")
            return None
    return _PADDLE_OCR

def call_paddle_ocr(image_bgr):
    """
    Backup OCR using local PaddleOCR.
    Returns: A string of all detected text joined by spaces.
    """
    ocr_engine = get_paddle_ocr()
    if not ocr_engine:
        return ""
    
    try:
        results = ocr_engine.ocr(image_bgr)
        if not results or not results[0]:
            return ""
        
        text_blobs = []
        for line in results[0]:
            # Each entry is like: [[(x1,y1), ...], (text, confidence)]
            text_blobs.append(line[1][0])
            
        return " ".join(text_blobs)
    except Exception as e:
        logging.error(f"PaddleOCR error: {e}")
        return ""

def extract_text_from_images(image_paths):
    """
    Groq VLM single-pass extraction with local PaddleOCR fallback.
    Returns:
        merged_llm_data: dict with the normalized schema
        all_extracted_texts: dict[path] -> Raw text (json from VLM or string from OCR)
        all_extracted_imgs: dict[path] -> processed image path
    """
    merged_llm_data = _empty_schema()
    all_extracted_texts = {}
    all_extracted_imgs = {}

    for image_path in image_paths:
        try:
            enhanced_image = process_image(image_path, scale=2)

            current_time = datetime.now()
            unique_id = current_time.strftime("%Y%m%d%H%M%S%f")
            result_image_path = os.path.join(
                RESULT_FOLDER,
                f"result_{unique_id}_{os.path.basename(image_path)}"
            )

            cv2.imwrite(result_image_path, enhanced_image)
            all_extracted_imgs[image_path] = result_image_path

            # Attempt Primary: Groq VLM
            try:
                single_data = call_groq_vlm(
                    enhanced_image,
                    build_vlm_prompt()
                )
                
                # Merge into combined schema
                for key in merged_llm_data.keys():
                    merged_llm_data[key].extend(_coerce_list(single_data.get(key)))

                # Store VLM output JSON
                all_extracted_texts[image_path] = json.dumps(single_data, ensure_ascii=False)
                logging.info(f"Groq VLM success for: {image_path}")
                
            except Exception as vlm_e:
                logging.warning(f"Groq VLM failed for {image_path}, trying PaddleOCR: {vlm_e}")
                
                # Attempt Fallback: PaddleOCR
                raw_text = call_paddle_ocr(enhanced_image)
                if raw_text:
                    all_extracted_texts[image_path] = raw_text
                    logging.info(f"PaddleOCR success for: {image_path}")
                else:
                    logging.error(f"All OCR/VLM failed for: {image_path}")

        except Exception as e:
            logging.exception(f"Fatal error processing image {image_path}: {e}")
            continue

    return merged_llm_data, all_extracted_texts, all_extracted_imgs



def extract_contact_details(text):
    # Regex patterns
    # Phone numbers with at least 5 digits in any segment 
    combined_phone_regex = re.compile(r'''
    (?: 
        \+1\s\(\d{3}\)\s\d{3}-\d{4} |               # USA/Canada Intl +1 (XXX) XXX-XXXX
        \(\d{3}\)\s\d{3}-\d{4} |                    # USA/Canada STD (XXX) XXX-XXXX
        \(\d{3}\)\s\d{3}\s\d{4} |                   # USA/Canada (XXX) XXX XXXX
        \(\d{3}\)\s\d{3}\s\d{3} |                   # USA/Canada (XXX) XXX XXX
        \+1\d{10} |                                 # +1 XXXXXXXXXX
        \d{10} |                                    # XXXXXXXXXX
        \+44\s\d{4}\s\d{6} |                        # UK Intl +44 XXXX XXXXXX
        \+44\s\d{3}\s\d{3}\s\d{4} |                 # UK Intl +44 XXX XXX XXXX
        0\d{4}\s\d{6} |                             # UK STD 0XXXX XXXXXX
        0\d{3}\s\d{3}\s\d{4} |                      # UK STD 0XXX XXX XXXX
        \+44\d{10} |                                # +44 XXXXXXXXXX
        0\d{10} |                                   # 0XXXXXXXXXX
        \+61\s\d\s\d{4}\s\d{4} |                    # Australia Intl +61 X XXXX XXXX
        0\d\s\d{4}\s\d{4} |                         # Australia STD 0X XXXX XXXX
        \+61\d{9} |                                 # +61 XXXXXXXXX
        0\d{9} |                                    # 0XXXXXXXXX
        \+91\s\d{5}-\d{5} |                         # India Intl +91 XXXXX-XXXXX
        \+91\s\d{4}-\d{6} |                         # India Intl +91 XXXX-XXXXXX
        \+91\s\d{10} |                              # India Intl +91 XXXXXXXXXX
        \+91\s\d{3}\s\d{3}\s\d{4} |                 # India Intl +91 XXX XXX XXXX
        \+91\s\d{3}-\d{3}-\d{4} |                   # India Intl +91 XXX-XXX-XXXX
        \+91\s\d{2}\s\d{4}\s\d{4} |                 # India Intl +91 XX XXXX XXXX
        \+91\s\d{2}-\d{4}-\d{4} |                   # India Intl +91 XX-XXXX-XXXX
        \+91\s\d{5}\s\d{5} |                        # India Intl +91 XXXXX XXXXX 
        \d{5}\s\d{5} |                              # India XXXXX XXXXX 
        \d{5}-\d{5} |                               # India XXXXX-XXXXX 
        0\d{2}-\d{7} |                              # India STD 0XX-XXXXXXX
        \+91\d{10} |                                # +91 XXXXXXXXXX
        \d{10} |                                    # XXXXXXXXXX
        \d{6}-\d{4} |                               # XXXXXX-XXXX
        \d{4}-\d{6} |                               # XXXX-XXXXXX
        \d{3}\s\d{3}\s\d{4} |                       # XXX XXX XXXX
        \d{3}-\d{3}-\d{4} |                         # XXX-XXX-XXXX
        \d{4}\s\d{3}\s\d{3} |                       # XXXX XXX XXX
        \d{4}-\d{3}-\d{3} |                         # XXXX-XXX-XXX
        \+49\s\d{4}\s\d{8} |                        # Germany Intl +49 XXXX XXXXXXXX
        \+49\s\d{3}\s\d{7} |                        # Germany Intl +49 XXX XXXXXXX
        0\d{3}\s\d{8} |                             # Germany STD 0XXX XXXXXXXX
        \+49\d{12} |                                # +49 XXXXXXXXXXXX
        \+49\d{10} |                                # +49 XXXXXXXXXX
        0\d{11} |                                   # 0XXXXXXXXXXX
        \+86\s\d{3}\s\d{4}\s\d{4} |                 # China Intl +86 XXX XXXX XXXX
        0\d{3}\s\d{4}\s\d{4} |                      # China STD 0XXX XXXX XXXX
        \+86\d{11} |                                # +86 XXXXXXXXXXX
        \+81\s\d\s\d{4}\s\d{4} |                    # Japan Intl +81 X XXXX XXXX
        \+81\s\d{2}\s\d{4}\s\d{4} |                 # Japan Intl +81 XX XXXX XXXX
        0\d\s\d{4}\s\d{4} |                         # Japan STD 0X XXXX XXXX
        \+81\d{10} |                                # +81 XXXXXXXXXX
        \+81\d{9} |                                 # +81 XXXXXXXXX
        0\d{9} |                                    # 0XXXXXXXXX
        \+55\s\d{2}\s\d{5}-\d{4} |                  # Brazil Intl +55 XX XXXXX-XXXX
        \+55\s\d{2}\s\d{4}-\d{4} |                  # Brazil Intl +55 XX XXXX-XXXX
        0\d{2}\s\d{4}\s\d{4} |                      # Brazil STD 0XX XXXX XXXX
        \+55\d{11} |                                # +55 XXXXXXXXXXX
        \+55\d{10} |                                # +55 XXXXXXXXXX
        0\d{10} |                                   # 0XXXXXXXXXX
        \+33\s\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |      # France Intl +33 X XX XX XX XX
        0\d\s\d{2}\s\d{2}\s\d{2}\s\d{2} |           # France STD 0X XX XX XX XX
        \+33\d{9} |                                 # +33 XXXXXXXXX
        0\d{9} |                                    # 0XXXXXXXXX
        \+7\s\d{3}\s\d{3}-\d{2}-\d{2} |             # Russia Intl +7 XXX XXX-XX-XX
        8\s\d{3}\s\d{3}-\d{2}-\d{2} |               # Russia STD 8 XXX XXX-XX-XX
        \+7\d{10} |                                 # +7 XXXXXXXXXX
        8\d{10} |                                   # 8 XXXXXXXXXX
        \+27\s\d{2}\s\d{3}\s\d{4} |                 # South Africa Intl +27 XX XXX XXXX
        0\d{2}\s\d{3}\s\d{4} |                      # South Africa STD 0XX XXX XXXX
        \+27\d{9} |                                 # +27 XXXXXXXXX
        0\d{9} |                                    # 0XXXXXXXXX
        \+52\s\d{3}\s\d{3}\s\d{4} |                 # Mexico Intl +52 XXX XXX XXXX
        \+52\s\d{2}\s\d{4}\s\d{4} |                 # Mexico Intl +52 XX XXXX XXXX
        01\s\d{3}\s\d{4} |                          # Mexico STD 01 XXX XXXX
        \+52\d{10} |                                # +52 XXXXXXXXXX
        01\d{7} |                                   # 01 XXXXXXX
        \+234\s\d{3}\s\d{3}\s\d{4} |                # Nigeria Intl +234 XXX XXX XXXX
        0\d{3}\s\d{3}\s\d{4} |                      # Nigeria STD 0XXX XXX XXXX
        \+234\d{10} |                               # +234 XXXXXXXXXX
        0\d{10} |                                   # 0XXXXXXXXXX
        \+971\s\d\s\d{3}\s\d{4} |                   # UAE Intl +971 X XXX XXXX
        0\d\s\d{3}\s\d{4} |                         # UAE STD 0X XXX XXXX
        \+971\d{8} |                                # +971 XXXXXXXX
        0\d{8} |                                    # 0XXXXXXXX
        \+54\s9\s\d{3}\s\d{3}\s\d{4} |              # Argentina Intl +54 9 XXX XXX XXXX
        \+54\s\d{1}\s\d{4}\s\d{4} |                 # Argentina Intl +54 X XXXX XXXX
        0\d{3}\s\d{4} |                             # Argentina STD 0XXX XXXX
        \+54\d{10} |                                # +5 Argentian +54 9 XXXXXXXXXX
        \+54\d{9} |                                 # +54 XXXXXXXXX
        0\d{7} |                                    # 0XXXXXXX
        \+966\s\d\s\d{3}\s\d{4} |                   # Saudi Intl +966 X XXX XXXX
        0\d\s\d{3}\s\d{4} |                         # Saudi STD 0X XXX XXXX
        \+966\d{8} |                                # +966 XXXXXXXX
        0\d{8} |                                    # 0XXXXXXXX
        \+\d{3}-\d{3}-\d{4} |                        # Generic +XXX-XXX-XXXX
        (?:\+?\d{1,3})?[-.\s()]?\d{3,5}[-.\s()]?\d{3,5}[-.\s()]?\d{3,5} # Highly flexible generic
    )  
    ''', re.VERBOSE)


    email_regex = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b')
    link_regex = re.compile(r'\b(?:https?:\/\/)?(?:www\.)[a-zA-Z0-9-]+\.(?:com|co\.in|co|io|org|net|edu|gov|mil|int|uk|us|in|de|au|app|tech|xyz|info|biz|fr|dev)\b')

    phone_numbers = [num for num in combined_phone_regex.findall(text) if len(num) >= 5]
    emails = email_regex.findall(text)
    links_RE = [link for link in link_regex.findall(text) if len(link) >= 11]
    links_RE = [link for link in links_RE if not any(email in link for email in emails)]

    return {
        "phone_numbers": phone_numbers,
        "emails": emails,
        "links_RE": links_RE
    }


def process_extracted_text(extracted_text):
    data = json.loads(json.dumps(extracted_text, indent=4))

    combined_results = {
        "phone_numbers": [],
        "emails": [],
        "links_RE": []
    }

    for filename, text in data.items():
        contact_details = extract_contact_details(text)
        combined_results["phone_numbers"].extend(contact_details["phone_numbers"])
        combined_results["emails"].extend(contact_details["emails"])
        combined_results["links_RE"].extend(contact_details["links_RE"])

    print("Combined contact details in JSON format:")
    print(combined_results)

    return combined_results


def remove_duplicates_case_insensitive(data_dict):
    for key, value_list in data_dict.items():
        if not isinstance(value_list, list):
            continue

        seen = set()
        unique_list = []

        for item in value_list:
            item_str = str(item)
            key_lower = item_str.lower()
            if key_lower not in seen:
                unique_list.append(item)
                seen.add(key_lower)

        data_dict[key] = unique_list

    return data_dict


def process_resume_data(LLMdata, cont_data, extracted_text):
    """
    Final merge step.
    Keeps the output structure exactly as you currently use in result.html.
    """
    LLMdata = normalize_llm_schema(LLMdata)
    cont_data = cont_data or {}

    cont_data.setdefault("emails", [])
    cont_data.setdefault("phone_numbers", [])
    cont_data.setdefault("links_RE", [])

    # Merge regex-detected emails
    existing_emails = {str(e).lower() for e in LLMdata["Email"]}
    for email in cont_data["emails"]:
        if str(email).lower() not in existing_emails:
            LLMdata["Email"].append(email)
            existing_emails.add(str(email).lower())

    # Merge regex-detected links
    existing_links = {str(l).lower() for l in LLMdata["Link"]}
    for link in cont_data["links_RE"]:
        if str(link).lower() not in existing_links:
            LLMdata["Link"].append(link)
            existing_links.add(str(link).lower())

    # Merge regex-detected contacts using last-10-digit normalization
    normalized_contacts = {str(num)[-10:] for num in LLMdata["Contact"] if num}
    for num in cont_data["phone_numbers"]:
        norm = str(num)[-10:]
        if norm not in normalized_contacts:
            LLMdata["Contact"].append(num)
            normalized_contacts.add(norm)

    LLMdata = remove_duplicates_case_insensitive(LLMdata)

    processed_data = {
        "name": LLMdata.get("Name", []),
        "contact_number": LLMdata.get("Contact", []),
        "Designation": LLMdata.get("Designation", []),
        "email": LLMdata.get("Email", []),
        "Location": LLMdata.get("Address", []),
        "Link": LLMdata.get("Link", []),
        "Company": LLMdata.get("Company", []),
        "extracted_text": extracted_text,
        "status_message": f"Source: {LLMdata.get('meta', 'Primary+Backup')}"
    }

    for key in ["name", "contact_number", "Designation", "email", "Location", "Link", "Company"]:
        processed_data[key] = [
            v for v in processed_data[key]
            if str(v).strip().lower() not in {"not found", "none", "null", ""}
        ]

    return processed_data


# Optional compatibility helper; no longer needed by the main flow.
def json_to_llm_str(textJson):
    s = ""
    for _, item in textJson.items():
        s += str(item) + " "
    return s