Track large files with LFS

.gitattributes

@@ -0,0 +1,4 @@
+Backend/*.txt filter=lfs diff=lfs merge=lfs -text
+Backend/*.numbers filter=lfs diff=lfs merge=lfs -text
+Backend/checkpoints/* filter=lfs diff=lfs merge=lfs -text
+Backend/idioms_structured_1/*.jsonl filter=lfs diff=lfs merge=lfs -text

Backend/Database/Idiom_Id_Generator.py

@@ -0,0 +1,34 @@
+from supabase import create_client, Client
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+url = os.getenv("database_url")
+key = os.getenv("database_service_Key")
+
+supabase: Client = create_client(url, key)
+
+def generate_id(lang_code: str, dialect: str | None) -> str:
+    # Normaliza dialecto si existe
+    if dialect and dialect.strip():
+        dialect_clean = dialect.replace(" ", "_").lower()
+    else:
+        dialect_clean = None
+    
+    # Construye la query base
+    query = supabase.table("idioms").select("id", count="exact").eq("language", lang_code)
+    if dialect_clean:
+        query = query.eq("language", lang_code)
+    else:
+        query = query.eq("language", lang_code )  # si lo guardas como NULL
+    
+    res = query.execute()
+    counter = res.count + 1
+    
+    # Genera ID
+    if dialect_clean:
+        return f"{lang_code}_{dialect_clean}_{str(counter).zfill(4)}"
+    else:
+        return f"{lang_code}_unspecified_{str(counter).zfill(4)}"
+

Backend/Database/crud.py

@@ -0,0 +1,211 @@
+# crud.py  (reemplaza la versión SQLAlchemy)
+import os
+import httpx
+from dotenv import load_dotenv
+from typing import Dict, Optional, List
+from fastapi import HTTPException
+load_dotenv()
+
+SUPABASE_URL = os.getenv("SUPABASE_URL").rstrip("/")
+SUPABASE_KEY = os.getenv("SUPABASE_KEY")
+database_service_Key = os.getenv("database_service_Key")
+TABLE = os.getenv("TABLE_NAME", "idioms")
+
+HEADERS = {
+    "apikey": database_service_Key,
+    "Authorization": f"Bearer {database_service_Key}",
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+}
+
+# helpers
+async def _client():
+    return httpx.AsyncClient(timeout=30.0)
+
+# CRUD
+async def get_idioms(skip: int = 0, limit: int = 100000):
+    async with httpx.AsyncClient() as client:
+        url = f"{SUPABASE_URL}/rest/v1/{TABLE}?select=*&offset={skip}&limit={limit}"
+        print("Supabase GET URL:", url)
+        r = await client.get(url, headers=HEADERS)
+        print("Supabase GET status:", r.status_code)
+        print("Supabase GET response:", r.text)
+        r.raise_for_status()
+        return r.json()
+
+async def get_all_idioms():
+    all_idioms = []
+    limit = 1000   # Supabase max per request
+    offset = 0
+
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        while True:
+            url = f"{SUPABASE_URL}/rest/v1/{TABLE}?select=*&limit={limit}&offset={offset}"
+            r = await client.get(url, headers=HEADERS)
+            r.raise_for_status()
+            data = r.json()
+            if not data:
+                break
+            # ensure validation_count is a dict
+            for item in data:                 
+                if not isinstance(item.get("validation_count"), dict):
+                    item["validation_count"] = {}
+            all_idioms.extend(data)
+            offset += limit
+
+    return all_idioms
+    
+async def get_idiom(idiom_id: str):
+    async with httpx.AsyncClient() as client:
+        try:
+            # Include both examples and meanings via foreign key embedding
+            url = (
+                f"{SUPABASE_URL}/rest/v1/{TABLE}?"
+                f"id=eq.{idiom_id}&"
+                f"select=*,"
+                f"idiom_meanings!idiom_meanings_idiom_id_fkey(*),"
+                f"examples!examples_idiom_id_fkey(*)"
+                
+            )
+            print(f"Fetching idiom from Supabase: {url}")  # debug
+            r = await client.get(url, headers=HEADERS)
+            print("HTTP status code:", r.status_code)  # debug
+            r.raise_for_status()
+            data = r.json()
+            print("Raw data from Supabase:", data)  # debug
+        except httpx.RequestError as e:
+            print("Request failed:", e)
+            raise HTTPException(status_code=500, detail=f"Supabase request failed: {e}")
+        except httpx.HTTPStatusError as e:
+            print("HTTP error:", e)
+            raise HTTPException(status_code=500, detail=f"Supabase HTTP error: {e}")
+        except Exception as e:
+            print("Unexpected error:", e)
+            raise HTTPException(status_code=500, detail=f"Unexpected error: {e}")
+
+        if not data:
+            print(f"No idiom found for id: {idiom_id}")  # debug
+            return None
+
+        idiom = data[0]
+        if not isinstance(idiom, dict):
+            print(f"Unexpected data type for idiom: {type(idiom)}")  # debug
+            raise ValueError(f"Expected dict, got: {type(idiom)}")
+
+        # --- Transform examples ---
+        raw_examples = idiom.get("examples") or []
+        idiom["examples"] = [
+            {
+                "id": ex.get("id"),
+                "source_text": ex.get("source_text") or "",
+                "source_language": ex.get("source_language") or idiom.get("language"),
+                "translations": json.loads(ex["translations"]) if isinstance(ex.get("translations"), str) else ex.get("translations") or [],
+                "dialect": ex.get("dialect"),
+                "url": ex.get("url"),
+                "source": ex.get("source"),
+            }
+            for ex in raw_examples
+        ]
+        print(f"Found {len(idiom['examples'])} examples")  # debug
+
+        # --- Transform meanings ---
+        raw_meanings = idiom.get("idiom_meanings") or []
+        print("Raw meanings data:", raw_meanings)  # debug
+        idiom["meanings"] = [
+            {
+                "meaning_id": m.get("meaning_id"),
+                "idiom_id": m.get("idiom_id"),
+                "sense_number": m.get("sense_number"),
+                "register": m.get("register") or [],
+                "region": m.get("region") or [],
+                "definitions": m.get("definitions") or [],
+                "version": m.get("version"),  # optional, if you need it
+            }
+            for m in raw_meanings
+        ]
+        print("Transformed meanings data:", idiom["meanings"])  # debug
+        print(f"Found {len(idiom['meanings'])} meanings")  # debug
+        return idiom
+
+
+
+async def search_idioms(query: str = "", language: Optional[str] = None, skip: int = 0, limit: int = 50):
+    async with httpx.AsyncClient() as client:
+        # Compose select param to embed idiom_meanings
+        select_query = "*,idiom_meanings!idiom_meanings_idiom_id_fkey(*)"
+        
+        url = (
+            f"{SUPABASE_URL}/rest/v1/{TABLE}"
+            f"?offset={skip}&limit={limit}&select={select_query}"
+        )
+        
+        # Maintain partial text match on idiom column
+        if query:
+            url += f"&idiom=ilike.*{query}*"
+        
+        # Maintain language filter if specified and not "all"
+        if language and language.lower() not in ("all", "*"):
+            url += f"&language=eq.{language}"
+        
+        r = await client.get(url, headers=HEADERS)
+        r.raise_for_status()
+        data = r.json()
+        
+        # Ensure validation_count is a dict
+        for item in data:
+            if not isinstance(item.get("validation_count"), dict):
+                item["validation_count"] = {}
+            
+            # Transform embedded idiom_meanings to meanings field for UI use
+            raw_meanings = item.get("idiom_meanings") or []
+            item["meanings"] = [
+                {
+                    "meaning_id": m.get("meaning_id"),
+                    "idiom_id": m.get("idiom_id"),
+                    "sense_number": m.get("sense_number"),
+                    "register": m.get("register") or [],
+                    "region": m.get("region") or [],
+                    "definitions": m.get("definitions") or [],
+                    "version": m.get("version"),
+                }
+                for m in raw_meanings
+            ]
+        
+        return data
+
+
+    
+async def create_idiom(item: dict):
+    async with httpx.AsyncClient() as client:
+        url = f"{SUPABASE_URL}/rest/v1/{TABLE}"
+        r = await client.post(url, json=item, headers=HEADERS)
+        r.raise_for_status()  # fail if not 2xx
+
+        try:
+            data = r.json()
+        except ValueError:
+            # Supabase returned empty body, fallback to the original item
+            data = item
+
+        if isinstance(data, list) and data:
+            return data[0]
+        if isinstance(data, dict) and data:
+            return data
+        # final fallback
+        return item
+async def update_idiom(idiom_id: str, item: dict):
+    async with httpx.AsyncClient() as client:
+        url = f"{SUPABASE_URL}/rest/v1/{TABLE}?id=eq.{idiom_id}"
+        r = await client.patch(url, json=item, headers=HEADERS)
+        if r.status_code not in (200, 204):
+            raise httpx.HTTPStatusError("Update failed", request=r.request, response=r)
+        # After patch, fetch the updated row
+        return await get_idiom(idiom_id)
+
+async def delete_idiom(idiom_id: str):
+    async with httpx.AsyncClient() as client:
+        url = f"{SUPABASE_URL}/rest/v1/{TABLE}?id=eq.{idiom_id}"
+        r = await client.delete(url, headers=HEADERS)
+        if r.status_code not in (200, 204):
+            raise httpx.HTTPStatusError("Delete failed", request=r.request, response=r)
+        return {"status": "deleted"}

Backend/Database/db.py

@@ -0,0 +1,17 @@
+import os
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, declarative_base
+from dotenv import load_dotenv
+
+load_dotenv()
+
+DATABASE_URL = os.getenv("DATABASE_URL")
+db_url = DATABASE_URL
+# Engine with SSL (required by Supabase)
+engine = create_engine(db_url, connect_args={"sslmode": "require"})
+
+# Session factory
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+# Base class for models
+Base = declarative_base()

Backend/Database/main.py

@@ -0,0 +1,120 @@
+# main.py
+from fastapi import FastAPI, HTTPException, APIRouter
+from fastapi.middleware.cors import CORSMiddleware
+import crud
+import schemas
+from Idiom_Id_Generator import generate_id
+from dotenv import load_dotenv
+import os
+from pydantic import BaseModel
+from typing import List, Dict, Optional
+
+
+
+load_dotenv()
+
+app = FastAPI(title="Idioms API - Supabase REST")
+
+allowed_origins = [
+    "http://localhost",
+    "http://localhost:3000",
+    "http://localhost:8000",
+    "https://idiomator.vercel.app",
+    "https://www.idiomator.vercel.app",
+    "https://idiomator.com",
+    "https://www.idiomator.com",
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins= allowed_origins,#allowed_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/idioms", response_model=list[schemas.IdiomResponse])
+async def read_idioms(skip: int = 0, limit: int = 50):
+    try:
+        data = await crud.get_idioms(skip=skip, limit=limit)
+        return data
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/idioms", response_model=schemas.IdiomResponse)
+async def create_idiom(idiom: schemas.IdiomCreate):
+    try:
+        # keep your ID generation
+        idiom_id = generate_id(idiom.language, idiom.dialect)
+        idiom_dict = idiom.dict()
+        idiom_dict["id"] = idiom_id
+        created = await crud.create_idiom(idiom_dict)
+        return created
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/idioms/all_ids", response_model=List[Dict[str, str]])
+async def get_all_idioms_route():
+    idioms = await crud.get_all_idioms()
+    print(f"Fetched {len(idioms)} idioms")
+    print(f"Sample idioms: {idioms[:3]}")  # Print first 3 idioms for verification
+    return [
+        {"id": idiom["id"], "idiom": idiom["idiom"], "language": idiom["language"]}
+        for idiom in idioms
+    ]
+
+@app.get("/idioms/search", response_model=List[schemas.IdiomResponse])
+async def search_idioms(
+    q: Optional[str] = None,
+    language: Optional[str] = None,
+    skip: int = 0,
+    limit: int = 50
+):
+
+    try:
+        results = await crud.search_idioms(query=q, language=language, skip=skip, limit=limit)
+        return results
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get("/idioms/{idiom_id}", response_model=schemas.IdiomResponse)
+async def read_idiom(idiom_id: str):
+    try:
+        db_idiom = await crud.get_idiom(idiom_id)
+        if not db_idiom:
+            raise HTTPException(status_code=404, detail="Idiom not found")
+        # Transform validation_count if needed
+        if isinstance(db_idiom.get("validation_count"), int):
+            db_idiom["validation_count"] = {"count": db_idiom["validation_count"]}
+        return db_idiom
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+    
+@app.patch("/idioms/{idiom_id}", response_model=schemas.IdiomResponse)
+async def patch_idiom(idiom_id: str, idiom_update: schemas.IdiomBase):
+    try:
+        updated = await crud.update_idiom(idiom_id, idiom_update.dict(exclude_unset=True))
+        if not updated:
+            raise HTTPException(status_code=404, detail="Idiom not found")
+        return updated
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.delete("/idioms/{idiom_id}")
+async def delete_idiom(idiom_id: str):
+    try:
+        return await crud.delete_idiom(idiom_id)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/ping")
+async def ping():
+    return {"status": "ok"}
+

Backend/Database/models.py

@@ -0,0 +1,16 @@
+from sqlalchemy import Column, String, Integer, JSON
+from db import Base
+
+class Idiom(Base):
+    __tablename__ = "idioms"
+
+    id = Column(String, primary_key=True, index=True)
+    idiom = Column(String, nullable=False)
+    language = Column(String, nullable=False)
+    dialect = Column(String, default="unspecified")
+    idiomatic_meaning = Column(String, default="")
+    literal_meaning = Column(String, default="")
+    example = Column(String, default="")
+    validation_count = Column(JSON, default={"approved": 0, "rejected": 0})  # { "approved": 0, "rejected": 0 }
+    quality = Column(String, default="seed")
+    status = Column(String, default="pending")

Backend/Database/requirements.txt

@@ -0,0 +1,12 @@
+uvicorn
+gunicorn
+sqlalchemy
+supabase
+fastapi
+pydantic
+databases
+asyncpg
+alembic
+psycopg2-binary
+python-dotenv
+requests

Backend/Database/schemas.py

@@ -0,0 +1,59 @@
+from pydantic import BaseModel, ConfigDict
+from typing import Dict, Optional, List
+
+
+class Translation(BaseModel):
+    language: str
+    text: str
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class Example(BaseModel):
+    id: int
+    source_language: str
+    source_text: str
+    translations: List[Translation] = []
+    dialect: Optional[str] = None
+    url: Optional[str] = None
+    source: Optional[str] = None
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class IdiomBase(BaseModel):
+    idiom: str
+    language: str
+    dialect: str = "unspecified"
+    idiomatic_meaning: Optional[str] = ""
+    literal_meaning: Optional[str] = ""
+    validation_count: Dict[str, int] = {}
+    quality: str = "seed"
+    status: str = "pending"
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class Meaning(BaseModel):
+    meaning_id: Optional[str] = None
+    idiom_id: Optional[str] = None
+    sense_number: Optional[int] = None
+    register: List[str] = []
+    region: List[str] = []
+    definitions: List[str] = []
+    version: Optional[int] = None
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class IdiomResponse(IdiomBase):
+    id: str
+    examples: List[Example] = []
+    meanings: List[Meaning] = []   # ← add this
+
+    model_config = ConfigDict(extra="ignore")
+
+
+class IdiomCreate(IdiomBase):
+    id: str
+    examples: List[Example] = []

Backend/Idiom_lexicon.py

@@ -0,0 +1,3 @@
+KNOWN_IDIOMS = {
+    
+}

Backend/checkpoints/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e46bcd33d5c550de1d2d1219ee5ba9e76ad3150e4f8db750bcc06d427db2580e
+size 5102

Backend/checkpoints/adapter_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baf0bbe4850ef1d5b7e70925a08c46c7bf6f77983a4b389d5d61fefedd4fe9c3
+size 813

Backend/checkpoints/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe56effed85c0e03c87510d661319fac10c770a2510078d307742b6acc2d6385
+size 1785652

Backend/checkpoints/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fa8a09531fb96548794a2bf557b30a66009d2f4b5bef9da95974faaa562f191
+size 710950671

Backend/checkpoints/model_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:018070b58b7084b2f3bb109d8d8f9e867ef69fbe0ad0f2de417931973187c5ea
+size 114

Backend/checkpoints/special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3
+size 125

Backend/checkpoints/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:672146ee6867dc02a01c474090e237789f8a066ee7247bb2cb6c8688a27536a8
+size 2919627

Backend/checkpoints/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c441dfe412d9d7e47c960029a48a0159c23a38a9ab41b465d90fb1f520d4ced
+size 1222

Backend/checkpoints/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3f804318f6d6b34171ceee3619b326cca29abcc800f2d37fcb85fb0714f761f
+size 5649

Backend/checkpoints/vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe0fda7c425b48c516fc8f160d594c8022a0808447475c1a7c6d6479763f310c
+size 995526

Backend/idioms_structured_1/seed_idioms_en_cleaned.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67f1af4458c3f0defb5f12bccdc3c4cacfeb68e73a03174735e0d64787891af5
+size 12976644

Backend/idioms_structured_1/seed_idioms_es_cleaned.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09b6e067e5ac4e5ddff228011494f1e221a52dfe42ffb45b9ee1183f920c9c4d
+size 1613110

Backend/inference.py

@@ -0,0 +1,219 @@
+# filepath: src/model/inference.py
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+from peft import PeftModel, PeftConfig
+import torch
+import torch.nn.functional as F
+from Idiom_lexicon import KNOWN_IDIOMS
+import fitz  # PyMuPDF
+import tempfile
+from PIL import Image
+import pytesseract
+import nltk
+import spacy
+import json
+from pathlib import Path
+from fastapi import HTTPException
+
+nltk.download('punkt_tab', quiet=True)
+from langdetect import detect
+from nltk.tokenize import sent_tokenize
+import re 
+LANG_MAP = {
+    'en': 'english',
+    'es': 'spanish',
+    # add more if needed
+}
+
+def split_text_by_language(text, language: str):
+    # Map input language (e.g., 'en', 'es') to NLTK language codes
+    nltk_lang = LANG_MAP.get(language.lower(), 'english')
+    sentences = sent_tokenize(text, language=nltk_lang)
+    return sentences
+
+
+def load_model(checkpoint_path):
+    config = PeftConfig.from_pretrained(checkpoint_path)
+    base_model = AutoModelForTokenClassification.from_pretrained(
+        config.base_model_name_or_path,
+        num_labels=3 # O, B-IDIOM, I-IDIOM
+    )
+    model = PeftModel.from_pretrained(base_model, checkpoint_path)
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
+    return model, tokenizer
+
+def normalize_text(text):
+    # Join hyphenated words split across lines
+    text = re.sub(r'-\s*\n\s*', '', text)
+    # Replace newlines with spaces
+    text = re.sub(r'\n+', ' ', text)
+    # Collapse multiple spaces into one
+    text = re.sub(r'\s+', ' ', text)
+    return text.strip()
+
+def filter_idioms(candidate_idioms, known_idioms, min_len=2):
+    filtered = []
+    for idiom in candidate_idioms:
+        norm = idiom.lower().strip()
+        if norm in known_idioms or len(norm.split()) >= min_len:
+            filtered.append(idiom)
+    return filtered
+
+
+import spacy
+import json
+from pathlib import Path
+
+class IdiomMatcher:
+    def __init__(self, idiom_files: dict[str, str]):
+        self.models = {
+            "en": spacy.load("en_core_web_sm"),
+            "es": spacy.load("es_core_news_sm"),
+        }
+        self.idioms_by_lang = {lang: [] for lang in idiom_files}
+        self._load_idioms(idiom_files)
+
+    def _lemmatize(self, text: str, lang: str) -> str:
+        doc = self.models[lang](text)
+        return " ".join(token.lemma_ for token in doc)
+
+    def _load_idioms(self, idiom_files: dict[str, str]):
+        for lang, file_path in idiom_files.items():
+            path = Path(file_path)
+            if not path.exists():
+                raise FileNotFoundError(f"Idiom file not found for {lang}: {file_path}")
+            
+            with open(path, "r", encoding="utf-8") as f:
+                for line in f:
+                    entry = json.loads(line)
+                    idiom_text = entry.get("idiom", "").strip()
+                    if not idiom_text:
+                        continue
+                    entry["lemmatized"] = self._lemmatize(idiom_text, lang)
+                    self.idioms_by_lang[lang].append(entry)
+
+    def match(self, sentence: str, lang: str):
+        if lang not in self.models:
+            raise ValueError(f"Unsupported language: {lang}")
+        sent_lemma = self._lemmatize(sentence, lang)
+        return [
+            idiom for idiom in self.idioms_by_lang[lang]
+            if idiom["lemmatized"] in sent_lemma
+        ]
+
+
+def predict_idiom(text, model, tokenizer, device, conf_threshold=0.9):
+    words = text.split()
+    if not words:
+        print("[⚠️] Empty input text")
+        return []
+
+    inputs = tokenizer(
+        words,
+        is_split_into_words=True,
+        truncation=True,
+        padding=True,
+        max_length=128,
+        return_tensors="pt"
+    ).to(device)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probs = F.softmax(logits, dim=-1)
+
+    max_probs, predictions = torch.max(probs, dim=-1)
+    max_probs = max_probs.cpu().numpy()[0]
+    predictions = predictions.cpu().numpy()[0]
+    word_ids = inputs.word_ids(batch_index=0)
+
+    idioms = []
+    current_idiom_start = -1
+    current_idiom_end = -1
+
+    for i, (pred_label, conf, word_idx) in enumerate(zip(predictions, max_probs, word_ids)):
+        if word_idx is None:
+            if current_idiom_start != -1:
+                idioms.append(' '.join(words[current_idiom_start:current_idiom_end + 1]))
+                current_idiom_start = -1
+                current_idiom_end = -1
+            continue
+
+        if conf < conf_threshold:
+            pred_label = 0
+
+        if pred_label == 1:  # B-IDIOM
+            if current_idiom_start != -1:
+                idioms.append(' '.join(words[current_idiom_start:current_idiom_end + 1]))
+            current_idiom_start = word_idx
+            current_idiom_end = word_idx
+        elif pred_label == 2:  # I-IDIOM
+            if current_idiom_start != -1 and (word_idx == current_idiom_end or word_idx == current_idiom_end + 1):
+                current_idiom_end = word_idx
+            else:
+                if current_idiom_start != -1:
+                    idioms.append(' '.join(words[current_idiom_start:current_idiom_end + 1]))
+                current_idiom_start = -1
+                current_idiom_end = -1
+        else:  # O
+            if current_idiom_start != -1:
+                idioms.append(' '.join(words[current_idiom_start:current_idiom_end + 1]))
+            current_idiom_start = -1
+            current_idiom_end = -1
+
+    if current_idiom_start != -1:
+        idioms.append(' '.join(words[current_idiom_start:current_idiom_end + 1]))
+
+    idioms = filter_idioms(idioms, known_idioms=KNOWN_IDIOMS)
+    return idioms
+
+import pdfplumber
+
+def extract_text_from_pdf(pdf_bytes: bytes) -> str:
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
+        tmp.write(pdf_bytes)
+        tmp_path = tmp.name
+
+    doc = fitz.open(tmp_path)
+    text = ""
+    for i, page in enumerate(doc):
+        page_text = page.get_text()
+        print(f"[DEBUG] Page {i+1} extracted text (first 100 chars): {repr(page_text[:100])}")
+        text += page_text
+    doc.close()
+    text = normalize_text(text)
+    print("[DEBUG] Cleaned extracted text from PDF (first 500 chars):", repr(text[:500]))
+    if not text:
+        print("[⚠️] No text extracted from PDF. It may be blank or not readable.")
+    return text
+
+
+
+def reconstruct_words(tokens, labels):
+    """
+    Reconstruct words from BERT tokens and their corresponding labels.
+    This function is used to map the BERT token predictions back to the original words.
+    """
+    words = []
+    current_word = []
+    current_label = None
+
+    for token, label in zip(tokens, labels):
+        if label == 'O':
+            if current_word:
+                words.append(''.join(current_word))
+                current_word = []
+            continue
+
+        if label.startswith('B-'):
+            if current_word:
+                words.append(''.join(current_word))
+                current_word = []
+            current_label = label[2:]  # Get the idiom type
+            current_word.append(token)
+        elif label.startswith('I-') and current_label == label[2:]:
+            current_word.append(token)
+
+    if current_word:
+        words.append(''.join(current_word))
+
+    return words

Backend/main.py

@@ -0,0 +1,122 @@
+import os
+import torch
+from fastapi import FastAPI
+from pydantic import BaseModel
+from fastapi import UploadFile, File
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi import UploadFile, File
+from inference import extract_text_from_pdf, split_text_by_language, predict_idiom, normalize_text, load_model, IdiomMatcher
+from nltk.tokenize import sent_tokenize
+from langdetect import detect
+from fastapi import HTTPException 
+import re
+import fitz  # PyMuPDF
+# Allow requests from your React app
+origins = [
+    "http://localhost:3000",  # React dev server
+    "https://language-learning-base-website.vercel.app",
+    "https://www.idiomator.com"
+    "https://idiomator.com"
+
+]
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins= origins,  # or ["*"] for all origins (not recommended in production)
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Load model once at startup
+device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
+checkpoint_path = os.path.join(os.path.dirname(__file__), "checkpoints")
+model, tokenizer = load_model(checkpoint_path)
+model = model.to(device)
+model.eval()
+
+class TextRequest(BaseModel):
+    text: str
+    language: str = "en"  # Default to English
+
+class IdiomResponse(BaseModel):
+    idioms: list[str]
+    language: str = "en"  # Default to English
+
+@app.get("/")
+def root():
+    return {"status": "ok"}
+
+
+@app.post("/extract_idioms_ai", response_model=IdiomResponse)
+def extract_idioms(request: TextRequest):
+    import time
+    start = time.time()
+    print(f"[📥] Request received at: {start}")
+
+    text = normalize_text(request.text)
+    language = request.language.lower()  # Get the user-selected language
+
+    sentences = split_text_by_language(text, language=language)
+    idioms = []
+    for sent in sentences:
+        idioms.extend(predict_idiom(sent, model, tokenizer, device))
+    print(f"[✅] Done in {time.time() - start:.3f}s")
+    return {"idioms": idioms}
+
+
+from fastapi import Form
+
+def check_pdf_page_limit(pdf_bytes, max_pages=10):
+    with fitz.open(stream=pdf_bytes, filetype="pdf") as doc:
+        if len(doc) > max_pages:
+            raise HTTPException(status_code=400, detail=f"PDF has {len(doc)} pages. Limit is {max_pages}.")
+
+@app.post("/extract_idioms_pdf_ai", response_model=IdiomResponse)
+async def extract_idioms_pdf(
+    file: UploadFile = File(...),
+    language: str = Form(...)  # ✅ Get language from the client
+):
+    
+    pdf_bytes = await file.read()
+    check_pdf_page_limit(pdf_bytes, max_pages=10)
+    text = extract_text_from_pdf(pdf_bytes)
+    # Normalize the extracted text!
+    text = normalize_text(text)
+    sentences = split_text_by_language(text, language=language)
+    idioms = []
+    for sent in sentences:
+        idioms.extend(predict_idiom(sent, model, tokenizer, device))
+    return {"idioms": idioms}
+
+
+idiom_matcher = IdiomMatcher({
+    "en": "idioms_structured_1/seed_idioms_en_cleaned.jsonl",
+    "es": "idioms_structured_1/seed_idioms_es_cleaned.jsonl"
+})
+
+@app.post("/extract_idioms_heuristic", response_model=IdiomResponse)
+def extract_idioms_heuristic(request: TextRequest):
+    text = normalize_text(request.text)
+    language = request.language.lower()  # get the language from request
+    idiom_matches = idiom_matcher.match(text, lang=language)
+    idioms = [idiom["idiom"] for idiom in idiom_matches]
+
+    return {"idioms": idioms}
+@app.post("/extract_idioms_pdf_heuristic", response_model=IdiomResponse)
+async def extract_idioms_pdf_(
+    file: UploadFile = File(...),
+    language: str = Form(...)  # ✅ Get language from the client
+):
+    pdf_bytes = await file.read()
+    check_pdf_page_limit(pdf_bytes, max_pages=10)
+    text = extract_text_from_pdf(pdf_bytes)
+    # Normalize the extracted text!
+    text = normalize_text(text)
+    sentences = split_text_by_language(text, language=language)
+    idioms = []
+    idiom_matches = idiom_matcher.match(text, lang=language) 
+    idioms = [idiom["idiom"] for idiom in idiom_matches]
+    return {"idioms": idioms}

Backend/requirements.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa0d502a504c0e0ebdff60428904ef42b61af5846a8ff1b0673e6501ef89ff38
+size 515

Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.10-slim
+
+# Create user but don't switch yet
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+# Install Tesseract OCR and system dependencies as root
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends tesseract-ocr libglib2.0-0 libsm6 libxext6 libxrender-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Now switch to the non-root user
+USER user
+
+# Copy requirements.txt
+COPY --chown=user Backend/requirements.txt .
+
+# Install Python dependencies (including nltk and langdetect)
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# (Re)install nltk and download punkt as user, ensuring clean install and data in user dir
+RUN pip install --upgrade --force-reinstall nltk && \
+    python -m nltk.downloader punkt_tab
+
+# Verify punkt is present (will print path in build logs)
+RUN python -c "import nltk; print(nltk.data.find('tokenizers/punkt_tab'))"
+
+# Set PATH for user-installed Python packages
+ENV PATH="/home/user/.local/bin:${PATH}"
+
+# Copy all backend code
+COPY --chown=user Backend/ .
+
+EXPOSE 7860
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,14 @@
+---
+title: Multilingual Idiom Extractor
+emoji: 🌍
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+sdk_version: "latest"
+app_file: main.py
+pinned: false
+---
+
+# Language_Learning_BaseWebsite
+I am just changing this so I have a change
+