Update app.py

app.py

@@ -1,6 +1,13 @@
-
 # app.py
-import os, io, math, json, warnings, hashlib, random
+# Efficient Portfolio Advisor — with dataset-based Low/Medium/High suggestions
+# Modality: Text. Models: yfinance (prices), FRED (risk-free), simple CAPM math,
+# optional reranking with sentence-transformers "FinLang/finance-embeddings-investopedia".
+
+import os
+import io
+import math
+import json
+import warnings
 warnings.filterwarnings("ignore")
 
 from typing import List, Tuple, Dict, Optional
@@ -8,26 +15,24 @@ from typing import List, Tuple, Dict, Optional
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
-import gradio as gr
 from PIL import Image
 import requests
+import gradio as gr
 import yfinance as yf
 
-from sklearn.neighbors import KNeighborsRegressor
-from sklearn.preprocessing import StandardScaler
+# Optional (lazy) import for embeddings
+_ST_MODEL = None
 
-# ---------------- config ----------------
+# ---------- Config ----------
 DATA_DIR = "data"
 os.makedirs(DATA_DIR, exist_ok=True)
 
+MARKET_TICKER = "VOO"
 MAX_TICKERS = 30
 DEFAULT_LOOKBACK_YEARS = 10
-MARKET_TICKER = "VOO"  # fall back to SPY if needed
+DATASET_ROWS = 1000
 
-# UI tables
-POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
-
-# FRED curve mapping: horizon -> series code
+# FRED mappings by horizon
 FRED_MAP = [
     (1,  "DGS1"),
     (2,  "DGS2"),
@@ -40,7 +45,25 @@ FRED_MAP = [
     (100, "DGS30"),
 ]
 
-# ---------------- helpers ----------------
+POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
+SUG_COLS_HOLD = ["pick", "ticker", "weight_%", "amount_$"]
+
+# ---------- Small helpers ----------
+def fmt_pct(x: float, dec: int = 2) -> str:
+    try:
+        return f"{x*100:.{dec}f}%"
+    except Exception:
+        return "—"
+
+def fmt_usd(x: float) -> str:
+    try:
+        return f"${x:,.2f}"
+    except Exception:
+        return "—"
+
+def ensure_dir(p: str):
+    os.makedirs(os.path.dirname(p), exist_ok=True)
+
 def fred_series_for_horizon(years: float) -> str:
     y = max(1.0, min(100.0, float(years)))
     for cutoff, code in FRED_MAP:
@@ -59,31 +82,61 @@ def fetch_fred_yield_annual(code: str) -> float:
     except Exception:
         return 0.03
 
+# ---------- Prices & returns (fix for 'Close' KeyError) ----------
 def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
     start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
     end = pd.Timestamp.today(tz="UTC")
-    df = yf.download(
+
+    raw = yf.download(
         list(dict.fromkeys(tickers)),
         start=start.date(),
         end=end.date(),
         interval="1mo",
-        auto_adjust=True,
+        auto_adjust=False,         # prefer 'Adj Close' if present
         progress=False,
         group_by="ticker",
-    )["Close"]
-    if isinstance(df, pd.Series):
-        df = df.to_frame()
-    df = df.dropna(how="all").fillna(method="ffill")
-    # If yfinance returns MultiIndex columns for multiple tickers, flatten
-    if isinstance(df.columns, pd.MultiIndex):
-        df.columns = [c[0] for c in df.columns]
-    return df
+        threads=False,
+    )
+    if raw is None or len(raw) == 0:
+        return pd.DataFrame()
+
+    # MultiIndex (ticker, field) vs single-index
+    if isinstance(raw.columns, pd.MultiIndex):
+        price = None
+        for field in ("Adj Close", "Close"):
+            if field in raw.columns.get_level_values(-1):
+                price = raw.xs(field, axis=1, level=-1, drop_level=True)
+                break
+        if price is None:
+            price = raw.copy()
+            price.columns = [c[0] if isinstance(c, tuple) else c for c in price.columns]
+    else:
+        if "Adj Close" in raw.columns:
+            price = raw["Adj Close"]
+        elif "Close" in raw.columns:
+            price = raw["Close"]
+        else:
+            price = raw
+
+    if isinstance(price, pd.Series):
+        price = price.to_frame()
+
+    price = price.dropna(how="all").fillna(method="ffill")
+    price = price.loc[:, ~pd.Index(price.columns).duplicated()]
+    return price
 
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
     return prices.pct_change().dropna()
 
+def annualize_mean(m):
+    return np.asarray(m, dtype=float) * 12.0
+
+def annualize_sigma(s):
+    return np.asarray(s, dtype=float) * math.sqrt(12.0)
+
+# ---------- Search & validation ----------
 def yahoo_search(query: str):
-    if not query or len(query.strip()) == 0:
+    if not query or not query.strip():
         return []
     url = "https://query1.finance.yahoo.com/v1/finance/search"
     params = {"q": query.strip(), "quotesCount": 10, "newsCount": 0}
@@ -100,52 +153,35 @@ def yahoo_search(query: str):
             if sym and sym.isascii():
                 out.append({"symbol": sym, "name": name, "exchange": exch})
         if not out:
-            out = [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n/a"}]
+            out = [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "—"}]
         return out[:10]
     except Exception:
-        return [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n/a"}]
+        return [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "—"}]
 
 def validate_tickers(symbols: List[str], years: int) -> List[str]:
-    if not symbols:
-        return []
-    # Always include market proxy so alignment works
-    base = [s for s in dict.fromkeys(symbols)]
+    # include market to keep alignment, but validate only user symbols
+    base = list(dict.fromkeys([s.strip().upper() for s in symbols if s.strip()]))[:MAX_TICKERS]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
     ok = [s for s in base if s in px.columns]
-    # If market ticker missing, try SPY as fallback
-    if MARKET_TICKER not in px.columns and "SPY" not in px.columns:
-        # Try once more with SPY added
-        px2 = fetch_prices_monthly(base + ["SPY"], years)
-        ok = [s for s in base if s in px2.columns]
     return ok
 
-# -------------- aligned moments --------------
+# ---------- Aligned CAPM moments ----------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols) if c]
+    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
     tickers = uniq + [MARKET_TICKER]
     px = fetch_prices_monthly(tickers, years)
-    # if VOO missing, try SPY as market
-    mkt = MARKET_TICKER if MARKET_TICKER in px.columns else ("SPY" if "SPY" in px.columns else None)
-    if mkt is None:
-        return pd.DataFrame()
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns] + [mkt]
+    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
     R = rets[cols].dropna(how="any")
-    return R, mkt
-
-def annualize_mean(m):
-    return np.asarray(m, dtype=float) * 12.0
-
-def annualize_sigma(s):
-    return np.asarray(s, dtype=float) * math.sqrt(12.0)
+    return R.loc[:, ~R.columns.duplicated()]
 
 def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
-    R, mkt = get_aligned_monthly_returns(symbols, years)
-    if R is None or R.empty or mkt is None or R.shape[0] < 3:
-        raise ValueError("Not enough aligned data for selected tickers / lookback.")
+    R = get_aligned_monthly_returns(symbols, years)
+    if MARKET_TICKER not in R.columns or R.shape[0] < 3:
+        raise ValueError("Not enough aligned data to estimate moments.")
     rf_m = rf_ann / 12.0
 
-    m = R[mkt]
+    m = R[MARKET_TICKER]
     if isinstance(m, pd.DataFrame):
         m = m.iloc[:, 0].squeeze()
 
@@ -158,17 +194,16 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
     var_m = max(var_m, 1e-6)
 
     betas: Dict[str, float] = {}
-    for s in [c for c in R.columns if c != mkt]:
+    for s in [c for c in R.columns if c != MARKET_TICKER]:
         ex_s = R[s] - rf_m
         betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
+    betas[MARKET_TICKER] = 1.0
 
-    betas[mkt] = 1.0
-    # asset covariance (annualized) excluding market column
-    asset_cols = [c for c in R.columns if c != mkt]
+    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
     cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
     covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
 
-    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann, "mkt": mkt}
+    return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
 
 def capm_er(beta: float, rf_ann: float, erp_ann: float) -> float:
     return float(rf_ann + beta * erp_ann)
@@ -179,19 +214,20 @@ def portfolio_stats(weights: Dict[str, float],
                     rf_ann: float,
                     erp_ann: float) -> Tuple[float, float, float]:
     tickers = list(weights.keys())
+    if len(tickers) == 0:
+        return 0.0, 0.0, 0.0
     w = np.array([weights[t] for t in tickers], dtype=float)
     gross = float(np.sum(np.abs(w)))
-    if gross == 0:
+    if gross <= 1e-12:
         return 0.0, 0.0, 0.0
     w_expo = w / gross
     beta_p = float(np.dot([betas.get(t, 0.0) for t in tickers], w_expo))
     er_p = capm_er(beta_p, rf_ann, erp_ann)
     cov = cov_ann.reindex(index=tickers, columns=tickers).fillna(0.0).to_numpy()
-    v = float(w_expo.T @ cov @ w_expo)
-    sigma_p = math.sqrt(max(v, 0.0))
+    sigma_p = math.sqrt(float(max(w_expo.T @ cov @ w_expo, 0.0)))
     return beta_p, er_p, sigma_p
 
-# -------------- CML helpers --------------
+# ---------- CML helpers (plot in %) ----------
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
     if sigma_mkt <= 1e-12:
         return 0.0, 1.0, rf_ann
@@ -204,55 +240,43 @@ def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma
     a = (mu_target - rf_ann) / erp_ann
     return a, 1.0 - a, abs(a) * sigma_mkt
 
-def plot_cml(
+def plot_cml_percent(
     rf_ann, erp_ann, sigma_mkt,
     pt_sigma, pt_mu,
     same_sigma_sigma, same_sigma_mu,
     same_mu_sigma, same_mu_mu,
-    sugg_sigma=None, sugg_mu=None
+    suggestion_sigma=None, suggestion_mu=None
 ) -> Image.Image:
-    fig = plt.figure(figsize=(6.2, 4.2), dpi=120)
+    fig = plt.figure(figsize=(6, 4), dpi=120)
 
     xmax = max(
-        0.30,
+        0.3,
         sigma_mkt * 2.0,
         pt_sigma * 1.4,
         same_mu_sigma * 1.4,
         same_sigma_sigma * 1.4,
-        (sugg_sigma or 0.0) * 1.4,
+        (suggestion_sigma or 0.0) * 1.4,
     )
     xs = np.linspace(0, xmax, 160)
     slope = erp_ann / max(sigma_mkt, 1e-12)
     cml = rf_ann + slope * xs
-    plt.plot(xs * 100.0, cml * 100.0, label="CML via Market")
-
-    # key points
-    plt.scatter([0.0], [rf_ann * 100.0], label="Risk-free (FRED)")
-    plt.scatter([sigma_mkt * 100.0], [(rf_ann + erp_ann) * 100.0], label="Market (VOO)")
-    plt.scatter([pt_sigma * 100.0], [pt_mu * 100.0], label="Your portfolio")
-
-    plt.scatter([same_sigma_sigma * 100.0], [same_sigma_mu * 100.0], label="Efficient same sigma")
-    plt.scatter([same_mu_sigma * 100.0], [same_mu_mu * 100.0], label="Efficient same return")
-
-    if sugg_sigma is not None and sugg_mu is not None:
-        plt.scatter([sugg_sigma * 100.0], [sugg_mu * 100.0], label="Suggestion")
-
-    # simple guides
-    plt.plot(
-        [pt_sigma * 100.0, same_sigma_sigma * 100.0],
-        [pt_mu * 100.0, same_sigma_mu * 100.0],
-        linestyle="--", linewidth=1.1, alpha=0.7, color="gray",
-    )
-    plt.plot(
-        [pt_sigma * 100.0, same_mu_sigma * 100.0],
-        [pt_mu * 100.0, same_mu_mu * 100.0],
-        linestyle="--", linewidth=1.1, alpha=0.7, color="gray",
-    )
-
-    plt.xlabel("σ (annualized)")
-    plt.ylabel("Expected return (annual)")
-    plt.gca().xaxis.set_major_formatter(lambda v, pos: f"{v:.0f}%")
-    plt.gca().yaxis.set_major_formatter(lambda v, pos: f"{v:.0f}%")
+    plt.plot(xs * 100, cml * 100, label="CML via Market")
+
+    # Points
+    plt.scatter([0.0], [rf_ann * 100], label="Risk-free (FRED)")
+    plt.scatter([sigma_mkt * 100], [(rf_ann + erp_ann) * 100], label="Market VOO")
+    plt.scatter([pt_sigma * 100], [pt_mu * 100], label="Your portfolio")
+    plt.scatter([same_sigma_sigma * 100], [same_sigma_mu * 100], label="Efficient same sigma")
+    plt.scatter([same_mu_sigma * 100], [same_mu_mu * 100], label="Efficient same return")
+    if suggestion_sigma is not None and suggestion_mu is not None:
+        plt.scatter([suggestion_sigma * 100], [suggestion_mu * 100], label="Suggestion")
+
+    # simple dotted guides
+    plt.plot([pt_sigma * 100, same_sigma_sigma * 100], [pt_mu * 100, same_sigma_mu * 100], linestyle="--", lw=1, alpha=0.7, color="gray")
+    plt.plot([pt_sigma * 100, same_mu_sigma * 100], [pt_mu * 100, same_mu_mu * 100], linestyle="--", lw=1, alpha=0.7, color="gray")
+
+    plt.xlabel("σ (annualized, %)")
+    plt.ylabel("Expected return (annual, %)")
     plt.legend(loc="best", fontsize=8)
     plt.tight_layout()
 
@@ -262,10 +286,10 @@ def plot_cml(
     buf.seek(0)
     return Image.open(buf)
 
-# -------------- synthetic dataset --------------
-def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
+# ---------- Synthetic dataset (built only from current universe) ----------
+def _row_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
     try:
-        ts = [t.strip().upper() for t in str(row["tickers"]).split(",") if t.strip()]
+        ts = [t.strip() for t in str(row["tickers"]).split(",")]
         ws = [float(x) for x in str(row["weights"]).split(",")]
         wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
         w = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
@@ -276,163 +300,131 @@ def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarra
     except Exception:
         return None
 
-def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float) -> pd.DataFrame:
-    symbols = list(sorted(set([s for s in universe if s])))
-    moms = estimate_all_moments_aligned(symbols, years, rf_ann)
+def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float, n_rows: int = DATASET_ROWS) -> pd.DataFrame:
+    # require MARKET_TICKER present for moments; weights exclude it unless random pick includes
+    moms = estimate_all_moments_aligned(universe, years, rf_ann)
     covA, betas = moms["cov_ann"], moms["betas"]
 
-    rows, rng = [], np.random.default_rng(12345)
-    for i in range(1000):
-        k = int(rng.integers(low=min(2, len(symbols)), high=min(8, len(symbols)) + 1))
-        picks = list(rng.choice(symbols, size=k, replace=False))
-        signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
+    rng = np.random.default_rng(12345)
+    rows = []
+    for i in range(n_rows):
+        k = int(rng.integers(low=min(2, len(universe)), high=min(8, len(universe)) + 1))
+        picks = list(rng.choice(universe, size=k, replace=False))
+        signs = rng.choice([-1.0, 1.0], size=k, p=[0.2, 0.8])
         raw = rng.dirichlet(np.ones(k))
-        gross = 1.0 + float(rng.gamma(2.0, 0.7))
+        gross = 1.0 + float(rng.gamma(2.0, 0.5))
         w = gross * signs * raw
+        # portfolio stats
         beta_p, er_p, sigma_p = portfolio_stats({picks[j]: w[j] for j in range(k)}, covA, betas, rf_ann, erp_ann)
         rows.append({
             "id": i,
             "tickers": ",".join(picks),
             "weights": ",".join(f"{x:.6f}" for x in w),
-            "beta_p": beta_p,
             "er_p": er_p,
-            "sigma_p": sigma_p
+            "sigma_p": sigma_p,
+            "beta_p": beta_p
         })
     return pd.DataFrame(rows)
 
-def save_synth_csv(df: pd.DataFrame, universe: List[str]) -> str:
-    sig = hashlib.md5((",".join(sorted(universe)) + f":{len(df)}").encode()).hexdigest()[:8]
-    path = os.path.join(DATA_DIR, f"investor_profiles_{sig}.csv")
-    df.to_csv(path, index=False)
-    return path
-
-# -------------- suggestion logic (dataset only, optional embeddings) --------------
-def describe_candidate_text(row: pd.Series, universe: List[str]) -> str:
-    xs = _row_to_exposures(row, universe)
-    if xs is None:
-        return ""
-    parts = []
-    for t, w in sorted(zip(universe, xs), key=lambda z: -abs(z[1]))[:8]:
-        if abs(w) > 1e-4:
-            parts.append(f"{t} {w:+.2f}")
-    desc = " ".join(parts)
-    return f"weights {desc}; beta {row['beta_p']:.2f}; sigma {row['sigma_p']:.2f}; return {row['er_p']:.2f}"
-
-def pick_by_risk_from_dataset(csv_path: str,
-                              universe: List[str],
-                              risk_label: str,
-                              use_embeddings: bool) -> Optional[Dict]:
+def dataset_path_for_universe(universe: List[str]) -> str:
+    key = ",".join(sorted(universe))
+    h = abs(hash(key)) % (10**8)
+    p = os.path.join(DATA_DIR, f"investor_profiles_{h}.csv")
+    return p
+
+# ---------- Suggestion logic (Low / Medium / High) ----------
+def _risk_targets(sigmas: np.ndarray) -> Dict[str, float]:
+    # choose targets by quantiles of dataset sigma
+    return {
+        "Low": float(np.quantile(sigmas, 0.15)),
+        "Medium": float(np.quantile(sigmas, 0.50)),
+        "High": float(np.quantile(sigmas, 0.85)),
+    }
+
+def _describe_row_for_embeddings(row: pd.Series, universe: List[str]) -> str:
+    # text description for semantic reranking
+    parts = [f"sigma {row['sigma_p']:.4f}", f"beta {row['beta_p']:.2f}", f"expected return {row['er_p']:.4f}"]
+    ex = _row_exposures(row, universe)
+    if ex is not None:
+        top = sorted([(universe[i], float(abs(ex[i]))) for i in range(len(universe))], key=lambda kv: -kv[1])[:4]
+        parts.append("focus on " + ", ".join([f"{t}:{w:.2f}" for t, w in top]))
+    return " ".join(parts)
+
+def _get_prompt(risk_level: str) -> str:
+    if risk_level == "Low":
+        return "low risk, stable, conservative diversified portfolio"
+    if risk_level == "High":
+        return "high risk, growth oriented, aggressive portfolio"
+    return "balanced moderate risk diversified portfolio"
+
+def _maybe_load_st_model():
+    global _ST_MODEL
+    if _ST_MODEL is None:
+        from sentence_transformers import SentenceTransformer
+        _ST_MODEL = SentenceTransformer("FinLang/finance-embeddings-investopedia")
+    return _ST_MODEL
+
+def suggest_from_dataset(csv_path: str,
+                         universe: List[str],
+                         total_amount: float,
+                         risk_level: str,
+                         use_embeddings: bool = False):
     try:
         df = pd.read_csv(csv_path)
     except Exception:
-        return None
+        return pd.DataFrame(columns=SUG_COLS_HOLD), None
+
     if df.empty:
-        return None
+        return pd.DataFrame(columns=SUG_COLS_HOLD), None
 
-    # candidates by sigma
-    sigmas = df["sigma_p"].astype(float).values
-    order_low = np.argsort(sigmas)
-    order_high = order_low[::-1]
-    med_value = float(np.median(sigmas))
-    order_mid = np.argsort(np.abs(sigmas - med_value))
-
-    if risk_label.lower() == "low":
-        idxs = order_low[:30]
-    elif risk_label.lower() == "high":
-        idxs = order_high[:30]
-    else:
-        idxs = order_mid[:30]
+    sigmas = df["sigma_p"].to_numpy(dtype=float)
+    targets = _risk_targets(sigmas)
+    target_sigma = targets.get(risk_level, targets["Medium"])
 
-    sub = df.iloc[idxs].copy()
-    if sub.empty:
-        return None
+    # distance to target sigma
+    df = df.copy()
+    df["dist"] = (df["sigma_p"] - target_sigma).abs()
+
+    # Take a reasonable candidate pool
+    cand = df.nsmallest(100, "dist").reset_index(drop=True)
 
-    # optional: rerank with finance embeddings against a risk prompt
+    # Optional semantic rerank
     if use_embeddings:
-        prompt_map = {
-            "low": "low risk, stable, diversified, defensive, downside protection",
-            "medium": "balanced risk, moderate volatility, diversified growth and income",
-            "high": "high risk, aggressive growth, momentum, high volatility"
-        }
-        prompt = prompt_map.get(risk_label.lower(), prompt_map["medium"])
-        try:
-            from sentence_transformers import SentenceTransformer, util
-            model = SentenceTransformer("FinLang/finance-embeddings-investopedia")
-            cand_texts = [describe_candidate_text(r, universe) for _, r in sub.iterrows()]
-            emb_prompt = model.encode([prompt], normalize_embeddings=True)
-            emb_cands = model.encode(cand_texts, normalize_embeddings=True)
-            sims = util.cos_sim(emb_prompt, emb_cands).cpu().numpy()[0]
-            best_i = int(np.argsort(-sims)[0])
-            chosen = sub.iloc[best_i]
-        except Exception:
-            chosen = sub.iloc[0]
-    else:
-        chosen = sub.iloc[0]
+        model = _maybe_load_st_model()
+        prompt = _get_prompt(risk_level)
+        texts = [prompt] + [ _describe_row_for_embeddings(r, universe) for _, r in cand.iterrows() ]
+        embs = model.encode(texts)
+        S = model.similarity(embs[0:1], embs[1:]).flatten()  # cosine similarity
+        cand = cand.assign(sim=S).sort_values("sim", ascending=False).head(50).reset_index(drop=True)
+
+    # Now pick the top 3 by a combined score (distance, then ER desc)
+    cand["score"] = cand["dist"] - 0.2 * cand["er_p"]  # small bias toward higher ER
+    picks = cand.nsmallest(3, "score").reset_index(drop=True)
+
+    # Build a simple holdings table: percent and dollars
+    hold_rows = []
+    first_pick_mu = None
+    first_pick_sigma = None
+    for i, row in picks.iterrows():
+        expo = _row_exposures(row, universe)
+        if expo is None:
+            continue
+        if first_pick_mu is None:
+            first_pick_mu = float(row["er_p"])
+            first_pick_sigma = float(row["sigma_p"])
+        wmap = {universe[j]: float(expo[j]) for j in range(len(universe)) if abs(float(expo[j])) > 1e-4}
+        for t, w in sorted(wmap.items(), key=lambda kv: -abs(kv[1]))[:12]:
+            hold_rows.append({
+                "pick": i + 1,
+                "ticker": t,
+                "weight_%": round(w * 100.0, 2),
+                "amount_$": round(w * total_amount, 2)
+            })
 
-    # convert chosen row to exposure map on universe
-    xs = _row_to_exposures(chosen, universe)
-    if xs is None:
-        return None
-    wmap = {t: float(xs[i]) for i, t in enumerate(universe) if abs(xs[i]) > 1e-4}
-    return {"weights": wmap,
-            "er": float(chosen["er_p"]),
-            "sigma": float(chosen["sigma_p"]),
-            "beta": float(chosen["beta_p"])}
-
-def build_simple_suggestion_table(weights_exposure: Dict[str, float],
-                                  gross_capital: float,
-                                  top_n: int = 12) -> pd.DataFrame:
-    rows = []
-    for t, w in sorted(weights_exposure.items(), key=lambda kv: -abs(kv[1]))[:top_n]:
-        rows.append({
-            "ticker": t,
-            "weight_%": round(float(w) * 100.0, 2),
-            "dollars_$": round(float(w) * float(gross_capital), 0)
-        })
-    return pd.DataFrame(rows, columns=["ticker", "weight_%", "dollars_$"])
-
-# -------------- summary builder --------------
-def fmt_pct(x: float) -> str:
-    return f"{x*100:.2f}%"
-
-def build_summary_md(lookback, horizon, rf, rf_code, erp, sigma_mkt,
-                     beta_p, er_p, sigma_p,
-                     a_sigma, b_sigma, mu_eff_sigma,
-                     a_mu, b_mu, sigma_eff_mu,
-                     sugg=None, risk_label=None) -> str:
-    lines = []
-    lines.append("### Inputs")
-    lines.append(f"- Lookback years: **{lookback}**")
-    lines.append(f"- Horizon years: **{int(round(horizon))}**")
-    lines.append(f"- Risk-free: **{fmt_pct(rf)}** (FRED {rf_code})")
-    lines.append(f"- Market ERP: **{fmt_pct(erp)}**")
-    lines.append(f"- Market σ: **{fmt_pct(sigma_mkt)}**")
-    lines.append("")
-    lines.append("### Your portfolio (CAPM expectations)")
-    lines.append(f"- Beta: **{beta_p:.2f}**")
-    lines.append(f"- σ: **{fmt_pct(sigma_p)}**")
-    lines.append(f"- Expected return: **{fmt_pct(er_p)}**")
-    lines.append("")
-    lines.append("### Efficient alternatives on CML")
-    lines.append("Same σ as your portfolio")
-    lines.append(f"- Market weight **{a_sigma:.2f}**, Bills weight **{b_sigma:.2f}**")
-    lines.append(f"- Expected return **{fmt_pct(mu_eff_sigma)}**")
-    lines.append("Same μ as your portfolio")
-    lines.append(f"- Market weight **{a_mu:.2f}**, Bills weight **{b_mu:.2f}**")
-    lines.append(f"- σ **{fmt_pct(sigma_eff_mu)}**")
-    if sugg is not None:
-        lines.append("")
-        lines.append(f"### Dataset-based suggestion (risk: **{risk_label}**)")
-        lines.append(f"- Suggested β **{sugg['beta']:.2f}**, σ **{fmt_pct(sugg['sigma'])}**, μ **{fmt_pct(sugg['er'])}**")
-    return "\n".join(lines)
-
-# -------------- global state --------------
-UNIVERSE = [MARKET_TICKER, "QQQ", "XLK", "XLP", "XLE", "VNQ", "IEF", "HYG", "GLD", "EEM"]
-HORIZON_YEARS = 10
-RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
-RF_ANN = fetch_fred_yield_annual(RF_CODE)
+    hold_df = pd.DataFrame(hold_rows, columns=SUG_COLS_HOLD)
+    return hold_df, (first_pick_mu, first_pick_sigma)
 
-# -------------- gradio callbacks --------------
+# ---------- UI callbacks ----------
 def search_tickers_cb(q: str):
     hits = yahoo_search(q)
     if not hits:
@@ -442,12 +434,13 @@ def search_tickers_cb(q: str):
 
 def add_symbol(selection: str, table: pd.DataFrame):
     if not selection:
-        return table, "Pick a row from Matches first"
+        return table, "Pick a row from Matches first."
     symbol = selection.split("|")[0].strip().upper()
     current = [] if table is None or len(table) == 0 else [str(x).upper() for x in table["ticker"].tolist() if str(x) != "nan"]
     tickers = current if symbol in current else current + [symbol]
     val = validate_tickers(tickers, years=DEFAULT_LOOKBACK_YEARS)
     tickers = [t for t in tickers if t in val]
+    # preserve amounts
     amt_map = {}
     if table is not None and len(table) > 0:
         for _, r in table.iterrows():
@@ -471,6 +464,11 @@ def lock_ticker_column(tb: pd.DataFrame):
     amounts = amounts[:len(tickers)] + [0.0] * max(0, len(tickers) - len(amounts))
     return pd.DataFrame({"ticker": tickers, "amount_usd": amounts})
 
+# Global horizon & rf on change (persisted during session)
+HORIZON_YEARS = 10
+RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
+RF_ANN = fetch_fred_yield_annual(RF_CODE)
+
 def set_horizon(years: float):
     y = max(1.0, min(100.0, float(years)))
     code = fred_series_for_horizon(y)
@@ -479,141 +477,157 @@ def set_horizon(years: float):
     HORIZON_YEARS = y
     RF_CODE = code
     RF_ANN = rf
-    return f"Risk-free series {code}. Latest annual rate {rf:.2%}. Will be used on compute."
+    return f"Risk-free series {code}. Latest annual rate {fmt_pct(rf)}. Horizon set to {int(round(y))} years."
 
-def compute(lookback: int,
+def compute(lookback_years: int,
             table: pd.DataFrame,
-            risk_label: str,
+            risk_level: str,
             use_embeddings: bool):
-
-    if table is None or len(table) == 0:
-        return None, "Add at least one ticker", "Universe empty", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
-
-    df = table.dropna().copy()
+    # ---- read table
+    df = table.dropna()
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
 
     symbols = [t for t in df["ticker"].tolist() if t]
-    symbols = validate_tickers(symbols, lookback)
     if len(symbols) == 0:
-        return None, "Could not validate any tickers", "Universe invalid", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
+        return None, "Add at least one ticker.", "—", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
 
-    global UNIVERSE
-    UNIVERSE = list(sorted(set([s for s in symbols])))[:MAX_TICKERS]
+    symbols = validate_tickers(symbols, lookback_years)
+    if len(symbols) == 0:
+        return None, "Could not validate any tickers.", "—", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
+
+    universe = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
 
-    # amounts & gross (gross = sum of absolute exposures)
-    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows() if r["ticker"] in UNIVERSE}
-    gross = float(sum(abs(v) for v in amounts.values()))
-    rf_ann = RF_ANN
+    df = df[df["ticker"].isin(symbols)].copy()
+    amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
+    total_amt = float(sum(abs(v) for v in amounts.values()))
+    if total_amt <= 1e-12:
+        return None, "All amounts are zero.", f"Universe set to {', '.join(universe)}", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=SUG_COLS_HOLD), None
+    weights = {k: v / total_amt for k, v in amounts.items()}
 
-    # aligned moments
-    moms = estimate_all_moments_aligned(UNIVERSE, lookback, rf_ann)
+    # ---- moments & portfolio metrics
+    moms = estimate_all_moments_aligned(universe, lookback_years, RF_ANN)
     betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
+    beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, RF_ANN, erp_ann)
 
-    if gross <= 1e-12:
-        return None, "All amounts are zero", f"Universe set to: {', '.join(UNIVERSE)}", pd.DataFrame(columns=POS_COLS), pd.DataFrame(columns=["ticker","weight_%","dollars_$"]), None
+    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, RF_ANN, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, RF_ANN, erp_ann, sigma_mkt)
 
-    weights = {k: v / gross for k, v in amounts.items()}
-    beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
+    # ---- dataset build (only for current universe)
+    csv_path = dataset_path_for_universe(universe)
+    if not os.path.exists(csv_path):
+        synth = build_synthetic_dataset(universe, lookback_years, RF_ANN, erp_ann, n_rows=DATASET_ROWS)
+        ensure_dir(csv_path)
+        synth.to_csv(csv_path, index=False)
 
-    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
-    a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, rf_ann, erp_ann, sigma_mkt)
+    # ---- dataset-based suggestions (simple table: percent & dollars)
+    hold_df, first_pick_pt = suggest_from_dataset(csv_path, universe, total_amt, risk_level, use_embeddings)
+    sug_mu, sug_sigma = (first_pick_pt if first_pick_pt is not None else (None, None))
 
-    # build (or reuse) synthetic dataset for this universe
-    csv_path = None
-    # make a stable filename per-universe
-    sig = hashlib.md5((",".join(sorted(UNIVERSE)) + f":{lookback}:{RF_CODE}").encode()).hexdigest()[:8]
-    candidate_path = os.path.join(DATA_DIR, f"investor_profiles_{sig}.csv")
-    if os.path.exists(candidate_path):
-        csv_path = candidate_path
-    else:
-        synth_df = build_synthetic_dataset(UNIVERSE, years=lookback, rf_ann=rf_ann, erp_ann=erp_ann)
-        csv_path = save_synth_csv(synth_df, UNIVERSE)
-
-    # dataset-based suggestion by risk
-    sug = pick_by_risk_from_dataset(csv_path, UNIVERSE, risk_label=risk_label, use_embeddings=use_embeddings)
-    suggestion_df = pd.DataFrame(columns=["ticker","weight_%","dollars_$"])
-    sugg_sigma_plot = None
-    sugg_mu_plot = None
-    if sug is not None:
-        suggestion_df = build_simple_suggestion_table(sug["weights"], gross_capital=gross)
-        sugg_sigma_plot = sug["sigma"]
-        sugg_mu_plot = sug["er"]
-
-    # positions table (computed from user's inputs)
-    rows = []
-    for t in UNIVERSE:
-        if t in amounts:
-            beta_val = 1.0 if t == moms["mkt"] else betas.get(t, np.nan)
-            rows.append({
-                "ticker": t,
-                "amount_usd": float(amounts.get(t, 0.0)),
-                "weight_exposure": float(weights.get(t, 0.0)),
-                "beta": float(beta_val),
-            })
-    pos_table = pd.DataFrame(rows, columns=POS_COLS)
-
-    # plot & summary
-    img = plot_cml(
-        rf_ann, erp_ann, sigma_mkt,
+    # ---- plot
+    img = plot_cml_percent(
+        RF_ANN, erp_ann, sigma_mkt,
         sigma_p, er_p,
         sigma_p, mu_eff_sigma,
         sigma_eff_mu, er_p,
-        sugg_sigma=sugg_sigma_plot, sugg_mu=sugg_mu_plot
+        suggestion_sigma=sug_sigma, suggestion_mu=sug_mu
     )
 
-    info = build_summary_md(
-        lookback, HORIZON_YEARS, rf_ann, RF_CODE, erp_ann, sigma_mkt,
-        beta_p, er_p, sigma_p,
-        a_sigma, b_sigma, mu_eff_sigma,
-        a_mu, b_mu, sigma_eff_mu,
-        sugg=sug, risk_label=risk_label
-    )
+    # ---- summary (percent everywhere)
+    info_lines = []
+    info_lines.append("### Inputs")
+    info_lines.append(f"- Lookback years {int(lookback_years)}")
+    info_lines.append(f"- Horizon years {int(round(HORIZON_YEARS))}")
+    info_lines.append(f"- Risk-free {fmt_pct(RF_ANN)} from {RF_CODE}")
+    info_lines.append(f"- Market ERP {fmt_pct(erp_ann)}")
+    info_lines.append(f"- Market σ {fmt_pct(sigma_mkt)}")
+    info_lines.append("")
+    info_lines.append("### Your portfolio (CAPM expectations)")
+    info_lines.append(f"- Beta {beta_p:.2f}")
+    info_lines.append(f"- σ {fmt_pct(sigma_p)}")
+    info_lines.append(f"- Expected return {fmt_pct(er_p)}")
+    info_lines.append("")
+    info_lines.append("### Efficient alternatives on CML")
+    info_lines.append(f"- Same σ as your portfolio → Market weight {a_sigma:.2f}, Bills weight {b_sigma:.2f}, return {fmt_pct(mu_eff_sigma)}")
+    info_lines.append(f"- Same expected return → Market weight {a_mu:.2f}, Bills weight {b_mu:.2f}, σ {fmt_pct(sigma_eff_mu)}")
+    info_lines.append("")
+    info_lines.append(f"### Dataset-based suggestions (risk: {risk_level})")
+    info_lines.append("- Shown below as simple holdings: percent of exposure and dollars allocated.")
+    if use_embeddings:
+        info_lines.append("- Reranked with finance embeddings (FinLang/finance-embeddings-investopedia).")
 
-    uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
-    return img, info, uni_msg, pos_table, suggestion_df, csv_path
+    info = "\n".join(info_lines)
 
-# -------------- UI --------------
+    # ---- positions table for current inputs
+    rows = []
+    for t in symbols:
+        beta_val = 1.0 if t == MARKET_TICKER else betas.get(t, np.nan)
+        rows.append({
+            "ticker": t,
+            "amount_usd": round(amounts.get(t, 0.0), 2),
+            "weight_exposure": round(weights.get(t, 0.0), 6),
+            "beta": round(beta_val, 6),
+        })
+    pos_table = pd.DataFrame(rows, columns=POS_COLS)
+
+    uni_msg = f"Universe set to: {', '.join(universe)}"
+    return img, info, uni_msg, pos_table, hold_df, csv_path
+
+# ---------- UI ----------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
+    with gr.Accordion("About (assignment section 1)", open=False):
+        gr.Markdown(
+            "**Modality**: Text.\n\n"
+            "**Use case**: Given a user’s stock/ETF universe and current dollar amounts, the system recommends three "
+            "alternative mixes (Low / Medium / High risk) generated from a 1,000-row dataset of random portfolios built "
+            "only from the user’s current universe.\n\n"
+            "**System goal**: User provides text inputs (tickers and amounts). The system returns three similar items "
+            "(suggested mixes) from the dataset. Optional reranking uses the text-embedding model "
+            "`FinLang/finance-embeddings-investopedia`."
+        )
+
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
-        "Search symbols, enter dollar amounts, set your horizon. Prices from Yahoo Finance. Risk-free from FRED. "
-        "Low/Medium/High suggestions are chosen only from a 1,000-row dataset generated from your current universe, "
+        "Search symbols, enter dollar amounts, set your horizon. Prices from Yahoo Finance. "
+        "Risk-free from FRED. Low/Medium/High suggestions are chosen only from a 1,000-row dataset generated from your current universe, "
         "optionally refined with finance embeddings."
     )
 
     with gr.Row():
         with gr.Column(scale=1):
+            # search
             q = gr.Textbox(label="Search symbol")
-            search_note = gr.Markdown()
+            search_note = gr.Markdown(" ")
             matches = gr.Dropdown(choices=[], label="Matches")
             with gr.Row():
                 search_btn = gr.Button("Search")
                 add_btn = gr.Button("Add selected to portfolio")
 
+            # portfolio table
             gr.Markdown("### Portfolio positions — type dollar amounts (negatives allowed for shorts)")
             table = gr.Dataframe(
                 headers=["ticker", "amount_usd"],
                 datatype=["str", "number"],
                 row_count=0,
                 col_count=(2, "fixed"),
-                wrap=True,
+                value=pd.DataFrame(columns=["ticker", "amount_usd"])
             )
 
+            # horizon & lookback
             horizon = gr.Number(label="Horizon in years (1–100)", value=HORIZON_YEARS, precision=0)
             lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for beta & sigma")
 
+            # suggestions controls
             gr.Markdown("### Suggestions")
-            risk = gr.Radio(choices=["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
-            use_emb = gr.Checkbox(label="Use finance embeddings to refine picks", value=True)
+            risk = gr.Radio(["Low", "Medium", "High"], value="Medium", label="Risk tolerance")
+            use_st = gr.Checkbox(label="Use finance embeddings to refine picks", value=False)
 
             run_btn = gr.Button("Compute (build dataset & suggest)")
 
         with gr.Column(scale=1):
             plot = gr.Image(label="Capital Market Line (CML)", type="pil")
-            summary = gr.Markdown(label="Inputs & Results")
+            summary = gr.Markdown(label="Summary")
             universe_msg = gr.Textbox(label="Universe status", interactive=False)
-
             positions = gr.Dataframe(
                 label="Computed positions",
                 headers=POS_COLS,
@@ -622,18 +636,17 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=pd.DataFrame(columns=POS_COLS),
                 interactive=False
             )
-
             suggestions = gr.Dataframe(
-                label="Suggested holdings (weights are % of gross capital; negatives = shorts)",
-                headers=["ticker", "weight_%", "dollars_$"],
-                datatype=["str", "number", "number"],
-                col_count=(3, "fixed"),
-                value=pd.DataFrame(columns=["ticker","weight_%","dollars_$"]),
+                label="Dataset-based suggestions (top 3 — holdings shown as % and $)",
+                headers=SUG_COLS_HOLD,
+                datatype=["number", "str", "number", "number"],
+                col_count=(len(SUG_COLS_HOLD), "fixed"),
+                value=pd.DataFrame(columns=SUG_COLS_HOLD),
                 interactive=False
             )
-
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
+    # wiring
     def do_search(query):
         note, options = search_tickers_cb(query)
         return note, gr.update(choices=options)
@@ -645,7 +658,7 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
 
     run_btn.click(
         fn=compute,
-        inputs=[lookback, table, risk, use_emb],
+        inputs=[lookback, table, risk, use_st],
         outputs=[plot, summary, universe_msg, positions, suggestions, dl]
     )