Update app.py

app.py

@@ -1,21 +1,17 @@
+# app.py
 import os, io, math, warnings
 warnings.filterwarnings("ignore")
 
 from typing import List, Tuple, Dict, Optional
-from functools import partial
 
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
-from matplotlib.ticker import PercentFormatter
 import gradio as gr
 from PIL import Image
 import requests
 import yfinance as yf
 
-from sklearn.neighbors import KNeighborsRegressor
-from sklearn.preprocessing import StandardScaler
-
 # ---------------- config ----------------
 DATA_DIR = "data"
 DATASET_PATH = os.path.join(DATA_DIR, "investor_profiles.csv")
@@ -24,9 +20,8 @@ MAX_TICKERS = 30
 DEFAULT_LOOKBACK_YEARS = 5
 MARKET_TICKER = "VOO"
 
-# column schemas (weights shown in percent in UI tables)
-POS_COLS = ["ticker", "amount_usd", "weight_%", "beta"]
-SUG_RISK_COLS = ["ticker", "suggested_weight_%"]
+POS_COLS = ["ticker", "amount_usd", "weight_exposure", "beta"]
+SUG_COLS = ["ticker", "suggested_weight_pct"]
 
 FRED_MAP = [
     (1,  "DGS1"),
@@ -47,8 +42,8 @@ def ensure_data_dir():
 def empty_positions_df():
     return pd.DataFrame(columns=POS_COLS)
 
-def empty_risk_df():
-    return pd.DataFrame(columns=SUG_RISK_COLS)
+def empty_suggest_df():
+    return pd.DataFrame(columns=SUG_COLS)
 
 def fred_series_for_horizon(years: float) -> str:
     y = max(1.0, min(100.0, float(years)))
@@ -71,28 +66,27 @@ def fetch_fred_yield_annual(code: str) -> float:
 def fetch_prices_monthly(tickers: List[str], years: int) -> pd.DataFrame:
     start = pd.Timestamp.today(tz="UTC") - pd.DateOffset(years=years, days=7)
     end = pd.Timestamp.today(tz="UTC")
+    syms = [str(t).upper().strip() for t in dict.fromkeys(tickers)]
     df = yf.download(
-        list(dict.fromkeys(tickers)),
-        start=start.date(),
-        end=end.date(),
-        interval="1mo",
-        auto_adjust=True,
-        progress=False
+        syms, start=start.date(), end=end.date(),
+        interval="1mo", auto_adjust=True, progress=False
     )["Close"]
     if isinstance(df, pd.Series):
         df = df.to_frame()
-    if isinstance(df.columns, pd.MultiIndex):
-        df.columns = [c[-1] if isinstance(c, tuple) else str(c) for c in df.columns]
-    else:
-        df.columns = [str(c) for c in df.columns]
     df = df.dropna(how="all").fillna(method="ffill")
+    # columns become single Index if single ticker
+    if isinstance(df.columns, pd.MultiIndex):
+        df.columns = [c[1] if isinstance(c, tuple) else c for c in df.columns]
     return df
 
 def monthly_returns(prices: pd.DataFrame) -> pd.DataFrame:
     return prices.pct_change().dropna()
 
-def annualize_mean(m):  return np.asarray(m, dtype=float) * 12.0
-def annualize_sigma(s): return np.asarray(s, dtype=float) * math.sqrt(12.0)
+def annualize_mean(m):
+    return np.asarray(m, dtype=float) * 12.0
+
+def annualize_sigma(s):
+    return np.asarray(s, dtype=float) * math.sqrt(12.0)
 
 def yahoo_search(query: str):
     if not query or len(query.strip()) == 0:
@@ -109,7 +103,7 @@ def yahoo_search(query: str):
             sym = q.get("symbol")
             name = q.get("shortname") or q.get("longname") or ""
             exch = q.get("exchDisp") or ""
-            if sym and isinstance(sym, str) and sym.isascii():
+            if sym and sym.isascii():
                 out.append({"symbol": sym, "name": name, "exchange": exch})
         if not out:
             out = [{"symbol": query.strip().upper(), "name": "typed symbol", "exchange": "n a"}]
@@ -126,19 +120,18 @@ def validate_tickers(symbols: List[str], years: int) -> List[str]:
 
 # -------------- aligned moments --------------
 def get_aligned_monthly_returns(symbols: List[str], years: int) -> pd.DataFrame:
-    uniq = [c for c in dict.fromkeys(symbols)]
-    if MARKET_TICKER not in uniq:
-        uniq = uniq + [MARKET_TICKER]
-    px = fetch_prices_monthly(uniq, years)
+    uniq = [c for c in dict.fromkeys(symbols) if c != MARKET_TICKER]
+    tickers = uniq + [MARKET_TICKER]
+    px = fetch_prices_monthly(tickers, years)
     rets = monthly_returns(px)
-    cols = [c for c in uniq if c in rets.columns]
+    cols = [c for c in uniq if c in rets.columns] + ([MARKET_TICKER] if MARKET_TICKER in rets.columns else [])
     R = rets[cols].dropna(how="any")
     return R.loc[:, ~R.columns.duplicated()]
 
 def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
     R = get_aligned_monthly_returns(symbols, years)
     if MARKET_TICKER not in R.columns or R.shape[0] < 3:
-        raise ValueError("Not enough aligned data")
+        raise ValueError("Could not align data with market or not enough rows.")
     rf_m = rf_ann / 12.0
 
     m = R[MARKET_TICKER]
@@ -154,15 +147,12 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
     var_m = max(var_m, 1e-6)
 
     betas: Dict[str, float] = {}
-    for s in [c for c in R.columns]:
-        ex_s = (R[s] - rf_m) if s in R.columns else None
-        if s == MARKET_TICKER:
-            betas[s] = 1.0
-        elif ex_s is not None:
-            betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
-
-    # include market in covariance so risk is measured correctly when VOO is held
-    asset_cols = list(R.columns)
+    for s in [c for c in R.columns if c != MARKET_TICKER]:
+        ex_s = R[s] - rf_m
+        betas[s] = float(np.cov(ex_s.values, ex_m.values, ddof=1)[0, 1] / var_m)
+    betas[MARKET_TICKER] = 1.0
+
+    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
     cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
     covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
 
@@ -177,6 +167,8 @@ def portfolio_stats(weights: Dict[str, float],
                     rf_ann: float,
                     erp_ann: float) -> Tuple[float, float, float]:
     tickers = list(weights.keys())
+    if len(tickers) == 0:
+        return 0.0, 0.0, 0.0
     w = np.array([weights[t] for t in tickers], dtype=float)
     gross = float(np.sum(np.abs(w)))
     if gross == 0:
@@ -188,7 +180,7 @@ def portfolio_stats(weights: Dict[str, float],
     sigma_p = math.sqrt(float(max(w_expo.T @ cov @ w_expo, 0.0)))
     return beta_p, er_p, sigma_p
 
-# -------------- CML helpers --------------
+# -------------- CML helpers + plot (percent axes) --------------
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
     if sigma_mkt <= 1e-12:
         return 0.0, 1.0, rf_ann
@@ -201,7 +193,7 @@ def efficient_same_return(mu_target: float, rf_ann: float, erp_ann: float, sigma
     a = (mu_target - rf_ann) / erp_ann
     return a, 1.0 - a, abs(a) * sigma_mkt
 
-def plot_cml(
+def plot_cml_percent(
     rf_ann, erp_ann, sigma_mkt,
     pt_sigma, pt_mu,
     same_sigma_sigma, same_sigma_mu,
@@ -221,36 +213,25 @@ def plot_cml(
     xs = np.linspace(0, xmax, 160)
     slope = erp_ann / max(sigma_mkt, 1e-12)
     cml = rf_ann + slope * xs
-    plt.plot(xs, cml, label="CML through VOO")
 
-    plt.scatter([0.0], [rf_ann], label="Risk free")
-    plt.scatter([sigma_mkt], [rf_ann + erp_ann], label="Market VOO")
-    plt.scatter([pt_sigma], [pt_mu], label="Your portfolio")
-    plt.scatter([same_sigma_sigma], [same_sigma_mu], label="Efficient same sigma")
-    plt.scatter([same_mu_sigma], [same_mu_mu], label="Efficient same return")
+    def pct(x): return 100.0 * np.asarray(x)
+
+    plt.plot(pct(xs), pct(cml), label="CML through VOO")
+
+    plt.scatter([0.0], [pct(rf_ann)], label="Risk free")
+    plt.scatter([pct(sigma_mkt)], [pct(rf_ann + erp_ann)], label="Market VOO")
+    plt.scatter([pct(pt_sigma)], [pct(pt_mu)], label="Your portfolio")
+    plt.scatter([pct(same_sigma_sigma)], [pct(same_sigma_mu)], label="Efficient same sigma")
+    plt.scatter([pct(same_mu_sigma)], [pct(same_mu_mu)], label="Efficient same return")
     if targ_sigma is not None and targ_mu is not None:
-        plt.scatter([targ_sigma], [targ_mu], label="Dataset suggestion")
-
-    # Guides + annotations (in percent)
-    plt.plot([pt_sigma, same_sigma_sigma], [pt_mu, same_sigma_mu],
-             linestyle="--", linewidth=1.2, alpha=0.7, color="gray")
-    d_ret = (same_sigma_mu - pt_mu) * 100.0
-    plt.annotate(f"Return gain at same sigma {d_ret:+.2f}%",
-                 xy=(same_sigma_sigma, same_sigma_mu),
-                 xytext=(same_sigma_sigma + 0.02 * xmax, same_sigma_mu),
-                 arrowprops=dict(arrowstyle="->", lw=1.0), fontsize=9, va="center")
-
-    plt.plot([pt_sigma, same_mu_sigma], [pt_mu, same_mu_mu],
-             linestyle="--", linewidth=1.2, alpha=0.7, color="gray")
-    d_sig = (same_mu_sigma - pt_sigma) * 100.0
-    plt.annotate(f"Risk change at same return {d_sig:+.2f}%",
-                 xy=(same_mu_sigma, same_mu_mu),
-                 xytext=(same_mu_sigma, same_mu_mu + 0.03),
-                 arrowprops=dict(arrowstyle="->", lw=1.0), fontsize=9, ha="center")
-
-    ax = plt.gca()
-    ax.yaxis.set_major_formatter(PercentFormatter(1.0))
-    ax.xaxis.set_major_formatter(PercentFormatter(1.0))
+        plt.scatter([pct(targ_sigma)], [pct(targ_mu)], label="Target suggestion")
+
+    # Guides (keep simple)
+    plt.plot([pct(pt_sigma), pct(same_sigma_sigma)], [pct(pt_mu), pct(same_sigma_mu)],
+             linestyle="--", linewidth=1.0, alpha=0.7, color="gray")
+    plt.plot([pct(pt_sigma), pct(same_mu_sigma)], [pct(pt_mu), pct(same_mu_mu)],
+             linestyle="--", linewidth=1.0, alpha=0.7, color="gray")
+
     plt.xlabel("Standard deviation (%)")
     plt.ylabel("Expected return (%)")
     plt.legend(loc="best")
@@ -262,209 +243,126 @@ def plot_cml(
     buf.seek(0)
     return Image.open(buf)
 
-# -------------- synthetic dataset (for predictor + risk buttons) --------------
-def synth_profile(seed: int) -> str:
-    rng = np.random.default_rng(seed)
-    risk = rng.choice(["cautious", "balanced", "moderate", "growth", "aggressive"])
-    horizon = rng.choice(["three years", "five years", "seven years", "ten years", "fifteen years"])
-    goal = rng.choice(["retirement savings", "first home", "education fund", "wealth building", "travel fund", "emergency buffer"])
-    return f"{risk} investor, {horizon} horizon, goal is {goal}."
-
-def build_synthetic_dataset(universe: List[str], years: int, rf_ann: float, erp_ann: float) -> pd.DataFrame:
-    symbols = list(sorted(set([s for s in universe if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
-    moms = estimate_all_moments_aligned(symbols, years, rf_ann)
-    covA, betas = moms["cov_ann"], moms["betas"]
-    rows, rng = [], np.random.default_rng(123)
-    for i in range(1000):
-        k = rng.integers(low=min(2, len(symbols)), high=min(8, len(symbols)) + 1)
-        picks = list(rng.choice(symbols, size=k, replace=False))
-        signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
-        raw = rng.dirichlet(np.ones(k))
-        gross = 1.0 + float(rng.gamma(2.0, 0.5))
-        w = gross * signs * raw
-        beta_p, er_p, sigma_p = portfolio_stats({picks[j]: w[j] for j in range(k)}, covA, betas, rf_ann, erp_ann)
+# -------------- dataset over *current* tickers --------------
+def dirichlet_mixture(n: int, k: int, allow_shorts: bool, rng: np.random.Generator) -> np.ndarray:
+    """Return n weight vectors (exposures) across k assets; sum |w| = 1."""
+    out = []
+    n1 = int(n * 0.6)  # diversified
+    n2 = n - n1        # concentrated
+    for _ in range(n1):
+        w = rng.dirichlet(np.ones(k))
+        if allow_shorts:
+            signs = rng.choice([-1.0, 1.0], size=k, p=[0.25, 0.75])
+            w = w * signs
+        out.append(w)
+    for _ in range(n2):
+        hot = rng.integers(0, k)
+        alpha = np.ones(k) * 0.3
+        alpha[hot] = 3.0
+        w = rng.dirichlet(alpha)
+        if allow_shorts:
+            signs = rng.choice([-1.0, 1.0], size=k, p=[0.35, 0.65])
+            w = w * signs
+        out.append(w)
+    W = np.vstack(out)
+    # normalize to exposure space (sum |w| = 1)
+    denom = np.sum(np.abs(W), axis=1, keepdims=True)
+    denom[denom == 0] = 1.0
+    return W / denom
+
+def build_fixed_universe_dataset(
+    symbols: List[str], years: int, rf_ann: float, erp_ann: float,
+    covA: pd.DataFrame, betas: Dict[str, float],
+    allow_shorts: bool, n_rows: int = 1000
+) -> pd.DataFrame:
+    rng = np.random.default_rng(12345)
+    k = len(symbols)
+    W = dirichlet_mixture(n_rows, k, allow_shorts, rng)
+
+    rows = []
+    for i in range(W.shape[0]):
+        w = W[i]
+        wmap = {symbols[j]: float(w[j]) for j in range(k)}
+        beta_p, er_p, sigma_p = portfolio_stats(wmap, covA, betas, rf_ann, erp_ann)
         rows.append({
             "id": i,
-            "profile_text": synth_profile(10_000 + i),
-            "tickers": ",".join(picks),
-            "weights": ",".join(f"{x:.4f}" for x in w),
+            "tickers": ",".join(symbols),
+            "weights": ",".join(f"{x:.6f}" for x in w),
             "beta_p": beta_p,
             "er_p": er_p,
             "sigma_p": sigma_p
         })
     return pd.DataFrame(rows)
 
-def save_synth_csv(df: pd.DataFrame, path: str = DATASET_PATH):
+def save_dataset_csv(df: pd.DataFrame, path: str = DATASET_PATH):
     os.makedirs(os.path.dirname(path), exist_ok=True)
     df.to_csv(path, index=False)
 
 def _row_to_exposures(row: pd.Series, universe: List[str]) -> Optional[np.ndarray]:
     try:
-        ts = [t.strip() for t in str(row["tickers"]).split(",")]
+        ts = [t.strip().upper() for t in str(row["tickers"]).split(",")]
         ws = [float(x) for x in str(row["weights"]).split(",")]
         wmap = {t: ws[i] for i, t in enumerate(ts) if i < len(ws)}
-        w = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
-        gross = float(np.sum(np.abs(w)))
-        if gross <= 1e-12:
+        x = np.array([wmap.get(t, 0.0) for t in universe], dtype=float)
+        g = float(np.sum(np.abs(x)))
+        if g <= 1e-12:
             return None
-        return w / gross
+        return x / g
     except Exception:
         return None
 
-def fit_surrogate_from_csv(csv_path: str, universe: List[str]):
-    try:
-        df = pd.read_csv(csv_path)
-    except Exception:
-        return None, None, 0
-    X_list, Y_list = [], []
+def pick_low_med_high(csv_path: str, universe: List[str]):
+    df = pd.read_csv(csv_path)
+    rows = []
     for _, r in df.iterrows():
         x = _row_to_exposures(r, universe)
         if x is None:
             continue
-        y = np.array([float(r["er_p"]), float(r["sigma_p"]), float(r["beta_p"])], dtype=float)
-        X_list.append(x); Y_list.append(y)
-    if not X_list:
-        return None, None, 0
-    X = np.vstack(X_list); Y = np.vstack(Y_list)
-    scaler = StandardScaler().fit(X)
-    Xn = scaler.transform(X)
-    k = min(25, len(Xn))
-    knn = KNeighborsRegressor(n_neighbors=k, weights="distance")
-    knn.fit(Xn, Y)
-    return scaler, knn, len(Xn)
-
-def predict_from_surrogate(amounts_map: Dict[str, float], universe: List[str],
-                           scaler: StandardScaler, knn: KNeighborsRegressor):
-    gross = sum(abs(v) for v in amounts_map.values())
-    if gross <= 1e-12:
-        return None
-    w = np.array([amounts_map.get(t, 0.0) for t in universe], dtype=float) / gross
-    yhat = knn.predict(scaler.transform([w]))[0]
-    er_hat, sigma_hat, beta_hat = float(yhat[0]), float(yhat[1]), float(yhat[2])
-    return er_hat, sigma_hat, beta_hat
-
-# ---- dataset risk buttons helpers (purely CSV-based) ----
-def pick_row_by_risk(df: pd.DataFrame, level: str) -> Optional[pd.Series]:
-    df = df.dropna(subset=["sigma_p"])
-    if df.empty:
+        rows.append((x, float(r["er_p"]), float(r["sigma_p"]), float(r["beta_p"])))
+    if not rows:
         return None
-    if level == "low":
-        return df.loc[df["sigma_p"].idxmin()]
-    if level == "high":
-        return df.loc[df["sigma_p"].idxmax()]
-    # medium: closest to median sigma
-    med = float(df["sigma_p"].median())
-    idx = (df["sigma_p"] - med).abs().idxmin()
-    return df.loc[idx]
-
-def row_to_suggestion(row: pd.Series, universe: List[str]) -> Optional[Dict]:
-    x = _row_to_exposures(row, universe)
-    if x is None:
-        return None
-    wmap = {universe[i]: float(x[i]) for i in range(len(universe)) if abs(float(x[i])) > 1e-4}
-    # sort top exposures
-    wmap = dict(sorted(wmap.items(), key=lambda kv: -abs(kv[1]))[:12])
-    return {
-        "weights": wmap,
-        "er": float(row["er_p"]),
-        "sigma": float(row["sigma_p"]),
-        "beta": float(row["beta_p"]),
-    }
-
-def suggest_by_risk(level: str, state: dict):
-    # State must come from a previous "Compute"
-    if not isinstance(state, dict) or not state.get("csv_path") or not os.path.exists(state["csv_path"]):
-        return gr.update(), empty_risk_df(), "Run analysis first to build the dataset."
-
-    try:
-        df = pd.read_csv(state["csv_path"])
-    except Exception:
-        return gr.update(), empty_risk_df(), "Could not read dataset."
-
-    row = pick_row_by_risk(df, {"low":"low","med":"med","high":"high"}[level])
-    if row is None:
-        return gr.update(), empty_risk_df(), "Dataset is empty."
-
-    cand = row_to_suggestion(row, UNIVERSE)
-    if cand is None:
-        return gr.update(), empty_risk_df(), "No suggestion available."
-
-    # Build table in percents
-    rows = [{"ticker": k, "suggested_weight_%": v * 100.0} for k, v in cand["weights"].items()]
-    risk_table = pd.DataFrame(rows, columns=SUG_RISK_COLS)
-
-    # Overlay the dataset suggestion on the existing CML
-    img = plot_cml(
-        state["rf_ann"], state["erp_ann"], state["sigma_mkt"],
-        state["pt_sigma"], state["pt_mu"],
-        state["same_sigma_sigma"], state["same_sigma_mu"],
-        state["same_mu_sigma"], state["same_mu_mu"],
-        targ_sigma=cand["sigma"], targ_mu=cand["er"]
-    )
-
-    msg = (
-        f"**Dataset suggestion ({'Lowest' if level=='low' else 'Medium' if level=='med' else 'Highest'} risk)**  \n"
-        f"- Predicted expected return: {fmt_pct(cand['er'])}  \n"
-        f"- Predicted sigma: {fmt_pct(cand['sigma'])}  \n"
-        f"- Predicted beta: {cand['beta']:.2f}"
-    )
-    return img, risk_table, msg
+    rows_sorted = sorted(rows, key=lambda t: t[2])  # by sigma
+    lo = rows_sorted[0]
+    hi = rows_sorted[-1]
+    med = rows_sorted[len(rows_sorted)//2]
+    return {"low": lo, "medium": med, "high": hi}
 
-# -------------- summary --------------
+# -------------- summary builder --------------
 def fmt_pct(x: float) -> str:
     return f"{x*100:.2f}%"
 
-def humanize_synth(er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta):
-    close_mu = abs(dmu) <= 0.005
-    close_sig = abs(dsig) <= 0.005
-    close_beta = abs(dbeta) <= 0.05
-    parts = []
-    parts.append(f"- Predicted annual return {fmt_pct(er_hat)} , difference {fmt_pct(dmu)}")
-    parts.append(f"- Predicted annual volatility {fmt_pct(sigma_hat)} , difference {fmt_pct(dsig)}")
-    parts.append(f"- Predicted beta {beta_hat:.2f} , difference {dbeta:+.02f}")
-    verdict = ("The synthetic model matches the historical calculation closely. "
-               "You can trust these quick predictions for similar mixes."
-               if (close_mu and close_sig and close_beta)
-               else "The synthetic model is not very close here. Rely more on the historical calculation for this mix.")
-    return "\n".join(parts + ["", f"**Verdict** {verdict}"])
-
 def build_summary_md(lookback, horizon, rf, rf_code, erp, sigma_mkt,
                      beta_p, er_p, sigma_p,
                      a_sigma, b_sigma, mu_eff_sigma,
                      a_mu, b_mu, sigma_eff_mu,
-                     synth=None, synth_nrows: int = 0) -> str:
+                     ds_info: str) -> str:
     lines = []
     lines.append("### Inputs")
-    lines.append(f"- Lookback years {lookback}")
-    lines.append(f"- Horizon years {int(round(horizon))}")
-    lines.append(f"- Risk free {fmt_pct(rf)} from {rf_code}")
-    lines.append(f"- Market ERP {fmt_pct(erp)}")
-    lines.append(f"- Market sigma {fmt_pct(sigma_mkt)}")
+    lines.append(f"- Lookback years: {lookback}")
+    lines.append(f"- Horizon years: {int(round(horizon))}")
+    lines.append(f"- Risk free: {fmt_pct(rf)} from {rf_code}")
+    lines.append(f"- Market ERP: {fmt_pct(erp)}")
+    lines.append(f"- Market sigma: {fmt_pct(sigma_mkt)}")
     lines.append("")
     lines.append("### Your portfolio")
-    lines.append(f"- Beta {beta_p:.2f}")
-    lines.append(f"- Sigma {fmt_pct(sigma_p)}")
-    lines.append(f"- Expected return {fmt_pct(er_p)}")
-    if synth is not None:
-        er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta = synth
-        lines.append("")
-        lines.append("### Synthetic prediction from data/investor_profiles.csv")
-        lines.append(f"- Samples used {synth_nrows}")
-        lines.append(humanize_synth(er_hat, sigma_hat, beta_hat, dmu, dsig, dbeta))
+    lines.append(f"- Beta: {beta_p:.2f}")
+    lines.append(f"- Sigma: {fmt_pct(sigma_p)}")
+    lines.append(f"- Expected return: {fmt_pct(er_p)}")
     lines.append("")
     lines.append("### Efficient alternatives on CML")
-    lines.append("Efficient same sigma")
-    lines.append(f"- Market weight {a_sigma:.2f} , Bills weight {b_sigma:.2f}")
-    lines.append(f"- Expected return {fmt_pct(mu_eff_sigma)}")
-    lines.append("Efficient same return")
-    lines.append(f"- Market weight {a_mu:.2f} , Bills weight {b_mu:.2f}")
-    lines.append(f"- Sigma {fmt_pct(sigma_eff_mu)}")
+    lines.append(f"- Same sigma ⇒ Market {a_sigma:.2f} , Bills {b_sigma:.2f} , ER {fmt_pct(mu_eff_sigma)}")
+    lines.append(f"- Same return ⇒ Market {a_mu:.2f} , Bills {b_mu:.2f} , Sigma {fmt_pct(sigma_eff_mu)}")
+    lines.append("")
+    lines.append("### Dataset for risk suggestions")
+    lines.append(ds_info)
     return "\n".join(lines)
 
-# -------------- app state on launch --------------
-ensure_data_dir()
-UNIVERSE = [MARKET_TICKER, "QQQ", "XLK", "XLP", "XLE", "VNQ", "IEF", "HYG", "GLD", "EEM"]
-HORIZON_YEARS = 5
+# -------------- globals to carry session state --------------
+LAST_MOMS = None
+LAST_BASE = None
+LAST_UNIVERSE = []
+LAST_DATASET_PATH = None
+HORIZON_YEARS = 5.0
 RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
 RF_ANN = fetch_fred_yield_annual(RF_CODE)
 
@@ -515,79 +413,62 @@ def set_horizon(years: float):
     HORIZON_YEARS = y
     RF_CODE = code
     RF_ANN = rf
-    return f"Risk free series {code}. Latest annual rate {rf:.2%}. Dataset will use this rate on compute."
+    return f"Risk free {fmt_pct(rf)} from {code}. Will be used on Compute."
 
-def compute(years_lookback: int, table: pd.DataFrame, use_synth: bool):
+def compute(years_lookback: int, table: pd.DataFrame):
     df = table.dropna()
     df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
     df["amount_usd"] = pd.to_numeric(df["amount_usd"], errors="coerce").fillna(0.0)
 
     symbols = [t for t in df["ticker"].tolist() if t]
     if len(symbols) == 0:
-        return None, "Add at least one ticker", "Universe empty", empty_positions_df(), None, {}
+        return None, "Add at least one ticker", "Universe empty", empty_positions_df(), empty_suggest_df(), "", None
 
     symbols = validate_tickers(symbols, years_lookback)
     if len(symbols) == 0:
-        return None, "Could not validate any tickers", "Universe invalid", empty_positions_df(), None, {}
-
-    global UNIVERSE
-    UNIVERSE = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+        return None, "Could not validate any tickers", "Universe invalid", empty_positions_df(), empty_suggest_df(), "", None
 
     df = df[df["ticker"].isin(symbols)].copy()
     amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
+    allow_shorts = any(v < 0 for v in amounts.values())
     rf_ann = RF_ANN
 
+    # moments
     moms = estimate_all_moments_aligned(symbols, years_lookback, rf_ann)
     betas, covA, erp_ann, sigma_mkt = moms["betas"], moms["cov_ann"], moms["erp_ann"], moms["sigma_m_ann"]
 
     gross = sum(abs(v) for v in amounts.values())
     if gross == 0:
-        return None, "All amounts are zero", "Universe ok", empty_positions_df(), None, {}
+        return None, "All amounts are zero", "Universe ok", empty_positions_df(), empty_suggest_df(), "", None
     weights = {k: v / gross for k, v in amounts.items()}
-
     beta_p, er_p, sigma_p = portfolio_stats(weights, covA, betas, rf_ann, erp_ann)
 
     a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
     a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, rf_ann, erp_ann, sigma_mkt)
 
-    # ensure synthetic dataset exists once (for predictor + risk buttons)
-    if not os.path.exists(DATASET_PATH):
-        synth_df = build_synthetic_dataset(
-            universe=list(sorted(set(symbols + [MARKET_TICKER]))),
-            years=DEFAULT_LOOKBACK_YEARS,
-            rf_ann=rf_ann,
-            erp_ann=erp_ann
-        )
-        save_synth_csv(synth_df)
-    csv_path = DATASET_PATH if os.path.exists(DATASET_PATH) else None
-
-    scaler, knn, nrows = None, None, 0
-    synth_tuple = None
-    if use_synth and csv_path:
-        scaler, knn, nrows = fit_surrogate_from_csv(csv_path, UNIVERSE)
-        if scaler is not None and knn is not None:
-            pred = predict_from_surrogate(amounts, UNIVERSE, scaler, knn)
-            if pred is not None:
-                er_hat, sigma_hat, beta_hat = pred
-                synth_tuple = (
-                    er_hat, sigma_hat, beta_hat,
-                    er_hat - er_p, sigma_hat - sigma_p, beta_hat - beta_p
-                )
-
-    img = plot_cml(
+    # dataset strictly over *these* symbols
+    ensure_data_dir()
+    ds = build_fixed_universe_dataset(
+        symbols=symbols, years=years_lookback, rf_ann=rf_ann, erp_ann=erp_ann,
+        covA=covA.loc[symbols, symbols], betas=betas, allow_shorts=allow_shorts, n_rows=1000
+    )
+    save_dataset_csv(ds, DATASET_PATH)
+    ds_info = f"- Built {len(ds)} simulated mixes over current tickers ({'shorts allowed' if allow_shorts else 'long-only'})."
+
+    # plot + summary
+    img = plot_cml_percent(
         rf_ann, erp_ann, sigma_mkt,
         sigma_p, er_p,
         sigma_p, mu_eff_sigma,
         sigma_eff_mu, er_p,
         targ_sigma=None, targ_mu=None
     )
-
     info = build_summary_md(
         years_lookback, HORIZON_YEARS, rf_ann, RF_CODE, erp_ann, sigma_mkt,
         beta_p, er_p, sigma_p,
         a_sigma, b_sigma, mu_eff_sigma,
         a_mu, b_mu, sigma_eff_mu,
-        synth=synth_tuple, synth_nrows=nrows
+        ds_info=ds_info
     )
 
     rows = []
@@ -596,27 +477,72 @@ def compute(years_lookback: int, table: pd.DataFrame, use_synth: bool):
         rows.append({
             "ticker": t,
             "amount_usd": amounts.get(t, 0.0),
-            "weight_%": weights.get(t, 0.0) * 100.0,
+            "weight_exposure": weights.get(t, 0.0),
             "beta": beta_val,
         })
     pos_table = pd.DataFrame(rows, columns=POS_COLS)
+    pos_table["weight_exposure"] = pos_table["weight_exposure"].astype(float)
+
+    uni_msg = f"Universe set to {', '.join(symbols)}"
+    # store globals for Suggest buttons
+    global LAST_MOMS, LAST_BASE, LAST_UNIVERSE, LAST_DATASET_PATH
+    LAST_MOMS = {"betas": betas, "covA": covA, "erp_ann": erp_ann, "sigma_mkt": sigma_mkt}
+    LAST_BASE = {"rf_ann": rf_ann, "er_p": er_p, "sigma_p": sigma_p}
+    LAST_UNIVERSE = list(symbols)
+    LAST_DATASET_PATH = DATASET_PATH
 
-    # Pack state for risk buttons
-    state = {
-        "csv_path": csv_path,
-        "rf_ann": rf_ann,
-        "erp_ann": erp_ann,
-        "sigma_mkt": sigma_mkt,
-        "pt_sigma": sigma_p,
-        "pt_mu": er_p,
-        "same_sigma_sigma": sigma_p,
-        "same_sigma_mu": mu_eff_sigma,
-        "same_mu_sigma": sigma_eff_mu,
-        "same_mu_mu": er_p,
-    }
-
-    uni_msg = f"Universe set to {', '.join(UNIVERSE)}"
-    return img, info, uni_msg, pos_table, csv_path, state
+    return img, info, uni_msg, pos_table, empty_suggest_df(), ds_info, DATASET_PATH
+
+def _overlay_plot_with_suggestion(sigma_s, er_s):
+    if not LAST_MOMS or not LAST_BASE:
+        return None
+    rf_ann = LAST_BASE["rf_ann"]
+    erp_ann = LAST_MOMS["erp_ann"]
+    sigma_mkt = LAST_MOMS["sigma_mkt"]
+    sigma_p = LAST_BASE["sigma_p"]
+    er_p = LAST_BASE["er_p"]
+    a_sigma, b_sigma, mu_eff_sigma = efficient_same_sigma(sigma_p, rf_ann, erp_ann, sigma_mkt)
+    a_mu, b_mu, sigma_eff_mu = efficient_same_return(er_p, rf_ann, erp_ann, sigma_mkt)
+    return plot_cml_percent(
+        rf_ann, erp_ann, sigma_mkt,
+        sigma_p, er_p,
+        sigma_p, mu_eff_sigma,
+        sigma_eff_mu, er_p,
+        targ_sigma=sigma_s, targ_mu=er_s
+    )
+
+def suggest_level(level: str):
+    if not LAST_DATASET_PATH or not os.path.exists(LAST_DATASET_PATH) or not LAST_UNIVERSE:
+        return empty_suggest_df(), "Run Compute first.", None
+    picks = pick_low_med_high(LAST_DATASET_PATH, LAST_UNIVERSE)
+    if picks is None or level not in picks:
+        return empty_suggest_df(), "No suggestion available.", None
+    x, er_p, sig_p, beta_p = picks[level]
+    # build table in percent
+    rows = [{"ticker": LAST_UNIVERSE[i], "suggested_weight_pct": float(x[i]) * 100.0} for i in range(len(LAST_UNIVERSE))]
+    df = pd.DataFrame(rows, columns=SUG_COLS)
+    msg = f"{level.capitalize()} risk → ER {fmt_pct(er_p)}, Sigma {fmt_pct(sig_p)}, Beta {beta_p:.2f}"
+    img = _overlay_plot_with_suggestion(sig_p, er_p)
+    return df, msg, img
+
+def apply_suggestion_to_amounts(level: str, table: pd.DataFrame):
+    if table is None or len(table) == 0:
+        return table
+    df_sug, _, _ = suggest_level(level)
+    if df_sug is None or len(df_sug) == 0:
+        return table
+    # compute gross dollars (use total |amounts|; if zero, default to 10,000)
+    t = table.copy()
+    t["ticker"] = t["ticker"].astype(str).str.upper().str.strip()
+    t["amount_usd"] = pd.to_numeric(t["amount_usd"], errors="coerce").fillna(0.0)
+    gross = float(np.sum(np.abs(t["amount_usd"].values)))
+    if gross <= 1e-9:
+        gross = 10000.0
+    w = {r["ticker"]: float(r["suggested_weight_pct"]) / 100.0 for _, r in df_sug.iterrows()}
+    # map to amounts using current ticker order; missing → 0
+    new_amounts = [gross * w.get(sym.upper(), 0.0) for sym in t["ticker"].tolist()]
+    t["amount_usd"] = new_amounts
+    return t
 
 # -------------- UI --------------
 ensure_data_dir()
@@ -625,19 +551,18 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     gr.Markdown(
         "## Efficient Portfolio Advisor\n"
         "Search symbols, enter dollar amounts, set your horizon. "
-        "Prices come from Yahoo Finance. Risk free comes from FRED.\n\n"
-        "**New:** Dataset-only risk suggestions (Low / Medium / High) from the 1,000-row synthetic CSV."
+        "Prices: Yahoo Finance. Risk free: FRED. "
+        "Suggestions (Low/Medium/High) come **only** from the 1,000-portfolio dataset built over your tickers."
     )
 
-    app_state = gr.State({})
-
     with gr.Row():
         with gr.Column(scale=1):
             q = gr.Textbox(label="Search symbol")
             search_note = gr.Markdown()
             matches = gr.Dropdown(choices=[], label="Matches")
-            search_btn = gr.Button("Search")
-            add_btn = gr.Button("Add selected to portfolio")
+            with gr.Row():
+                search_btn = gr.Button("Search")
+                add_btn = gr.Button("Add selected to portfolio")
 
             gr.Markdown("### Portfolio positions (type dollar amounts, negatives allowed for shorts)")
             table = gr.Dataframe(
@@ -650,19 +575,24 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
             horizon = gr.Number(label="Horizon in years (1–100)", value=5, precision=0)
             lookback = gr.Slider(1, 10, value=DEFAULT_LOOKBACK_YEARS, step=1, label="Lookback years for beta & sigma")
 
-            use_synth = gr.Checkbox(label="Use synthetic predictor (fast check)", value=True)
-
-            run_btn = gr.Button("Compute and suggest")
+            with gr.Row():
+                run_btn = gr.Button("Compute (build dataset)", variant="primary")
 
-            gr.Markdown("### Dataset-based risk suggestions")
+            gr.Markdown("### Risk tolerance suggestions (dataset-based only)")
+            with gr.Row():
+                btn_low = gr.Button("Low risk")
+                btn_med = gr.Button("Medium risk")
+                btn_high = gr.Button("High risk")
             with gr.Row():
-                btn_low = gr.Button("Lowest risk (dataset)")
-                btn_med = gr.Button("Medium risk (dataset)")
-                btn_high = gr.Button("Highest risk (dataset)")
+                apply_low = gr.Button("Apply Low → $")
+                apply_med = gr.Button("Apply Medium → $")
+                apply_high = gr.Button("Apply High → $")
+
         with gr.Column(scale=1):
             plot = gr.Image(label="Capital Market Line", type="pil")
             summary = gr.Markdown(label="Summary")
             universe_msg = gr.Textbox(label="Universe status", interactive=False)
+            dataset_info = gr.Markdown(label="Dataset info", value="")
             positions = gr.Dataframe(
                 label="Computed positions",
                 headers=POS_COLS,
@@ -671,17 +601,18 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
                 value=empty_positions_df(),
                 interactive=False
             )
-            risk_table = gr.Dataframe(
-                label="Suggested portfolio from dataset",
-                headers=SUG_RISK_COLS,
+            suggestions = gr.Dataframe(
+                label="Suggested weights (percent of exposure)",
+                headers=SUG_COLS,
                 datatype=["str", "number"],
-                col_count=(len(SUG_RISK_COLS), "fixed"),
-                value=empty_risk_df(),
+                col_count=(len(SUG_COLS), "fixed"),
+                value=empty_suggest_df(),
                 interactive=False
             )
-            risk_msg = gr.Markdown(label="Suggestion metrics")
-            dl = gr.File(label="Session CSV path (synthetic predictor data)", value=None, visible=True)
+            sugg_msg = gr.Markdown("")
+            dl = gr.File(label="Session dataset CSV", value=None, visible=True)
 
+    # wiring
     def do_search(query):
         note, options = search_tickers_cb(query)
         return note, gr.update(choices=options)
@@ -693,14 +624,24 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
 
     run_btn.click(
         fn=compute,
-        inputs=[lookback, table, use_synth],
-        outputs=[plot, summary, universe_msg, positions, dl, app_state]
+        inputs=[lookback, table],
+        outputs=[plot, summary, universe_msg, positions, suggestions, dataset_info, dl]
     )
 
-    # Risk buttons (purely dataset-driven)
-    btn_low.click(fn=partial(suggest_by_risk, "low"), inputs=[app_state], outputs=[plot, risk_table, risk_msg])
-    btn_med.click(fn=partial(suggest_by_risk, "med"), inputs=[app_state], outputs=[plot, risk_table, risk_msg])
-    btn_high.click(fn=partial(suggest_by_risk, "high"), inputs=[app_state], outputs=[plot, risk_table, risk_msg])
+    # suggest buttons
+    def wrap_suggest(level):
+        df, msg, img = suggest_level(level)
+        img_out = img if img is not None else gr.update()
+        return df, msg, img_out
+
+    btn_low.click(lambda: wrap_suggest("low"), outputs=[suggestions, sugg_msg, plot])
+    btn_med.click(lambda: wrap_suggest("medium"), outputs=[suggestions, sugg_msg, plot])
+    btn_high.click(lambda: wrap_suggest("high"), outputs=[suggestions, sugg_msg, plot])
+
+    # apply buttons (only updates the table; user can hit Compute again)
+    apply_low.click(lambda tb: apply_suggestion_to_amounts("low", tb), inputs=table, outputs=table)
+    apply_med.click(lambda tb: apply_suggestion_to_amounts("medium", tb), inputs=table, outputs=table)
+    apply_high.click(lambda tb: apply_suggestion_to_amounts("high", tb), inputs=table, outputs=table)
 
 if __name__ == "__main__":
     demo.launch()