Update app.py

app.py

@@ -12,7 +12,7 @@ import requests
 import yfinance as yf
 import gradio as gr
 
-# ---- runtime niceties (avoid MPL/Cache warnings in containers) ----
+# ---- runtime niceties ----
 os.environ.setdefault("MPLCONFIGDIR", os.getenv("MPLCONFIGDIR", "/home/user/.config/matplotlib"))
 os.makedirs(os.environ["MPLCONFIGDIR"], exist_ok=True)
 for d in [
@@ -123,7 +123,7 @@ def validate_tickers(symbols: List[str], years: int) -> List[str]:
     base = [s for s in dict.fromkeys([t.upper().strip() for t in symbols]) if s]
     px = fetch_prices_monthly(base + [MARKET_TICKER], years)
     ok = [s for s in base if s in px.columns]
-    # we require the market proxy to compute betas/ERP
+    # require market proxy to compute CAPM
     if MARKET_TICKER not in px.columns:
         return []
     return ok
@@ -163,9 +163,10 @@ def estimate_all_moments_aligned(symbols: List[str], years: int, rf_ann: float):
         betas[s] = cov_sm / var_m
     betas[MARKET_TICKER] = 1.0
 
-    asset_cols = [c for c in R.columns if c != MARKET_TICKER]
-    cov_m = np.cov(R[asset_cols].values.T, ddof=1) if asset_cols else np.zeros((0, 0))
-    covA = pd.DataFrame(cov_m * 12.0, index=asset_cols, columns=asset_cols)
+    # >>> FIX: include MARKET_TICKER in covariance so σ for portfolios holding VOO is correct
+    asset_cols_all = list(R.columns)  # includes market
+    cov_m_all = np.cov(R[asset_cols_all].values.T, ddof=1) if asset_cols_all else np.zeros((0, 0))
+    covA = pd.DataFrame(cov_m_all * 12.0, index=asset_cols_all, columns=asset_cols_all)
 
     return {"betas": betas, "cov_ann": covA, "erp_ann": erp_ann, "sigma_m_ann": sigma_m_ann}
 
@@ -190,7 +191,6 @@ def portfolio_stats(weights: Dict[str, float],
     return beta_p, mu_capm, sigma_hist
 
 def efficient_same_sigma(sigma_target: float, rf_ann: float, erp_ann: float, sigma_mkt: float):
-    # weights on (Market, Bills) that achieve same sigma as target, on the CML
     if sigma_mkt <= 1e-12:
         return 0.0, 1.0, rf_ann
     a = sigma_target / sigma_mkt
@@ -222,16 +222,16 @@ def plot_cml(rf_ann, erp_ann, sigma_mkt,
     plt.scatter([_pct(0)], [_pct(rf_ann)], label="Risk-free")
     plt.scatter([_pct(sigma_mkt)], [_pct(rf_ann + erp_ann)], label="Market")
 
-    # Your CAPM point: y clamped to CML at your σ_hist (display rule)
+    # Your CAPM point (y is clamped under CML at your σ for display)
     y_cml_at_sigma_p = rf_ann + slope * max(0.0, float(sigma_hist_p))
     y_you = min(float(mu_capm_p), y_cml_at_sigma_p)
     plt.scatter([_pct(sigma_hist_p)], [_pct(y_you)], label="Your CAPM point")
 
-    # Efficient points (on the CML by construction)
+    # Efficient points (on CML)
     plt.scatter([_pct(sigma_hist_p)], [_pct(same_sigma_mu)], marker="^", label="Efficient (same σ)")
     plt.scatter([_pct(same_mu_sigma)], [_pct(mu_capm_p)], marker="^", label="Efficient (same E[r])")
 
-    # Selected suggestion (clamped to CML for display)
+    # Selected suggestion
     if sugg_sigma_hist is not None and sugg_mu_capm is not None:
         y_cml_at_sugg = rf_ann + slope * max(0.0, float(sugg_sigma_hist))
         y_sugg = min(float(sugg_mu_capm), y_cml_at_sugg)
@@ -257,13 +257,10 @@ def build_synthetic_dataset(universe_user: List[str],
                             sigma_mkt: float,
                             n_rows: int = SYNTH_ROWS) -> pd.DataFrame:
     """
-    Generate long-only mixes **only from the user's tickers** (no VOO injected),
-    but we still use VOO internally for betas/ERP and the CML geometry.
+    Generate long-only mixes **from exactly the user's tickers** (VOO included only if the user holds it).
     """
     rng = np.random.default_rng(12345)
-    assets = [t for t in universe_user if t != MARKET_TICKER]
-    if not assets:
-        assets = universe_user[:]  # could be empty; handled below
+    assets = list(universe_user)  # <-- allow VOO if present
     if len(assets) == 0:
         return pd.DataFrame(columns=["tickers", "weights", "beta", "mu_capm", "sigma_hist"])
 
@@ -310,18 +307,12 @@ def rerank_and_pick_one(df_band: pd.DataFrame,
                         universe: List[str],
                         desired_band: str,
                         alpha: float = 0.6) -> pd.Series:
-    """
-    Re-rank with embeddings + exposure similarity + simple MMR,
-    then return **one** best pick (row).
-    """
     if df_band.empty:
         return pd.Series(dtype=object)
 
-    # exposure target = equal-weight over the user's universe
     exp_target = np.ones(len(universe))
     exp_target = exp_target / np.sum(exp_target)
 
-    # embeddings
     embs_ok = True
     try:
         from sentence_transformers import SentenceTransformer
@@ -332,19 +323,18 @@ def rerank_and_pick_one(df_band: pd.DataFrame,
             "high": "high risk growth aggressive portfolio higher expected return",
         }
         prompt = prompt_map.get(desired_band.lower(), prompt_map["medium"])
-        q = model.encode([prompt])  # (1, d)
+        q = model.encode([prompt])
     except Exception:
         embs_ok = False
         q = None
 
-    # score each candidate
-    scores = []
-    X_exp = np.stack([_exposure_vec(r, universe) for _, r in df_band.iterrows()], axis=0)
-    # cosine exposure similarity to target
+    # exposure similarity
     def _cos(a, b):
         an = np.linalg.norm(a) + 1e-12
         bn = np.linalg.norm(b) + 1e-12
         return float(np.dot(a, b) / (an * bn))
+
+    X_exp = np.stack([_exposure_vec(r, universe) for _, r in df_band.iterrows()], axis=0)
     exp_sims = np.array([_cos(x, exp_target) for x in X_exp])
 
     if embs_ok:
@@ -354,7 +344,7 @@ def rerank_and_pick_one(df_band: pd.DataFrame,
                 f"portfolio with tickers {r['tickers']} having beta {float(r['beta']):.2f}, "
                 f"expected return {float(r['mu_capm']):.3f}, sigma {float(r['sigma_hist']):.3f}"
             )
-        C = model.encode(cand_texts)  # (n, d)
+        C = model.encode(cand_texts)
         qv = q.reshape(-1)
         coss = (C @ qv) / (np.linalg.norm(C, axis=1) * (np.linalg.norm(qv) + 1e-12))
         coss = np.nan_to_num(coss, nan=0.0)
@@ -362,8 +352,6 @@ def rerank_and_pick_one(df_band: pd.DataFrame,
         coss = np.zeros(len(df_band))
 
     base = alpha * exp_sims + (1 - alpha) * coss
-
-    # simple MMR (λ = 0.7) for diversity; since we want top1, this is just argmax
     order = np.argsort(-base)
     best_idx = int(order[0])
     return df_band.iloc[best_idx]
@@ -375,7 +363,6 @@ def suggest_one_per_band(synth: pd.DataFrame, sigma_mkt: float, universe_user: L
         pick_pool = synth[(synth["sigma_hist"] >= lo) & (synth["sigma_hist"] <= hi)].copy()
         if pick_pool.empty:
             pick_pool = synth.copy()
-        # sort by CAPM E[r] first to bias pool, then rerank+MMR and return **one**
         pick_pool = pick_pool.sort_values("mu_capm", ascending=False).head(50).reset_index(drop=True)
         chosen = rerank_and_pick_one(pick_pool, universe_user, band)
         out[band.lower()] = chosen
@@ -479,7 +466,7 @@ def compute(
                "", "", "", None, None, None, None, None, None, None
 
     global UNIVERSE
-    UNIVERSE = list(sorted(set([s for s in symbols if s != MARKET_TICKER] + [MARKET_TICKER])))[:MAX_TICKERS]
+    UNIVERSE = list(sorted(set(symbols)))[:MAX_TICKERS]  # keep market if present
 
     df = df[df["ticker"].isin(symbols)].copy()
     amounts = {r["ticker"]: float(r["amount_usd"]) for _, r in df.iterrows()}
@@ -503,18 +490,17 @@ def compute(
     a_sigma, b_sigma, mu_eff_same_sigma = efficient_same_sigma(sigma_hist, rf_ann, erp_ann, sigma_mkt)
     a_mu, b_mu, sigma_eff_same_mu = efficient_same_return(mu_capm, rf_ann, erp_ann, sigma_mkt)
 
-    # Synthetic dataset & suggestions (ONLY user's tickers; no forced VOO)
-    user_universe_only = [t for t in symbols if t != MARKET_TICKER]  # suggestions must use same tickers as user entered
-    synth = build_synthetic_dataset(user_universe_only, covA, betas, rf_ann, erp_ann, sigma_mkt, n_rows=SYNTH_ROWS)
+    # Synthetic dataset & suggestions — exactly the user's tickers
+    user_universe = list(symbols)
+    synth = build_synthetic_dataset(user_universe, covA, betas, rf_ann, erp_ann, sigma_mkt, n_rows=SYNTH_ROWS)
     csv_path = os.path.join(DATA_DIR, f"investor_profiles_{int(time.time())}.csv")
     try:
         synth.to_csv(csv_path, index=False)
     except Exception:
         csv_path = None
 
-    picks = suggest_one_per_band(synth, sigma_mkt, user_universe_only)
+    picks = suggest_one_per_band(synth, sigma_mkt, user_universe)
 
-    # Build visible summaries
     def _fmt(row: pd.Series) -> str:
         if row is None or row.empty:
             return "No pick available."
@@ -524,7 +510,6 @@ def compute(
     txt_med   = _fmt(picks.get("medium", pd.Series(dtype=object)))
     txt_high  = _fmt(picks.get("high", pd.Series(dtype=object)))
 
-    # Choose which pick to display on the plot now
     chosen_band = (pick_band_to_show or "Medium").strip().lower()
     chosen = picks.get(chosen_band, pd.Series(dtype=object))
     if chosen is None or chosen.empty:
@@ -536,7 +521,6 @@ def compute(
         chosen_mu = float(chosen["mu_capm"])
         sugg_table = _holdings_table_from_row(chosen, budget=gross)
 
-    # positions table
     pos_table = pd.DataFrame(
         [{
             "ticker": t,
@@ -547,7 +531,6 @@ def compute(
         columns=["ticker", "amount_usd", "weight_exposure", "beta"]
     )
 
-    # plot
     img = plot_cml(
         rf_ann, erp_ann, sigma_mkt,
         sigma_hist, mu_capm,
@@ -575,7 +558,6 @@ def compute(
     ])
 
     uni_msg = f"Universe set to: {', '.join(UNIVERSE)}"
-    # Return also the scalars needed for re-plotting on band button clicks
     return (
         img, info, uni_msg, pos_table, sugg_table, csv_path,
         txt_low, txt_med, txt_high,
@@ -583,45 +565,6 @@ def compute(
         chosen_sigma, chosen_mu
     )
 
-def redraw_with_band(
-    band: str,
-    low_txt: str, med_txt: str, high_txt: str,     # just to keep signature consistent; not used
-    rf_ann: float, erp_ann: float, sigma_mkt: float,
-    sigma_hist: float, mu_capm: float,
-    same_sigma_mu: float, same_mu_sigma: float,
-    synth_csv_path: str,                            # not used; placeholder to keep wiring simple
-    # For building the selected df, we'll pass the three pick JSONs:
-    low_pick_json: str, med_pick_json: str, high_pick_json: str
-):
-    pick_map = {
-        "low":  json.loads(low_pick_json) if low_pick_json else None,
-        "medium": json.loads(med_pick_json) if med_pick_json else None,
-        "high": json.loads(high_pick_json) if high_pick_json else None,
-    }
-    chosen = pick_map.get((band or "medium").lower(), None)
-    if not chosen:
-        return gr.update(), empty_suggestion_df()
-
-    chosen_sigma = float(chosen["sigma_hist"])
-    chosen_mu = float(chosen["mu_capm"])
-    ts = [t.strip() for t in str(chosen["tickers"]).split(",") if t.strip()]
-    ws = [float(x) for x in str(chosen["weights"]).split(",")]
-    s = sum(ws) or 1.0
-    ws = [max(0.0, w) / s for w in ws]
-    budget = float(chosen.get("budget", 1.0))
-    sugg_table = pd.DataFrame(
-        [{"ticker": t, "weight_%": round(w*100.0, 2), "amount_$": round(w*budget, 0)} for t, w in zip(ts, ws)],
-        columns=["ticker", "weight_%", "amount_$"]
-    )
-
-    img = plot_cml(
-        rf_ann, erp_ann, sigma_mkt,
-        sigma_hist, mu_capm,
-        same_sigma_mu, same_mu_sigma,
-        sugg_sigma_hist=chosen_sigma, sugg_mu_capm=chosen_mu
-    )
-    return img, sugg_table
-
 # -------------- UI --------------
 with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
     gr.Markdown(
@@ -630,7 +573,6 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
         "Plot shows **your CAPM point on the CML** plus efficient market/bills points."
     )
 
-    # --- SEARCH & PORTFOLIO INPUTS
     with gr.Row():
         with gr.Column(scale=1):
             q = gr.Textbox(label="Search symbol")
@@ -683,55 +625,32 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
             )
             dl = gr.File(label="Generated dataset CSV", value=None, visible=True)
 
-    # Hidden state for re-plotting + picks (serialized)
-    st_rf = gr.State()
-    st_erp = gr.State()
-    st_sig_mkt = gr.State()
-    st_sig_p = gr.State()
-    st_mu_p = gr.State()
-    st_same_sigma_mu = gr.State()
-    st_same_mu_sigma = gr.State()
-
-    st_low_pick = gr.State()    # JSON string
-    st_med_pick = gr.State()
-    st_high_pick = gr.State()
-    st_budget = gr.State()
-
     # wire search / add / locking / horizon
     search_btn.click(fn=search_tickers_cb, inputs=q, outputs=[search_note, matches])
     add_btn.click(fn=add_symbol, inputs=[matches, table], outputs=[table, search_note])
     table.change(fn=lock_ticker_column, inputs=table, outputs=table)
     horizon.change(fn=set_horizon, inputs=horizon, outputs=universe_msg)
 
-    # main compute
-    def _compute_and_pack(lookback_v, table_v, band_to_show):
-        out = compute(lookback_v, table_v, band_to_show)
-        # Pack picks as JSON into states so the band buttons can re-draw without recomputing.
-        # We need to rebuild the same picks here to store them.
-        # To avoid recomputing heavy parts, we approximate by reading the dataset CSV (already saved)
-        # but since we returned the three text lines only, we’ll also store chosen pick info directly.
-        return out
-
+    # compute + render (default to Medium band)
     run_btn.click(
-        fn=_compute_and_pack,
+        fn=compute,
         inputs=[lookback, table, gr.State("Medium")],
         outputs=[
             plot, summary, universe_msg, positions, sugg_table, dl,
             low_txt, med_txt, high_txt,
-            st_rf, st_erp, st_sig_mkt, st_sig_p, st_mu_p, st_same_sigma_mu, st_same_mu_sigma,
-            gr.State(), gr.State()   # placeholders (unused chosen sigma/mu)
+            gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(),
+            gr.State(), gr.State()
         ]
     )
 
-    # To make the band buttons functional we recompute picks inside compute(),
-    # but for responsiveness, we’ll call compute again with the requested band.
+    # band buttons recompute picks quickly
     btn_low.click(
         fn=compute,
         inputs=[lookback, table, gr.State("Low")],
         outputs=[
             plot, summary, universe_msg, positions, sugg_table, dl,
             low_txt, med_txt, high_txt,
-            st_rf, st_erp, st_sig_mkt, st_sig_p, st_mu_p, st_same_sigma_mu, st_same_mu_sigma,
+            gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(),
             gr.State(), gr.State()
         ]
     )
@@ -741,7 +660,7 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
         outputs=[
             plot, summary, universe_msg, positions, sugg_table, dl,
             low_txt, med_txt, high_txt,
-            st_rf, st_erp, st_sig_mkt, st_sig_p, st_mu_p, st_same_sigma_mu, st_same_mu_sigma,
+            gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(),
             gr.State(), gr.State()
         ]
     )
@@ -751,7 +670,7 @@ with gr.Blocks(title="Efficient Portfolio Advisor") as demo:
         outputs=[
             plot, summary, universe_msg, positions, sugg_table, dl,
             low_txt, med_txt, high_txt,
-            st_rf, st_erp, st_sig_mkt, st_sig_p, st_mu_p, st_same_sigma_mu, st_same_mu_sigma,
+            gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(), gr.State(),
             gr.State(), gr.State()
         ]
     )
@@ -761,5 +680,4 @@ RF_CODE = fred_series_for_horizon(HORIZON_YEARS)
 RF_ANN = fetch_fred_yield_annual(RF_CODE)
 
 if __name__ == "__main__":
-    # Gradio 5.x — no concurrency_count in queue(); keep it simple
     demo.launch(server_name="0.0.0.0", server_port=7860, show_api=False)