first commit

.gitignore

@@ -0,0 +1,24 @@
+# Python cache
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+
+# Streamlit cache
+.streamlit/
+.cache/
+chache_file/
+
+# Model files 
+assets/model/
+*.pth
+*.pt
+*.joblib
+*.bin
+*.safetensors
+
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
+
+# Logs
+*.log

README.md

@@ -0,0 +1,2 @@
+# absa-indobert-web
+Analisis otomatis kritik dan saran berdasarkan aspek tertentu dan sentimen dengan model IndoBERT.

app.py

@@ -0,0 +1,353 @@
+"""
+app.py
+------
+Aplikasi berbasis web ABSA IndoBERT menggunakan Streamlit untuk analisis sentimen
+berbasis aspek dari kritik dan saran mahasiswa.
+"""
+import os
+import time
+from io import BytesIO
+import pandas as pd
+import streamlit as st
+from config import CONFIG, ASPEK_COLUMNS
+from model_utils import load_model_and_tokenizer, predict_multi_aspect
+from visualization import (
+    show_sentiment_bar_chart,
+    show_sentiment_pie_chart,
+    show_year_distribution,
+    show_semester_distribution,
+    show_prodi_distribution,
+    show_top10_matkul_distribution,
+    show_sentiment_by_year,
+    show_sentiment_by_semester,
+    show_sentiment_by_prodi,
+    show_sentiment_by_top10_matkul,
+    show_sentiment_stacked_percentage,
+)
+from preprocessing import text_preprocessing_pipeline
+
+os.makedirs("chache_file", exist_ok=True)
+# # 🔒 Opsional: Bersihkan cache hasil prediksi saat aplikasi dimulai ulang
+# if os.path.exists("chache_file/temp_predicted.csv"):
+#     os.remove("chache_file/temp_predicted.csv")
+
+# Konfigurasi halaman
+st.set_page_config(
+    page_title="ABSA IndoBERT",
+    layout="wide",
+    page_icon="💬"
+)
+
+# Load custom CSS
+with open(os.path.join("assets", "style.css"), encoding="utf-8") as f:
+    st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+
+
+# Fungsi cache untuk loading model
+@st.cache_resource(show_spinner=False)
+def get_model_resources():
+    """Memuat model dan tokenizer IndoBERT."""
+    return load_model_and_tokenizer()
+
+
+# Spinner loading model
+with st.spinner("Sedang memuat model IndoBERT dan tokenizer... Harap tunggu sebentar!"):
+    model, tokenizer, le, device = get_model_resources()
+
+success_placeholder = st.empty()
+success_placeholder.success("✅ Model dan tokenizer berhasil dimuat!")
+time.sleep(1)
+success_placeholder.empty()
+
+
+def convert_df_to_excel(df):
+    """Mengubah DataFrame menjadi file Excel dalam bentuk byte stream."""
+    output = BytesIO()
+    with pd.ExcelWriter(output, engine="openpyxl") as writer:
+        df.to_excel(writer, index=False)
+    return output.getvalue()
+
+
+# Judul aplikasi
+st.markdown("""
+    <h1 class='title-center'>💬 ABSA IndoBERT</h1>
+    <p style='text-align: center; font-size: 16px;'>
+        Analisis otomatis kritik dan saran berdasarkan aspek tertentu dan sentimen dengan model IndoBERT.
+    </p>
+""", unsafe_allow_html=True)
+
+# Upload file
+st.markdown("## 🧾 Unggah File Data")
+col1, col2 = st.columns([5, 1])
+with col1:
+    uploaded_file = st.file_uploader(" ", type=["xlsx"])
+
+if os.path.exists("chache_file/temp_predicted.csv"):
+    if st.button("🧹 Hapus Cache Prediksi"):
+        os.remove("chache_file/temp_predicted.csv")
+        st.markdown("""
+            <span style="color: green;">✅ Cache berhasil dihapus. Silahkan Refresh!</span>
+            <script>
+                setTimeout(function() {
+                    window.location.reload();
+                }, 100);
+            </script>
+        """, unsafe_allow_html=True)
+
+with col2:
+    st.markdown("")
+    st.markdown("")
+    with st.expander("📘 Petunjuk Penggunaan"):
+        st.markdown("""
+        - Unggah file `.xlsx` dengan kolom `kritik_saran`
+        - Sistem akan melakukan _preprocessing_ dan prediksi otomatis
+        - Hasil akhir dapat diunduh dalam bentuk file Excel dan divisualisasikan
+        """)
+
+# Inisialisasi session state
+if "df_predicted" not in st.session_state:
+    st.session_state.df_predicted = None
+
+# ✅ Jika belum ada df_predicted tapi ada file cache, load dari file
+if st.session_state.df_predicted is None and os.path.exists("chache_file/temp_predicted.csv"):
+    try:
+        df_cached = pd.read_csv("chache_file/temp_predicted.csv")
+        # Konversi kolom tahun ke integer jika ada
+        if "tahun" in df_cached.columns:
+            df_cached["tahun"] = pd.to_numeric(
+                df_cached["tahun"], errors='coerce').astype('Int64')
+        st.session_state.df_predicted = df_cached
+    except (pd.errors.ParserError, FileNotFoundError) as e:
+        st.warning(f"Gagal memuat cache: {e}")
+
+
+# Proses prediksi
+if uploaded_file:
+    file_bytes = uploaded_file.getvalue()
+    if "last_uploaded_file" not in st.session_state or st.session_state.last_uploaded_file != file_bytes:
+        st.session_state.last_uploaded_file = file_bytes
+        try:
+            df_uploaded = pd.read_excel(BytesIO(file_bytes))
+
+            # Konversi kolom tahun ke integer jika ada
+            if "tahun" in df_uploaded.columns:
+                df_uploaded["tahun"] = pd.to_numeric(
+                    df_uploaded["tahun"], errors='coerce').astype('Int64')
+
+        except ValueError as err:
+            st.error(f"❌ Gagal membaca file: {err}")
+        else:
+            if "kritik_saran" not in df_uploaded.columns:
+                st.error("❌ Kolom 'kritik_saran' tidak ditemukan.")
+            else:
+                df_uploaded = df_uploaded.drop_duplicates(
+                    subset=["kritik_saran"])
+                for aspek in ASPEK_COLUMNS:
+                    if aspek not in df_uploaded.columns:
+                        df_uploaded[aspek] = None
+
+                st.markdown("## ⚙️ Preprocessing dan Prediksi")
+                progress = st.progress(1, text="Menyiapkan...")
+
+                pred_results = []
+                with st.spinner("Sedang memproses data..."):
+                    for i, (_, row) in enumerate(df_uploaded.iterrows()):
+                        cleaned_text = text_preprocessing_pipeline(
+                            str(row["kritik_saran"]))
+                        hasil = predict_multi_aspect(
+                            model, tokenizer, cleaned_text, ASPEK_COLUMNS, le, device, CONFIG[
+                                "max_len"]
+                        )
+                        result_row = row.to_dict()
+                        result_row["kritik_saran"] = cleaned_text
+                        result_row.update(hasil)
+                        pred_results.append(result_row)
+                        progress.progress((i + 1) / len(df_uploaded),
+                                          text=f"Memproses baris ke-{i + 1} dari {len(df_uploaded)}")
+                # ✅ Simpan ke session dan file sementara
+                df_session = pd.DataFrame(pred_results)
+                st.session_state.df_predicted = df_session
+                df_session.to_csv(
+                    "chache_file/temp_predicted.csv", index=False)
+                st.success("✅ Preprocessing & Prediksi selesai!")
+
+# Setelah prediksi selesai
+if st.session_state.df_predicted is not None:
+    df_predicted = st.session_state.df_predicted
+
+    # Sidebar filter - Pindahkan ke atas sebelum menampilkan tabel
+    st.sidebar.header("🔍 Filter Data")
+
+    # Pastikan kolom yang difilter ada dan bersihkan nilai NaN
+    df_clean = df_predicted.copy()
+
+    # Filter Mata Kuliah
+    if "nama_matakuliah" in df_clean.columns:
+        matkul_options = sorted(
+            [x for x in df_clean["nama_matakuliah"].dropna().unique() if x])
+        selected_matkul = st.sidebar.multiselect(
+            "Nama Mata Kuliah", matkul_options, default=matkul_options)
+    else:
+        selected_matkul = []
+
+    # Filter Program Studi
+    if "nama_prodi" in df_clean.columns:
+        prodi_options = sorted(
+            [x for x in df_clean["nama_prodi"].dropna().unique() if x])
+        selected_prodi = st.sidebar.multiselect(
+            "Program Studi", prodi_options, default=prodi_options)
+    else:
+        selected_prodi = []
+
+    # Filter Tahun
+    if "tahun" in df_clean.columns:
+        tahun_options = sorted(
+            [x for x in df_clean["tahun"].dropna().unique() if pd.notna(x)])
+        selected_tahun = st.sidebar.multiselect(
+            "Tahun", tahun_options, default=tahun_options)
+    else:
+        selected_tahun = []
+
+    # Filter Semester
+    if "semester" in df_clean.columns:
+        semester_options = sorted(
+            [x for x in df_clean["semester"].dropna().unique() if pd.notna(x)])
+        selected_semester = st.sidebar.multiselect(
+            "Semester", semester_options, default=semester_options)
+    else:
+        selected_semester = []
+
+    # Apply filters dengan pengecekan kolom yang ada
+    df_filtered = df_clean.copy()
+
+    if selected_matkul and "nama_matakuliah" in df_filtered.columns:
+        df_filtered = df_filtered[df_filtered["nama_matakuliah"].isin(
+            selected_matkul)]
+
+    if selected_prodi and "nama_prodi" in df_filtered.columns:
+        df_filtered = df_filtered[df_filtered["nama_prodi"].isin(
+            selected_prodi)]
+
+    if selected_tahun and "tahun" in df_filtered.columns:
+        df_filtered = df_filtered[df_filtered["tahun"].isin(selected_tahun)]
+
+    if selected_semester and "semester" in df_filtered.columns:
+        df_filtered = df_filtered[df_filtered["semester"].isin(
+            selected_semester)]
+
+    # Tampilkan tabel hasil prediksi yang sudah difilter
+    st.markdown("### 📄 Tabel Hasil Prediksi")
+    st.dataframe(df_filtered, width="stretch")
+
+    # Download buttons dalam satu kolom
+    st.download_button(
+        label="⬇️ Unduh Hasil Excel (Data Terfilter)",
+        data=convert_df_to_excel(df_filtered),
+        file_name="hasil_prediksi_absa_filtered.xlsx",
+        mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    )
+
+    st.download_button(
+        label="⬇️ Unduh Semua Data Excel",
+        data=convert_df_to_excel(df_predicted),
+        file_name="hasil_prediksi_absa_all.xlsx",
+        mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+    )
+
+    st.info(
+        f"Menampilkan {len(df_filtered)} dari {len(df_predicted)} data ulasan setelah difilter."
+    )
+
+    # Ringkasan Cepat - menggunakan data yang sudah difilter
+    st.markdown("### 📌 Ringkasan Cepat")
+
+    total_pos = (df_filtered[ASPEK_COLUMNS] == "positif").sum().sum()
+    total_net = (df_filtered[ASPEK_COLUMNS] == "netral").sum().sum()
+    total_neg = (df_filtered[ASPEK_COLUMNS] == "negatif").sum().sum()
+
+    col1, col2, col3, col4, col5 = st.columns(5)
+    col1.metric("Jumlah Ulasan", len(df_filtered))
+    col2.metric("Jumlah Aspek", len(ASPEK_COLUMNS))
+
+    # Tambahkan pengecekan untuk kolom yang mungkin tidak ada
+    matkul_count = df_filtered['nama_matakuliah'].nunique(
+    ) if 'nama_matakuliah' in df_filtered.columns else 0
+    prodi_count = df_filtered['nama_prodi'].nunique(
+    ) if 'nama_prodi' in df_filtered.columns else 0
+    semester_count = df_filtered['semester'].nunique(
+    ) if 'semester' in df_filtered.columns else 0
+
+    col3.metric("Jumlah Mata Kuliah", matkul_count)
+    col4.metric("Jumlah Prodi", prodi_count)
+    col5.metric("Jumlah Semester", semester_count)
+
+    col6, col7, col8, col9, col10 = st.columns(5)
+    col6.metric("Sentimen Positif", f"{total_pos}")
+    col7.metric("Sentimen Netral", f"{total_net}")
+    col8.metric("Sentimen Negatif", f"{total_neg}")
+
+    # Tampilkan rentang tahun jika kolom tahun ada
+    if 'tahun' in df_filtered.columns and len(df_filtered) > 0:
+        tahun_min = int(df_filtered['tahun'].min())
+        tahun_max = int(df_filtered['tahun'].max())
+        col9.metric("Rentang Tahun", f"{tahun_min} - {tahun_max}")
+    else:
+        col9.metric("Rentang Tahun", "N/A")
+
+    # Tambahkan rata-rata panjang kata dalam kritik saran
+    if len(df_filtered) > 0 and 'kritik_saran' in df_filtered.columns:
+        word_counts = df_filtered['kritik_saran'].astype(
+            str).str.split().str.len()
+        avg_word_count = round(word_counts.mean(), 1)
+        col10.metric("Rata-rata Panjang Kata", f"{avg_word_count} kata")
+    else:
+        col10.metric("Rata-rata Panjang Kata", "0 kata")
+
+    st.markdown("---")
+    st.markdown("### 📊 Visualisasi Data")
+
+    col1, col2 = st.columns(2)
+    with col1:
+        show_sentiment_bar_chart(df_filtered, ASPEK_COLUMNS)
+    with col2:
+        show_sentiment_pie_chart(df_filtered, ASPEK_COLUMNS)
+
+    col1, col2 = st.columns(2)
+    with col1:
+        show_year_distribution(df_filtered)
+    with col2:
+        show_semester_distribution(df_filtered)
+
+    st.markdown("---")
+    show_prodi_distribution(df_filtered)
+
+    st.markdown("---")
+    show_top10_matkul_distribution(df_filtered)
+
+    st.markdown("---")
+    col1, col2 = st.columns(2)
+    with col1:
+        show_sentiment_by_year(df_filtered, ASPEK_COLUMNS)
+    with col2:
+        show_sentiment_by_semester(df_filtered, ASPEK_COLUMNS)
+
+    st.markdown("---")
+    show_sentiment_by_prodi(df_filtered, ASPEK_COLUMNS)
+
+    st.markdown("---")
+    show_sentiment_by_top10_matkul(df_filtered, ASPEK_COLUMNS)
+
+    st.markdown("---")
+    show_sentiment_stacked_percentage(df_filtered, ASPEK_COLUMNS)
+
+    #
+
+else:
+    st.info("ℹ️ Silakan unggah file Excel untuk memulai proses")
+
+# Footer
+st.markdown("""
+    <div class='footer'>
+        © 2025 Darmawan Jiddan – ABSA IndoBERT | Dibuat dengan ❤️ menggunakan Streamlit
+    </div>
+""", unsafe_allow_html=True)

assets/style.css

@@ -0,0 +1,46 @@
+/* Gaya Umum untuk Light & Dark Mode */
+/* .block-container {
+    padding-top: 2rem;
+} */
+
+/* Heading */
+h1, h2, h3, h4, h5, h6, p,
+.stMarkdown h2,
+.stMarkdown h3 {
+    color: var(--text-color);
+}
+
+/* Ukuran font tabel */
+.stDataFrame div {
+    font-size: 14px;
+}
+
+/* Background elemen tertentu */
+.css-1cpxqw2 {
+    background-color: var(--secondary-background-color);
+}
+
+/* Teks kecil */
+.small-text {
+    font-size: 0.9em;
+    color: var(--text-color-secondary);
+}
+
+/* Judul utama di tengah */
+h1.title-center {
+    text-align: center;
+    margin-bottom: 10px;
+}
+
+.footer {
+    position: relative;
+    bottom: 0;
+    width: 100%;
+    padding: 10px 0;
+    background-color: var(--background-color);
+    color: var(--text-color);
+    text-align: center;
+    font-size: 14px;
+    border-top: 1px solid var(--secondary-background-color);
+    margin-top: 50px;
+}

clean_reqs.py

@@ -0,0 +1,47 @@
+"""
+Clean_reqs.py
+-------------
+Script untuk membersihkan file requirements.txt dengan cara:
+- Menghapus versi duplikat paket.
+- Menyimpan versi terbaru untuk setiap paket.
+"""
+
+import re
+from packaging import version
+
+
+def clean_requirements(file_path="requirements.txt"):
+    """
+    Membersihkan file requirements.txt.
+
+    Args:
+        file_path (str): Path ke file requirements.txt yang akan dibersihkan.
+
+    Proses:
+        - Melewatkan komentar dan baris kosong.
+        - Menyimpan versi terbaru untuk setiap paket.
+        - Menulis ulang file requirements.txt yang sudah dibersihkan.
+    """
+    packages = {}
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+
+            match = re.match(r"([a-zA-Z0-9_\-]+)==(.+)", line)
+            if match:
+                name, ver = match.groups()
+                if name not in packages or version.parse(ver) > version.parse(packages[name]):
+                    packages[name] = ver
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        for name, ver in sorted(packages.items()):
+            f.write(f"{name}=={ver}\n")
+
+    print(f"✅ Cleaned requirements saved to {file_path}")
+
+
+if __name__ == "__main__":
+    clean_requirements()

config.py

@@ -0,0 +1,14 @@
+"""
+config.py
+---------
+Berisi konfigurasi utama untuk model ABSA (Aspect-Based Sentiment Analysis)
+menggunakan IndoBERT, termasuk parameter model dan daftar kolom aspek.
+"""
+
+CONFIG = {
+    "model_name": "indobenchmark/indobert-base-p1",
+    "dropout_rate": 0.3,
+    "max_len": 128
+}
+
+ASPEK_COLUMNS = ["tugas", "pengajaran", "materi", "metode", "interaksi"]

model_utils.py

@@ -0,0 +1,118 @@
+"""
+model_utils.py
+--------------
+Berisi utilitas untuk memuat model IndoBERT ABSA, tokenizer, dan label encoder,
+serta fungsi untuk melakukan prediksi multi-aspek.
+"""
+
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import joblib
+from transformers import AutoModel, AutoTokenizer
+from config import CONFIG
+
+
+def load_model_and_tokenizer():
+    """
+    Memuat model IndoBERT ABSA, tokenizer, dan label encoder.
+
+    Returns:
+        model (nn.Module): Model ABSA yang sudah diload.
+        tokenizer (AutoTokenizer): Tokenizer untuk IndoBERT.
+        label_encoder (LabelEncoder): Encoder untuk label sentimen.
+        device (torch.device): Device (cuda/cpu) yang digunakan.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(CONFIG["model_name"])
+
+    base_path = os.path.abspath(os.path.dirname(__file__))
+    model_dir = os.path.join(base_path, "assets", "model")
+    model_path = os.path.join(model_dir, "indobert_absa_model.pth")
+    label_path = os.path.join(model_dir, "label_encoder.joblib")
+
+    if not os.path.exists(model_path) or not os.path.exists(label_path):
+        raise FileNotFoundError("Model atau Label Encoder tidak ditemukan.")
+
+    label_encoder = joblib.load(label_path)
+
+    class IndoBertForABSA(nn.Module):
+        """
+        Model klasifikasi aspek berbasis IndoBERT untuk ABSA.
+        """
+
+        def __init__(self, num_labels):
+            super().__init__()
+            self.bert = AutoModel.from_pretrained(
+                CONFIG["model_name"], trust_remote_code=True, use_safetensors=True
+            )
+            self.norm = nn.LayerNorm(self.bert.config.hidden_size)
+            self.dropout = nn.Dropout(CONFIG["dropout_rate"])
+            self.classifier = nn.Linear(
+                self.bert.config.hidden_size, num_labels)
+
+        def forward(self, input_ids, attention_mask):
+            """
+            Forward pass untuk model ABSA.
+
+            Args:
+                input_ids (torch.Tensor): Tensor input token IDs.
+                attention_mask (torch.Tensor): Tensor mask perhatian.
+
+            Returns:
+                torch.Tensor: Logit prediksi.
+            """
+            output = self.bert(input_ids=input_ids,
+                               attention_mask=attention_mask)
+            pooled = output.pooler_output
+            normed = self.norm(pooled)
+            dropped = self.dropout(normed)
+            return self.classifier(dropped)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = IndoBertForABSA(num_labels=len(label_encoder.classes_))
+    model.load_state_dict(torch.load(model_path, map_location=device))
+    model.to(device)
+    model.eval()
+
+    return model, tokenizer, label_encoder, device
+
+
+def predict_multi_aspect(model, tokenizer, sentence, aspek_list, label_encoder, device, max_len):
+    """
+    Melakukan prediksi sentimen untuk setiap aspek pada satu kalimat.
+
+    Args:
+        model (nn.Module): Model ABSA yang sudah diload.
+        tokenizer (AutoTokenizer): Tokenizer IndoBERT.
+        sentence (str): Kalimat input.
+        aspek_list (list): Daftar aspek yang ingin diprediksi.
+        label_encoder (LabelEncoder): Encoder label.
+        device (torch.device): Device (cuda/cpu).
+        max_len (int): Panjang maksimum token.
+
+    Returns:
+        dict: Hasil prediksi berupa {aspek: label_sentimen}.
+    """
+    results = {}
+    for aspek in aspek_list:
+        combined = f"[ASPEK] {aspek} [TEXT] {sentence}"
+        encoded = tokenizer.encode_plus(
+            combined,
+            add_special_tokens=True,
+            padding="max_length",
+            max_length=max_len,
+            truncation=True,
+            return_attention_mask=True,
+            return_tensors="pt",
+        )
+        input_ids = encoded["input_ids"].to(device)
+        attention_mask = encoded["attention_mask"].to(device)
+
+        with torch.no_grad():
+            outputs = model(input_ids, attention_mask)
+            probs = F.softmax(outputs, dim=1).squeeze()
+            idx = torch.argmax(probs).item()
+            label = label_encoder.inverse_transform([idx])[0]
+            results[aspek] = label
+    return results

preprocessing.py

@@ -0,0 +1,111 @@
+"""
+preprocessing.py
+----------------
+Modul preprocessing teks untuk ABSA IndoBERT, termasuk pembersihan teks,
+normalisasi slang, tokenisasi, dan penghapusan emoji.
+"""
+
+import re
+import os
+import nltk
+from nltk.tokenize import word_tokenize
+from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
+
+# Tentukan folder NLTK di project atau environment
+NLTK_DATA_DIR = os.path.join(os.path.dirname(__file__), "nltk_data")
+os.makedirs(NLTK_DATA_DIR, exist_ok=True)
+
+# Tambahkan path NLTK supaya bisa menemukan data
+nltk.data.path.append(NLTK_DATA_DIR)
+
+# Download punkt jika belum ada
+try:
+    nltk.data.find("tokenizers/punkt")
+except LookupError:
+    nltk.download("punkt", download_dir=NLTK_DATA_DIR)
+
+# Stemmer Sastrawi
+stemmer = StemmerFactory().create_stemmer()
+
+slang_dict = {
+    'sy': 'saya', 'aku': 'saya', 'aq': 'saya', 'gua': 'saya', 'gue': 'saya',
+    'km': 'kamu', 'kmu': 'kamu', 'lu': 'kamu', 'loe': 'kamu', 'lo': 'kamu',
+    'gk': 'tidak', 'ga': 'tidak', 'nggak': 'tidak', 'tdk': 'tidak', 'enggak': 'tidak', 'g': 'tidak',
+    'bgt': 'banget', 'bangt': 'banget', 'bngt': 'banget',
+    'bnyk': 'banyak', 'smw': 'semua', 'bbrp': 'beberapa',
+    'udh': 'sudah', 'sdh': 'sudah', 'dah': 'sudah', 'td': 'tadi',
+    'bsk': 'besok', 'kmrn': 'kemarin', 'skrng': 'sekarang', 'skrg': 'sekarang', 'nnti': 'nanti',
+    'mlm': 'malam', 'pgi': 'pagi',
+    'jg': 'juga', 'aja': 'saja', 'aj': 'saja', 'jd': 'jadi', 'lg': 'lagi', 'lgi': 'lagi',
+    'tp': 'tapi', 'tpi': 'tapi', 'tpnya': 'tapi', 'trs': 'terus', 'trus': 'terus', 'trsnya': 'terusnya',
+    'krn': 'karena', 'karana': 'karena', 'utk': 'untuk', 'bsa': 'bisa',
+    'dr': 'dari', 'dpn': 'depan', 'blkg': 'belakang', 'dkt': 'dekat',
+    'ngajar': 'mengajar', 'ngasih': 'memberi', 'ngerti': 'mengerti', 'ngumpul': 'mengumpulkan', 'ngulang': 'mengulang',
+    'bikin': 'membuat', 'ajar': 'mengajar', 'ajarkan': 'mengajarkan', 'diajar': 'diajarkan', 'mengampu': 'mengajar', 'diampu': 'diajarkan',
+    'dosen2': 'dosen', 'mhs': 'mahasiswa', 'mhsw': 'mahasiswa', 'nilai2': 'nilai', 'mantul': 'mantap betul',
+    'ny': 'nya', 'gitu': 'seperti itu', 'bgtu': 'begitu',
+    'sm': 'sama', 'dtg': 'datang', 'mnt': 'menit', 'cepet': 'cepat', 'cpt': 'cepat',
+    'kayak': 'seperti', 'kyk': 'seperti', 'dpt': 'dapat', 'masingmasing': 'masing-masing',
+    'terimakasih': 'terima kasih', 'terimaksih': 'terima kasih', 'terimakasi': 'terima kasih', 'makasi': 'terima kasih', 'kasih': 'terima kasih',
+    'sebaiknya': 'sebaik nya', 'akan tetapi': 'tetapi',
+    'diperjelas': 'dijelaskan', 'diperbaiki': 'dibetulkan',
+    'diparkan': 'dipaparkan', 'refrensi': 'referensi',
+    'pemblajaran': 'pembelajaran', 'pemebelajaran': 'pembelajaran', 'pembelajran': 'pembelajaran', 'pembelajara': 'pembelajaran', 'palajaran': 'pelajaran',
+    'pendikan': 'pendidikan', 'matkul': 'matakuliah',
+    'ofline': 'offline', 'dapatkan': 'diperoleh', 'tatap': 'tatap muka',
+    'kouta': 'kuota', 'sekira': 'kira-kira', 'selow': 'santai', 'penyampain': 'penyampaian',
+    'membing': 'membimbing', 'memaksimalkan': 'maksimal',
+    'tap': 'tatap', 'wasan': 'wawasan', 'pelan': 'lambat', 'iya': 'ya', 'ijin': 'izin',
+    'seperti itu': '', 'dik': '', 'mpk': '', 'pu': '', 'ta': '', 'the': '',
+    'mendik': 'mendidik', 'efesien': 'efisien', 'menuhi': 'memenuhi', 'tep': 'tepat',
+    'pendik': 'pendidikan', 'semster': 'semester', 'vidio': 'video',
+    'asinkronus': 'asyncronous', 'sinkronus': 'syncronous', 'mahasiswai': 'mahasiswa',
+    'perkulihan': 'perkuliahan', 'kedepa': 'kedepan', 'projek': 'proyek', 'bertap': 'bertatap',
+    'perkuliah': 'perkuliahan', 'bajk': 'baik', 'diskus': 'diskusi', 'praktek': 'praktik',
+}
+
+pattern_slang = re.compile(r'\b(' + '|'.join(re.escape(k) for k in slang_dict) + r')\b', flags=re.IGNORECASE)  # noqa: C0301, C0201
+
+
+def cleaning_text(text: str) -> str:  # noqa: C0103
+    """Membersihkan teks: HTML, simbol, karakter aneh, dan rapikan tanda baca."""  # noqa: C0116
+    if not isinstance(text, str):
+        return ''
+    text = re.sub(r'<[^>]+>', '', text)
+    text = re.sub(r'&gt|&lt', ' ', text)
+    text = re.sub(r'([a-z])([A-Z])', r'\1. \2', text)
+    text = text.replace('\n', ' ')
+    text = re.sub(r'[^a-zA-Z0-9\s.,!?\-:;\'"()]', '', text)
+    text = re.sub(r'([!?.,])\1{2,}', r'\1\1', text)
+    text = re.sub(r'([.!?])(\w)', r'\1 \2', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+
+def remove_emoji(text):
+    """Hapus emoji dari teks."""  # noqa: C0116
+    emoji_pattern = re.compile(
+        "["
+        u"\U0001F600-\U0001F64F"
+        u"\U0001F300-\U0001F5FF"
+        u"\U0001F680-\U0001F6FF"
+        u"\U0001F1E0-\U0001F1FF"
+        u"\U00002702-\U000027B0"
+        u"\U000024C2-\U0001F251"
+        "]+", flags=re.UNICODE
+    )
+    return emoji_pattern.sub(r'', text)
+
+
+def normalize_text(tokens):  # noqa: C0103
+    """Normalisasi kata berdasarkan kamus slang."""  # noqa: C0116
+    return [slang_dict.get(w.lower(), w.lower()) for w in tokens]
+
+
+def text_preprocessing_pipeline(text):
+    """Pipeline preprocessing lengkap: cleaning, emoji removal, tokenisasi, normalisasi."""  # noqa: C0116
+    text = cleaning_text(text)  # noqa: C0103
+    text = remove_emoji(text)
+    tokens = word_tokenize(text)
+    tokens = normalize_text(tokens)  # noqa: C0103
+    return " ".join(tokens)

requirements.txt

@@ -0,0 +1,9 @@
+joblib==1.5.2
+nltk==3.9.1
+packaging==25.0
+pandas==2.3.2
+plotly==6.3.0
+Sastrawi==1.0.1
+streamlit==1.50.0
+torch==2.7.1+cu118
+transformers==4.56.2

visualization.py

@@ -0,0 +1,331 @@
+"""
+Modul visualization.py
+----------------------
+Berisi fungsi-fungsi untuk menampilkan berbagai visualisasi data kritik dan saran
+dalam bentuk bar chart, pie chart, serta distribusi berdasarkan tahun, semester,
+program studi, dan mata kuliah menggunakan Streamlit & Plotly.
+"""
+
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+from config import ASPEK_COLUMNS
+
+
+# Palet warna kustom
+sentimen_palette = {
+    "netral": "#FFE24C",
+    "positif": "#4CFF72",
+    "negatif": "#FF4C4C"
+}
+category_order = ["netral", "positif", "negatif"]
+
+# Konfigurasi Plotly
+config_options = {
+    "scrollZoom": False,
+    "displayModeBar": False
+}
+
+
+def show_sentiment_bar_chart(df_predicted, aspek_columns):
+    """Menampilkan bar chart distribusi sentimen per aspek."""
+    if df_predicted.empty or not set(aspek_columns).issubset(df_predicted.columns):
+        st.warning("Data atau kolom aspek tidak tersedia untuk ditampilkan.")
+        return
+
+    df_long = df_predicted.melt(
+        value_vars=aspek_columns,
+        var_name="aspek",
+        value_name="sentimen"
+    )
+    df_long["sentimen"] = pd.Categorical(
+        df_long["sentimen"],
+        categories=category_order,
+        ordered=True
+    )
+    count_data = df_long.groupby(
+        ["aspek", "sentimen"], observed=False
+    ).size().reset_index(name="jumlah")
+    fig = px.bar(
+        count_data,
+        x="aspek",
+        y="jumlah",
+        color="sentimen",
+        barmode="group",
+        color_discrete_map=sentimen_palette,
+        category_orders={"sentimen": category_order}
+    )
+    fig.update_layout(title="Distribusi Sentimen per Aspek")
+    st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_pie_chart(df_predicted, aspek_columns):
+    """Menampilkan pie chart distribusi total sentimen."""
+    sentimen_total = df_predicted[aspek_columns].values.ravel()
+    sentimen_counts = pd.Series(sentimen_total).value_counts().reset_index()
+    sentimen_counts.columns = ["sentimen", "jumlah"]
+    sentimen_counts = sentimen_counts.sort_values("jumlah", ascending=False)
+    fig = px.pie(sentimen_counts, names="sentimen", values="jumlah",
+                 color="sentimen", color_discrete_map=sentimen_palette,
+                 hole=0.3)
+    fig.update_layout(title="Total Komposisi Sentimen")
+    fig.update_traces(textposition='inside', textinfo='percent+label')
+    st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_year_distribution(df):
+    """Menampilkan distribusi jumlah kritik/saran per tahun."""
+    if 'tanggal' in df.columns:
+        df['tahun'] = pd.to_datetime(df['tanggal'], errors='coerce').dt.year
+
+    if 'tahun' in df.columns:
+        df_tahun = df.dropna(subset=['tahun']).copy()
+        df_tahun['tahun'] = df_tahun['tahun'].astype(int)
+
+        year_counts = df_tahun['tahun'].value_counts().reset_index()
+        year_counts.columns = ['tahun', 'jumlah']
+        year_counts = year_counts.sort_values('jumlah', ascending=False)
+
+        fig = px.bar(year_counts, x='tahun', y='jumlah',
+                     color='tahun', title="Distribusi Kritik/Saran per Tahun")
+        fig.update_layout(xaxis=dict(type='category'))
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_semester_distribution(df):
+    """Menampilkan distribusi jumlah kritik/saran per semester."""
+    if 'semester' in df.columns:
+        semester_counts = df['semester'].value_counts().reset_index()
+        semester_counts.columns = ['semester', 'jumlah']
+        semester_counts = semester_counts.sort_values(
+            'jumlah', ascending=False)
+        fig = px.bar(semester_counts, x='semester', y='jumlah',
+                     color='semester', title="Distribusi Kritik/Saran per Semester")
+        fig.update_layout(xaxis=dict(categoryorder='total descending'))
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_prodi_distribution(df):
+    """Menampilkan jumlah kritik/saran per program studi."""
+    if 'nama_prodi' in df.columns:
+        prodi_counts = df['nama_prodi'].value_counts().reset_index()
+        prodi_counts.columns = ['nama_prodi', 'jumlah']
+        prodi_counts = prodi_counts.sort_values(by='jumlah', ascending=True)
+        fig = px.bar(
+            prodi_counts,
+            x='jumlah',
+            y='nama_prodi',
+            orientation='h',
+            color='jumlah',
+            title="Jumlah Kritik/Saran per Program Studi"
+        )
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_top10_matkul_distribution(df):
+    """Menampilkan 10 mata kuliah dengan jumlah kritik/saran terbanyak."""
+    if 'nama_matakuliah' in df.columns and 'kode_matakuliah' in df.columns:
+        matkul_counts = (
+            df.groupby(['kode_matakuliah', 'nama_matakuliah'], observed=False)
+            .size()
+            .reset_index(name='jumlah')
+            .sort_values(by='jumlah', ascending=False)
+            .head(10)
+        )
+        matkul_counts['label'] = (
+            matkul_counts['kode_matakuliah'] + " - " +
+            matkul_counts['nama_matakuliah']
+        )
+        matkul_counts = matkul_counts.sort_values(by='jumlah', ascending=True)
+
+        fig = px.bar(
+            matkul_counts,
+            x='jumlah',
+            y='label',
+            orientation='h',
+            title="Top 10 Mata Kuliah Berdasarkan Kritik/Saran",
+            color='jumlah'
+        )
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_by_year(df, aspek_columns):
+    """Menampilkan distribusi sentimen per tahun."""
+    if 'tahun' in df.columns:
+        df_long = df.melt(id_vars=['tahun'],
+                          value_vars=aspek_columns,
+                          var_name='aspek',
+                          value_name='sentimen')
+        year_sentiment = df_long.groupby(
+            ['tahun', 'sentimen'], observed=False
+        ).size().reset_index(name='jumlah')
+        year_sentiment = year_sentiment.sort_values('jumlah', ascending=False)
+        fig = px.bar(year_sentiment, x='tahun', y='jumlah', color='sentimen',
+                     barmode='group', color_discrete_map=sentimen_palette)
+        fig.update_layout(title="Distribusi Sentimen Kritik/Saran per Tahun")
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_by_semester(df, aspek_columns):
+    """Menampilkan distribusi sentimen per semester."""
+    if 'semester' in df.columns:
+        df_long = df.melt(id_vars=['semester'],
+                          value_vars=aspek_columns,
+                          var_name='aspek',
+                          value_name='sentimen')
+        semester_sentiment = df_long.groupby(
+            ['semester', 'sentimen'], observed=False
+        ).size().reset_index(name='jumlah')
+        semester_sentiment = semester_sentiment.sort_values(
+            'jumlah', ascending=False)
+        fig = px.bar(semester_sentiment, x='semester', y='jumlah', color='sentimen',
+                     barmode='group', color_discrete_map=sentimen_palette)
+        fig.update_layout(
+            title="Distribusi Sentimen Kritik/Saran per Semester")
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_by_prodi(df, aspek_columns):
+    """Menampilkan distribusi sentimen per program studi."""
+    if 'nama_prodi' in df.columns:
+        df_long = df.melt(
+            id_vars=['nama_prodi'],
+            value_vars=aspek_columns,
+            var_name='aspek',
+            value_name='sentimen'
+        )
+
+        prodi_sentiment = (
+            df_long.groupby(['nama_prodi', 'sentimen'], observed=False)
+            .size()
+            .reset_index(name='jumlah')
+        )
+
+        total_per_prodi = (
+            prodi_sentiment.groupby('nama_prodi')['jumlah']
+            .sum()
+            .sort_values(ascending=False)
+        )
+        ordered_categories = total_per_prodi.index.tolist()[::-1]
+
+        prodi_sentiment['nama_prodi'] = pd.Categorical(
+            prodi_sentiment['nama_prodi'],
+            categories=ordered_categories,
+            ordered=True
+        )
+
+        fig = px.bar(
+            prodi_sentiment,
+            y='nama_prodi',
+            x='jumlah',
+            color='sentimen',
+            barmode='group',
+            orientation='h',
+            color_discrete_map=sentimen_palette
+        )
+        fig.update_layout(
+            title="Distribusi Sentimen per Program Studi",
+            yaxis={'categoryorder': 'array',
+                   'categoryarray': ordered_categories}
+        )
+        st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_by_top10_matkul(df, aspek_columns):
+    """Menampilkan distribusi sentimen pada 10 mata kuliah teratas."""
+    df_top10 = (
+        df.groupby(['kode_matakuliah', 'nama_matakuliah'], observed=False)
+        .size()
+        .sort_values(ascending=False)
+        .head(10)
+        .index
+    )
+
+    df_filtered = df[df.set_index(
+        ['kode_matakuliah', 'nama_matakuliah']).index.isin(df_top10)]
+
+    df_long = df_filtered.melt(
+        id_vars=['kode_matakuliah', 'nama_matakuliah'],
+        value_vars=aspek_columns,
+        var_name='aspek',
+        value_name='sentimen'
+    )
+
+    df_long['label'] = (
+        df_long['kode_matakuliah'] + " - " + df_long['nama_matakuliah']
+    )
+
+    matkul_sentiment = (
+        df_long.groupby(['label', 'sentimen'], observed=False)
+        .size()
+        .reset_index(name='jumlah')
+    )
+
+    total_per_label = (
+        matkul_sentiment.groupby('label')['jumlah']
+        .sum()
+        .sort_values(ascending=False)
+    )
+    ordered_labels = total_per_label.index.tolist()[::-1]
+
+    matkul_sentiment['label'] = pd.Categorical(
+        matkul_sentiment['label'],
+        categories=ordered_labels,
+        ordered=True
+    )
+
+    fig = px.bar(
+        matkul_sentiment,
+        y='label',
+        x='jumlah',
+        color='sentimen',
+        barmode='group',
+        orientation='h',
+        color_discrete_map=sentimen_palette
+    )
+    fig.update_layout(
+        title="Distribusi Sentimen pada Top 10 Mata Kuliah",
+        yaxis={'categoryorder': 'array', 'categoryarray': ordered_labels}
+    )
+    st.plotly_chart(fig, use_container_width=True, config=config_options)
+
+
+def show_sentiment_stacked_percentage(df, aspek_columns):
+    """Menampilkan stacked bar chart dengan persentase sentimen per aspek."""
+
+    if df.empty or not set(aspek_columns).issubset(df.columns):
+        st.warning("Data atau kolom aspek tidak tersedia.")
+        return
+
+    df_long = df.melt(
+        value_vars=aspek_columns,
+        var_name="aspek",
+        value_name="sentimen"
+    )
+
+    # Hitung persentase
+    count_data = df_long.groupby(
+        ['aspek', 'sentimen']).size().reset_index(name='jumlah')
+    total_per_aspek = count_data.groupby('aspek')['jumlah'].sum().reset_index()
+    total_per_aspek.columns = ['aspek', 'total']
+    count_data = count_data.merge(total_per_aspek, on='aspek')
+    count_data['persentase'] = (
+        count_data['jumlah'] / count_data['total']) * 100
+
+    fig = px.bar(
+        count_data,
+        x="aspek",
+        y="persentase",
+        color="sentimen",
+        title="Persentase Distribusi Sentimen per Aspek",
+        color_discrete_map=sentimen_palette,
+        category_orders={
+            "sentimen": category_order,
+            "aspek": ASPEK_COLUMNS  # Add this line to order aspects
+        }
+    )
+    fig.update_layout(
+        yaxis_title="Persentase (%)",
+        xaxis_title="Aspek"
+    )
+    st.plotly_chart(fig, use_container_width=True, config=config_options)