add

requirements.txt

@@ -1,11 +1,12 @@
 transformers[torch]==4.36.2
 numpy==1.26.3
 #scikit-learn==1.3.2
-#matplotlib==3.8.2
+matplotlib==3.8.2
 datasets==2.16.1
 evaluate==0.4.1
 accelerate==0.25.0
 seqeval==1.2.2
 pandas==2.1.4
 gradio==4.13.0
-pydantic_settings==2.1.0
+pydantic_settings==2.1.0
+sentencepiece==0.1.99

source/services/predicting_effective_arguments/train/02_classification copy.py

@@ -0,0 +1,35 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+
+TARGET = 'discourse_effectiveness'
+TEXT = "discourse_text"
+train_df = pd.read_csv("data/raw_data/train.csv")
+test_df = pd.read_csv("data/raw_data/test.csv")
+
+
+"""
+train_df[TARGET].value_counts(ascending=True).plot.barh()
+plt.title("Frequency of Classes")
+plt.show()
+
+train_df['discourse_type'].value_counts(ascending=True).plot.barh()
+plt.title("Frequency of discourse_type")
+plt.show()
+
+train_df["Words Per text"] = train_df[TEXT].str.split().apply(len)
+train_df.boxplot("Words Per text", by=TARGET, grid=False, showfliers=False,
+           color="black")
+plt.suptitle("")
+plt.xlabel("")
+plt.show()
+"""
+
+
+model_ckpt = "distilbert-base-uncased"
+tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+tokenizer.model_max_length
+pass