Update refine_paraphrases.py

refine_paraphrases.py

@@ -3,10 +3,12 @@ import pandas as pd
 from paraphraser import paraphrase_comment
 from metrics import compute_reward_scores
 from model_loader import paraphraser_model
+from datasets import load_dataset
+import os
 
 # Configuration
-DATA_PATH = "toxic-comment-classifier_rlhf/refined_paraphrases.csv"
-OUTPUT_PATH = "toxic-comment-classifier_rlhf/iterated_paraphrases.csv"
+DATA_PATH = "JanviMl/toxi_refined_paraphrases"
+OUTPUT_PATH = "iterated_paraphrases.csv"
 MAX_ITERATIONS = 3
 TARGET_SCORES = {
     "empathy": 0.9,
@@ -83,8 +85,12 @@ def refine_paraphrase(row: pd.Series) -> tuple:
     return current_paraphrase, current_scores, "; ".join(reasoning)
 
 def main():
-    # Load dataset
-    df = pd.read_csv(DATA_PATH)
+    # Load dataset from Hugging Face Hub
+    try:
+        df = load_dataset(DATA_PATH, split="train").to_pandas()
+    except Exception as e:
+        print(f"Error loading dataset: {str(e)}")
+        return
     
     # Process each row
     results = []
@@ -105,11 +111,20 @@ def main():
             "Iteration_Reasoning": reasoning
         }
         results.append(result)
-
-    # Save results
+    
+    # Save results locally
     result_df = pd.DataFrame(results)
     result_df.to_csv(OUTPUT_PATH, index=False)
     print(f"Refinement complete. Results saved to {OUTPUT_PATH}")
+    
+    # Push to Hugging Face Hub
+    try:
+        from datasets import Dataset
+        dataset = Dataset.from_pandas(result_df)
+        dataset.push_to_hub("JanviMl/toxi_iterated_paraphrases", token=os.getenv("HF_TOKEN"))
+        print("Pushed to Hugging Face Hub: JanviMl/toxi_iterated_paraphrases")
+    except Exception as e:
+        print(f"Error pushing to Hub: {str(e)}")
 
 if __name__ == "__main__":
     main()