Update load_data for own dataset sync

load_data.py

@@ -10,7 +10,7 @@ from datasets import load_dataset, concatenate_datasets
 from argilla.listeners import listener
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
-HUB_DATASET_NAME = os.environ.get('HUB_DATASET_NAME')
+HUB_DATASET_NAME = "mserras/alpaca-es-hackaton"
 
 @listener(
     dataset="somos-alpaca-es", 
@@ -44,20 +44,23 @@ class LoadDatasets:
             print(e)
             old_ds = None
             
-        dataset = load_dataset("somosnlp/somos-clean-alpaca-es", split="train")
+        # dataset = load_dataset("somosnlp/somos-clean-alpaca-es", split="train")
     
         
-        if old_ds:
-            print("Concatenating datasets")
-            dataset = concatenate_datasets([dataset, old_ds])
-            print("Concatenated dataset is:")
-            print(dataset)
+        # if old_ds:
+        #     print("Concatenating datasets")
+        #     dataset = concatenate_datasets([dataset, old_ds])
+        #     print("Concatenated dataset is:")
+        #     print(dataset)
             
-        dataset = dataset.remove_columns("metrics")
+        # dataset = dataset.remove_columns("metrics")
+        if not old_ds:
+            return
+        dataset = old_ds
         records = rg.DatasetForTextClassification.from_datasets(dataset)
 
         settings = rg.TextClassificationSettings(
-            label_schema=["BAD INSTRUCTION", "BAD INPUT", "BAD OUTPUT", "INAPPROPRIATE", "BIASED", "ALL GOOD"]
+            label_schema=["BAD INSTRUCTION", "BAD INPUT", "BAD OUTPUT", "INAPPROPRIATE", "BIASED", "ALL GOOD", "HALLUCINATION", "UNPROCESSABLE"]
         )
         rg.configure_dataset(name="somos-alpaca-es", settings=settings, workspace="team")