Update 3 files

- /trainer.py
- /trainer.cli.py
- /tokenizer.py

tokenizer.py

@@ -145,4 +145,4 @@ class Tokenizer:
 
 
     def c_encode(self, text): #TODO: Implement
-        return []
+        return [1, 2, 3, 4]

trainer.cli.py

@@ -31,8 +31,12 @@ if __name__ == '__main__':
     tokenizer.train(dataset.text, max_length=config.tokenizer.max_length)
     ids = tokenizer.c_encode(dataset.text)
     
+
     dataset += ids
-    dataset.batch(ids)
+    #dataset.batch(ids)
+
+    print(f"dataset ids: {dataset.ids}")
+
 
     trainer = Trainer(config)
     trainer.train(dataset)

trainer.py

@@ -11,4 +11,7 @@ class Trainer:
         #self.wandb = Wandb(config.wandb)
         
         self.model = Model(config.model)
-        
+        
+
+    def train(self, dataset): # TODO: Implement
+        pass