Update file tokenizer.h

c_tokenizer/tokenizer.h

@@ -35,17 +35,19 @@ struct Tokenizer {
 
 static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _binary_tokenizer_bin_start) / 3; }
 
+/*
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <stdio.h>
+*/
 
 static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
 	
-	for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
-		printf("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
+	//for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
+	//	printf("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
 
-	exit(0);
+	//exit(0);
 
 	for (uint16_t i = prev; i < tokenizer->get_vocab_size(); ++i)
 		if (tokenizer->vocab[i].byte == byte && tokenizer->vocab[i].prev == prev)