Upload tokenizer

tokenizer.py

@@ -88,6 +88,7 @@ class BinnedOmicTokenizer(PreTrainedTokenizer):
             gene_expr = gene_expr / self.normalization_factor
 
         token_ids = np.digitize(gene_expr, self.bin_edges).astype(int)
+        token_ids = np.clip(token_ids, 0, self.n_expressions_bins - 1)
         token_ids[gene_expr == 0.0] = 0
 
         if self.prepend_cls_token:

tokenizer_config.json

@@ -1,32 +1,13 @@
 {
-  "tokenizer_class": "MOJOTokenizer",
-  "n_expressions_bins": {
-    "rnaseq": 64,
-    "methylation": 64
-  },
-  "min_omic_value": {
-    "rnaseq": 0.0,
-    "methylation": 0.006863548701544
-  },
-  "max_omic_value": {
-    "rnaseq": 1.0,
-    "methylation": 0.992831582796998
-  },
-  "use_max_normalization": {
-    "rnaseq": true,
-    "methylation": false
-  },
-  "normalization_factor": {
-    "rnaseq": 5.52786861525666,
-    "methylation": 1.0
-  },
-  "prepend_cls_token": false,
-  "fixed_sequence_length": 17152,
-  "unpadded_length": 17116,
+  "added_tokens_decoder": {},
   "auto_map": {
     "AutoTokenizer": [
       "tokenizer.MOJOTokenizer",
       null
     ]
-  }
-}
+  },
+  "clean_up_tokenization_spaces": false,
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "MOJOTokenizer"
+}