Delete tokenizer.py

tokenizer.py

@@ -1,154 +0,0 @@
-import json
-import os
-from typing import List, Optional, Union
-
-import numpy as np
-import torch
-from transformers import PretrainedConfig, PreTrainedTokenizer
-
-
-class BinnedOmicTokenizerConfig(PretrainedConfig):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.n_expressions_bins = kwargs.get("n_expressions_bins", 64)
-        self.min_omic_value = kwargs.get("min_omic_value", 0.0)
-        self.max_omic_value = kwargs.get("max_omic_value", 1.0)
-        self.use_max_normalization = kwargs.get("use_max_normalization", True)
-        self.normalization_factor = kwargs.get(
-            "normalization_factor", 5.547176906585117
-        )
-        self.prepend_cls_token = kwargs.get("prepend_cls_token", False)
-        self.fixed_sequence_length = kwargs.get("fixed_sequence_length", None)
-        self.unpadded_length = kwargs.get("unpadded_length", None)
-
-
-class BinnedOmicTokenizer(PreTrainedTokenizer):
-    def __init__(
-        self,
-        n_expressions_bins: int = 64,
-        min_omic_value: float = 0.0,
-        max_omic_value: float = 1.0,
-        use_max_normalization: bool = True,
-        normalization_factor: float = 1.0,
-        prepend_cls_token: bool = False,
-        fixed_sequence_length: Optional[int] = None,
-        unpadded_length: Optional[int] = None,
-        **kwargs,
-    ):
-        bin_tokens = [str(i) for i in range(n_expressions_bins)]
-        special_tokens = ["<pad>", "<mask>", "<cls>"]
-
-        vocab = {tok: i for i, tok in enumerate(bin_tokens)}
-        offset = len(vocab)
-        for i, tok in enumerate(special_tokens):
-            vocab[tok] = offset + i
-
-        ids_to_tokens = {i: tok for tok, i in vocab.items()}
-
-        self.vocab = vocab
-        self.ids_to_tokens = ids_to_tokens
-
-        self.n_expressions_bins = n_expressions_bins
-        self.min_omic_value = min_omic_value
-        self.max_omic_value = max_omic_value
-        self.use_max_normalization = use_max_normalization
-        self.normalization_factor = normalization_factor
-        self.prepend_cls_token = prepend_cls_token
-        self.fixed_sequence_length = fixed_sequence_length
-        self.unpadded_length = unpadded_length
-
-        self.bin_edges = np.linspace(min_omic_value, max_omic_value, n_expressions_bins)
-
-        self.pad_token = "<pad>"
-        self.mask_token = "<mask>"
-        self.cls_token = "<cls>"
-
-        super().__init__(**kwargs)
-
-    def _convert_token_to_id(self, token: str) -> int:
-        return self.vocab.get(token, self.vocab[self.unk_token])
-
-    def _convert_id_to_token(self, index: int) -> str:
-        return self.ids_to_tokens.get(index, self.unk_token)
-
-    def get_vocab(self) -> dict:
-        return self.vocab
-
-    def _tokenize(self, text, **kwargs):
-        raise NotImplementedError("Use `encode` or `batch_encode_plus` methods.")
-
-    def encode(
-        self,
-        gene_expr: Union[np.ndarray, List[float]],
-        pad_to_fixed_length: bool = False,
-        max_length: Optional[int] = None,
-        return_tensors: Optional[str] = None,
-        **kwargs,
-    ) -> Union[List[int], torch.Tensor]:
-        gene_expr = np.array(gene_expr)
-
-        if self.use_max_normalization:
-            gene_expr = gene_expr / self.normalization_factor
-
-        token_ids = np.digitize(gene_expr, self.bin_edges).astype(int)
-        token_ids[gene_expr == 0.0] = 0
-
-        if self.prepend_cls_token:
-            token_ids = np.concatenate([[self.cls_token_id], token_ids])
-
-        if pad_to_fixed_length:
-            current_max_length = self.fixed_sequence_length or max_length
-            if current_max_length is None:
-                raise ValueError("fixed_sequence_length or max_length must be set.")
-            pad_len = current_max_length - len(token_ids)
-            if pad_len > 0:
-                token_ids = np.concatenate([token_ids, [self.pad_token_id] * pad_len])
-            else:
-                token_ids = token_ids[:current_max_length]
-
-        if return_tensors == "pt":
-            return torch.tensor(token_ids).unsqueeze(0)
-        return token_ids.tolist()  # type: ignore
-
-    def batch_encode_plus(
-        self,
-        batch_gene_expr: Union[np.ndarray, List[np.ndarray]],
-        pad_to_fixed_length: bool = False,
-        max_length: Optional[int] = None,
-        return_tensors: Optional[str] = None,
-        **kwargs,
-    ):
-        if isinstance(batch_gene_expr, list):
-            batch_gene_expr = np.array(batch_gene_expr)
-
-        encoded = [
-            self.encode(
-                gene_expr,
-                pad_to_fixed_length=pad_to_fixed_length,
-                max_length=max_length,
-                return_tensors=None,
-                **kwargs,
-            )
-            for gene_expr in batch_gene_expr
-        ]
-
-        encoded = np.array(encoded, dtype=np.int64)
-
-        if return_tensors == "pt":
-            return {"input_ids": torch.tensor(encoded)}
-        return {"input_ids": encoded}
-
-    @property
-    def vocab_size(self) -> int:
-        return len(self.vocab)
-
-    def save_vocabulary(
-        self, save_directory: str, filename_prefix: Optional[str] = None
-    ):
-        vocab_file = os.path.join(
-            save_directory,
-            (filename_prefix + "-" if filename_prefix else "") + "vocab.json",
-        )
-        with open(vocab_file, "w") as f:
-            json.dump(self.vocab, f)
-        return (vocab_file,)