| | --- |
| | license: apache-2.0 |
| | datasets: |
| | - MarkProMaster229/toxic_dialogue |
| | - MarkProMaster229/synthetic_classification |
| | language: |
| | - ru |
| | pipeline_tag: text-classification |
| | tags: |
| | - toxic |
| | - custom |
| | - transformer |
| | - text-classification |
| | --- |
| | |
| | ## Model Details |
| | - Architecture: Custom Transformer |
| | - Number of parameters: 33,791,235 (~33.8M) |
| | - Num layers: 4 |
| | - Num attention heads: 8 |
| | - Hidden size: 256 |
| |
|
| | ## Using |
| | ```python |
| | import torch |
| | import torch.nn as nn |
| | from transformers import PreTrainedTokenizerFast |
| | from huggingface_hub import hf_hub_download |
| | |
| | repo_id = "MarkProMaster229/ClassificationSmall" |
| | |
| | weights_path = hf_hub_download(repo_id=repo_id, filename="model_weights.pth") |
| | tokenizer_path = hf_hub_download(repo_id=repo_id, filename="tokenizer.json") |
| | vocab_path = hf_hub_download(repo_id=repo_id, filename="vocab.txt") |
| | |
| | class TransformerBlock(nn.Module): |
| | def __init__(self, sizeVector=256, numHeads=8, dropout=0.5): |
| | super().__init__() |
| | self.ln1 = nn.LayerNorm(sizeVector) |
| | self.attn = nn.MultiheadAttention(sizeVector, numHeads, batch_first=True) |
| | self.dropout_attn = nn.Dropout(dropout) |
| | self.ln2 = nn.LayerNorm(sizeVector) |
| | self.ff = nn.Sequential( |
| | nn.Linear(sizeVector, sizeVector*4), |
| | nn.GELU(), |
| | nn.Linear(sizeVector*4, sizeVector), |
| | nn.Dropout(dropout) |
| | ) |
| | |
| | def forward(self, x, attention_mask=None): |
| | key_padding_mask = ~attention_mask.bool() if attention_mask is not None else None |
| | h = self.ln1(x) |
| | attn_out, _ = self.attn(h, h, h, key_padding_mask=key_padding_mask) |
| | x = x + self.dropout_attn(attn_out) |
| | x = x + self.ff(self.ln2(x)) |
| | return x |
| | |
| | class TransformerRun(nn.Module): |
| | def __init__(self, vocabSize=120000, maxLen=100, sizeVector=256, numBlocks=4, numHeads=8, numClasses=3, dropout=0.5): |
| | super().__init__() |
| | self.token_emb = nn.Embedding(vocabSize, sizeVector) |
| | self.pos_emb = nn.Embedding(maxLen, sizeVector) |
| | self.layers = nn.ModuleList([ |
| | TransformerBlock(sizeVector=sizeVector, numHeads=numHeads, dropout=dropout) |
| | for _ in range(numBlocks) |
| | ]) |
| | self.dropout = nn.Dropout(dropout) |
| | self.ln = nn.LayerNorm(sizeVector*2) |
| | self.classifier = nn.Linear(sizeVector*2, numClasses) |
| | |
| | def forward(self, x, attention_mask=None): |
| | B, T = x.shape |
| | tok = self.token_emb(x) |
| | pos = self.pos_emb(torch.arange(T, device=x.device).unsqueeze(0).expand(B, T)) |
| | h = tok + pos |
| | |
| | for layer in self.layers: |
| | h = layer(h, attention_mask) |
| | |
| | cls_token = h[:,0,:] |
| | mean_pool = h.mean(dim=1) |
| | combined = torch.cat([cls_token, mean_pool], dim=1) |
| | combined = self.ln(self.dropout(combined)) |
| | logits = self.classifier(combined) |
| | return logits |
| | |
| | config_dict = { |
| | 'vocabSize': 119547, |
| | 'maxLong': 100, |
| | 'sizeVector': 256, |
| | 'numLayers': 4, |
| | 'numHeads': 8, |
| | 'numClasses': 3 |
| | } |
| | |
| | model = TransformerRun( |
| | vocabSize=config_dict['vocabSize'], |
| | maxLen=config_dict['maxLong'], |
| | sizeVector=config_dict['sizeVector'], |
| | numBlocks=config_dict['numLayers'], |
| | numHeads=config_dict['numHeads'], |
| | numClasses=config_dict['numClasses'], |
| | dropout=0.1 |
| | ) |
| | |
| | state_dict = torch.load(weights_path, map_location="cpu") |
| | model.load_state_dict(state_dict) |
| | model.eval() |
| | |
| | tokenizer = PreTrainedTokenizerFast( |
| | tokenizer_file=tokenizer_path, |
| | vocab_file=vocab_path |
| | ) |
| | |
| | label_map = { |
| | 0: "positive", |
| | 1: "negative", |
| | 2: "neutral" |
| | } |
| | |
| | texts = [ |
| | "Я люблю тебя", |
| | "Мне совсем не понравился этот фильм", |
| | "Кличка моей кошки - Ирис" |
| | ] |
| | |
| | for text in texts: |
| | inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=config_dict['maxLong']) |
| | with torch.no_grad(): |
| | logits = model(inputs['input_ids']) |
| | pred_idx = torch.argmax(logits, dim=1).item() |
| | pred_label = label_map[pred_idx] |
| | print(f"text: {text}") |
| | print(f"class: {pred_label} ({pred_idx})") |
| | ``` |
| |
|
| | ```python |
| | text: Я люблю тебя |
| | class: positive (0) |
| | text: Мне совсем не понравился этот фильм |
| | class: negative (1) |
| | text: Кличка моей кошки - Ирис |
| | class: neutral (2) |
| | ``` |