Update README.md
Browse files
README.md
CHANGED
|
@@ -10,6 +10,19 @@ from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassifica
|
|
| 10 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 11 |
identity_model = AutoModelForSequenceClassification.from_pretrained("Mridul2003/identity-hate-detector").to(device)
|
| 12 |
identity_tokenizer = AutoTokenizer.from_pretrained("Mridul2003/identity-hate-detector")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
```
|
| 14 |
|
| 15 |
# Offensive Language Classifier (Fine-Tuned on Custom Dataset)
|
|
|
|
| 10 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 11 |
identity_model = AutoModelForSequenceClassification.from_pretrained("Mridul2003/identity-hate-detector").to(device)
|
| 12 |
identity_tokenizer = AutoTokenizer.from_pretrained("Mridul2003/identity-hate-detector")
|
| 13 |
+
identity_inputs = identity_tokenizer(final_text, return_tensors="pt", padding=True, truncation=True)
|
| 14 |
+
if 'token_type_ids' in identity_inputs:
|
| 15 |
+
del identity_inputs['token_type_ids']
|
| 16 |
+
identity_inputs = {k: v.to(device) for k, v in identity_inputs.items()}
|
| 17 |
+
with torch.no_grad():
|
| 18 |
+
identity_outputs = identity_model(**identity_inputs)
|
| 19 |
+
identity_probs = torch.sigmoid(identity_outputs.logits)
|
| 20 |
+
identity_prob = identity_probs[0][1].item()
|
| 21 |
+
not_identity_prob = identity_probs[0][0].item()
|
| 22 |
+
|
| 23 |
+
results["identity_hate_custom"] = identity_prob
|
| 24 |
+
results["not_identity_hate_custom"] = not_identity_prob
|
| 25 |
+
|
| 26 |
```
|
| 27 |
|
| 28 |
# Offensive Language Classifier (Fine-Tuned on Custom Dataset)
|