Upload 13 files

README.md

@@ -1,3 +1,73 @@
 ---
-license: apache-2.0
+license: mit
+base_model: microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+- f1
+model-index:
+- name: test
+  results: []
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# test
+
+This model is a fine-tuned version of [microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract](https://huggingface.co/microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6886
+- Accuracy: 0.8143
+- F1: [0.92816572 0.56028369 0.1        0.2633452 ]
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 32
+- eval_batch_size: 64
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 10.0
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Accuracy | F1                                            |
+|:-------------:|:-----:|:----:|:---------------:|:--------:|:---------------------------------------------:|
+| No log        | 1.0   | 37   | 0.4891          | 0.8235   | [0.91702786 0.33333333 0.         0.10837438] |
+| No log        | 2.0   | 74   | 0.4762          | 0.8321   | [0.93139159 0.48466258 0.         0.22857143] |
+| No log        | 3.0   | 111  | 0.5084          | 0.8208   | [0.92995725 0.44887781 0.         0.19266055] |
+| No log        | 4.0   | 148  | 0.5519          | 0.8105   | [0.92421691 0.44444444 0.06557377 0.30769231] |
+| No log        | 5.0   | 185  | 0.5805          | 0.8294   | [0.93531353 0.52336449 0.09345794 0.27131783] |
+| No log        | 6.0   | 222  | 0.6778          | 0.7955   | [0.91344509 0.55305466 0.15463918 0.29166667] |
+| No log        | 7.0   | 259  | 0.6407          | 0.8213   | [0.93298292 0.51383399 0.10191083 0.2519084 ] |
+| No log        | 8.0   | 296  | 0.6639          | 0.8272   | [0.9326288  0.55052265 0.18181818 0.26271186] |
+| No log        | 9.0   | 333  | 0.6863          | 0.8192   | [0.93071286 0.55830389 0.11042945 0.2761194 ] |
+| No log        | 10.0  | 370  | 0.6886          | 0.8143   | [0.92816572 0.56028369 0.1        0.2633452 ] |
+
+
+### Framework versions
+
+- Transformers 4.37.2
+- Pytorch 2.2.0+cu121
+- Datasets 2.17.0
+- Tokenizers 0.15.2

added_tokens.json

@@ -0,0 +1,10 @@
+{
+  "</a1>": 28898,
+  "</a2>": 28900,
+  "</e>": 28896,
+  "<a1>": 28897,
+  "<a2>": 28899,
+  "<cr>": 28901,
+  "<e>": 28895,
+  "<neg>": 28902
+}

all_results.json

@@ -0,0 +1,15 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.81431645154953,
+    "eval_f1": "[0.92816572 0.56028369 0.1        0.2633452 ]",
+    "eval_loss": 0.6886363625526428,
+    "eval_runtime": 7.7747,
+    "eval_samples": 1858,
+    "eval_samples_per_second": 238.981,
+    "eval_steps_per_second": 3.859,
+    "train_loss": 0.21409231649862753,
+    "train_runtime": 515.9059,
+    "train_samples": 2338,
+    "train_samples_per_second": 45.318,
+    "train_steps_per_second": 0.717
+}

config.json

@@ -0,0 +1,39 @@
+
+{
+  "_name_or_path": "microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "ACTUAL",
+    "1": "GENERIC",
+    "2": "HEDGED",
+    "3": "HYPOTHETICAL"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "ACTUAL": 0,
+    "GENERIC": 1,
+    "HEDGED": 2,
+    "HYPOTHETICAL": 3
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 28903
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.81431645154953,
+    "eval_f1": "[0.92816572 0.56028369 0.1        0.2633452 ]",
+    "eval_loss": 0.6886363625526428,
+    "eval_runtime": 7.7747,
+    "eval_samples": 1858,
+    "eval_samples_per_second": 238.981,
+    "eval_steps_per_second": 3.859
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:370ff03698c998c3a98bfc5eb64053f4046eab460b116a97207506145bbec52c
+size 432991216

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "additional_special_tokens": [
+    "<e>",
+    "</e>",
+    "<a1>",
+    "</a1>",
+    "<a2>",
+    "</a2>",
+    "<cr>",
+    "<neg>"
+  ],
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,132 @@
+{
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28895": {
+      "content": "<e>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28896": {
+      "content": "</e>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28897": {
+      "content": "<a1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28898": {
+      "content": "</a1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28899": {
+      "content": "<a2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28900": {
+      "content": "</a2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28901": {
+      "content": "<cr>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "28902": {
+      "content": "<neg>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<e>",
+    "</e>",
+    "<a1>",
+    "</a1>",
+    "<a2>",
+    "</a2>",
+    "<cr>",
+    "<neg>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 10.0,
+    "train_loss": 0.21409231649862753,
+    "train_runtime": 515.9059,
+    "train_samples": 2338,
+    "train_samples_per_second": 45.318,
+    "train_steps_per_second": 0.717
+}

trainer_state.json

@@ -0,0 +1,130 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 370,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.8234661221504211,
+      "eval_f1": "[0.91702786 0.33333333 0.         0.10837438]",
+      "eval_loss": 0.48905953764915466,
+      "eval_runtime": 8.4871,
+      "eval_samples_per_second": 218.92,
+      "eval_steps_per_second": 3.535,
+      "step": 37
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.8320775032043457,
+      "eval_f1": "[0.93139159 0.48466258 0.         0.22857143]",
+      "eval_loss": 0.47622746229171753,
+      "eval_runtime": 8.4411,
+      "eval_samples_per_second": 220.115,
+      "eval_steps_per_second": 3.554,
+      "step": 74
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.820775032043457,
+      "eval_f1": "[0.92995725 0.44887781 0.         0.19266055]",
+      "eval_loss": 0.5084477066993713,
+      "eval_runtime": 8.3479,
+      "eval_samples_per_second": 222.571,
+      "eval_steps_per_second": 3.594,
+      "step": 111
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.8105489611625671,
+      "eval_f1": "[0.92421691 0.44444444 0.06557377 0.30769231]",
+      "eval_loss": 0.5519338846206665,
+      "eval_runtime": 8.1137,
+      "eval_samples_per_second": 228.996,
+      "eval_steps_per_second": 3.697,
+      "step": 148
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.8293864130973816,
+      "eval_f1": "[0.93531353 0.52336449 0.09345794 0.27131783]",
+      "eval_loss": 0.5805315971374512,
+      "eval_runtime": 8.8001,
+      "eval_samples_per_second": 211.135,
+      "eval_steps_per_second": 3.409,
+      "step": 185
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.7954789996147156,
+      "eval_f1": "[0.91344509 0.55305466 0.15463918 0.29166667]",
+      "eval_loss": 0.6778322458267212,
+      "eval_runtime": 7.8496,
+      "eval_samples_per_second": 236.701,
+      "eval_steps_per_second": 3.822,
+      "step": 222
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.8213132619857788,
+      "eval_f1": "[0.93298292 0.51383399 0.10191083 0.2519084 ]",
+      "eval_loss": 0.6407278776168823,
+      "eval_runtime": 8.767,
+      "eval_samples_per_second": 211.931,
+      "eval_steps_per_second": 3.422,
+      "step": 259
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.827233612537384,
+      "eval_f1": "[0.9326288  0.55052265 0.18181818 0.26271186]",
+      "eval_loss": 0.6639025807380676,
+      "eval_runtime": 8.0873,
+      "eval_samples_per_second": 229.742,
+      "eval_steps_per_second": 3.709,
+      "step": 296
+    },
+    {
+      "epoch": 9.0,
+      "eval_accuracy": 0.8191604018211365,
+      "eval_f1": "[0.93071286 0.55830389 0.11042945 0.2761194 ]",
+      "eval_loss": 0.6863259077072144,
+      "eval_runtime": 7.9866,
+      "eval_samples_per_second": 232.64,
+      "eval_steps_per_second": 3.756,
+      "step": 333
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.81431645154953,
+      "eval_f1": "[0.92816572 0.56028369 0.1        0.2633452 ]",
+      "eval_loss": 0.6886363625526428,
+      "eval_runtime": 7.7254,
+      "eval_samples_per_second": 240.504,
+      "eval_steps_per_second": 3.883,
+      "step": 370
+    },
+    {
+      "epoch": 10.0,
+      "step": 370,
+      "total_flos": 1537911734661120.0,
+      "train_loss": 0.21409231649862753,
+      "train_runtime": 515.9059,
+      "train_samples_per_second": 45.318,
+      "train_steps_per_second": 0.717
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 370,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "total_flos": 1537911734661120.0,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d1aab45051285d4d86e8ffbcab852ca26f8e59e13b6b06c115c64cb15b0664a
+size 4664