{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# CamemBERT fine-tuning\n", "\n", "Because of dependency conflicts, we will be fine-tuning the model here and then loading it and evaluating in [deepl_ner.ipynb](./deepl_ner.ipynb).\n" ] }, { "cell_type": "code", "execution_count": 86, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: transformers in ./venv/lib/python3.12/site-packages (4.47.1)\n", "Requirement already satisfied: tf-keras in ./venv/lib/python3.12/site-packages (2.18.0)\n", "Requirement already satisfied: focal-loss in ./venv/lib/python3.12/site-packages (0.0.7)\n", "Requirement already satisfied: filelock in ./venv/lib/python3.12/site-packages (from transformers) (3.16.1)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in ./venv/lib/python3.12/site-packages (from transformers) (0.26.5)\n", "Requirement already satisfied: numpy>=1.17 in ./venv/lib/python3.12/site-packages (from transformers) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.12/site-packages (from transformers) (24.2)\n", "Requirement already satisfied: pyyaml>=5.1 in ./venv/lib/python3.12/site-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in ./venv/lib/python3.12/site-packages (from transformers) (2024.11.6)\n", "Requirement already satisfied: requests in ./venv/lib/python3.12/site-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: tokenizers<0.22,>=0.21 in ./venv/lib/python3.12/site-packages (from transformers) (0.21.0)\n", "Requirement already satisfied: safetensors>=0.4.1 in ./venv/lib/python3.12/site-packages (from transformers) (0.4.5)\n", "Requirement already satisfied: tqdm>=4.27 in ./venv/lib/python3.12/site-packages (from transformers) (4.66.5)\n", "Requirement already satisfied: tensorflow<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tf-keras) (2.18.0)\n", "Requirement already satisfied: fsspec>=2023.5.0 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (2024.10.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (4.12.2)\n", "Requirement already satisfied: absl-py>=1.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.1.0)\n", "Requirement already satisfied: astunparse>=1.6.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.6.3)\n", "Requirement already satisfied: flatbuffers>=24.3.25 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (24.3.25)\n", "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.6.0)\n", "Requirement already satisfied: google-pasta>=0.1.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.2.0)\n", "Requirement already satisfied: libclang>=13.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (18.1.1)\n", "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.4.0)\n", "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (4.25.5)\n", "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (75.6.0)\n", "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.17.0)\n", "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.5.0)\n", "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.17.0)\n", "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.68.1)\n", "Requirement already satisfied: tensorboard<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n", "Requirement already satisfied: keras>=3.5.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.7.0)\n", "Requirement already satisfied: h5py>=3.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.12.1)\n", "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.4.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.4.0)\n", "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2024.8.30)\n", "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf-keras) (0.45.1)\n", "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (13.9.4)\n", "Requirement already satisfied: namex in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.0.8)\n", "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.13.1)\n", "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.7)\n", "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (0.7.2)\n", "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.1.3)\n", "Requirement already satisfied: MarkupSafe>=2.1.1 in ./venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.0.2)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (3.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n", "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.1.2)\n" ] } ], "source": [ "!pip install --upgrade transformers tf-keras focal-loss" ] }, { "cell_type": "code", "execution_count": 87, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "os.environ[\"TF_USE_LEGACY_KERAS\"] = \"1\"" ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [], "source": [ "import tensorflow as tf" ] }, { "cell_type": "code", "execution_count": 89, "metadata": {}, "outputs": [], "source": [ "from app.travel_resolver.libs.nlp import data_processing as dp\n", "\n", "sentences, labels, vocab, unique_labels = dp.from_bio_file_to_examples(\n", " \"./data/bio/fr.bio/10k_train_small_samples.bio\"\n", ")\n", "\n", "# To avoid overfitting the model on sentences that don't have any labels\n", "lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n", " \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n", ")\n", "\n", "long_sentences, long_labels, _, __ = dp.from_bio_file_to_examples(\n", " \"./data/bio/fr.bio/1k_train_large_samples.bio\"\n", ")" ] }, { "cell_type": "code", "execution_count": 90, "metadata": {}, "outputs": [], "source": [ "sentences = sentences + lambda_sentences + long_sentences\n", "labels = labels + lambda_labels + long_labels" ] }, { "cell_type": "code", "execution_count": 91, "metadata": {}, "outputs": [], "source": [ "import app.travel_resolver.libs.nlp.data_processing as dp\n", "\n", "processed_sentences, processed_labels = dp.process_sentences_and_labels(\n", " sentences, labels, return_tokens=True, stemming=False\n", ")" ] }, { "cell_type": "code", "execution_count": 92, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", " This variable will control the maximum length of the sentence \n", " as well as the embedding size\n", "\"\"\"\n", "\n", "MAX_LEN = 150" ] }, { "cell_type": "code", "execution_count": 93, "metadata": {}, "outputs": [], "source": [ "padded_labels = tf.keras.preprocessing.sequence.pad_sequences(\n", " processed_labels, maxlen=MAX_LEN, padding=\"post\"\n", ")" ] }, { "cell_type": "code", "execution_count": 94, "metadata": {}, "outputs": [], "source": [ "from transformers import TFAutoModelForTokenClassification, CamembertTokenizerFast\n", "import numpy as np\n", "\n", "tokenizer = CamembertTokenizerFast.from_pretrained(\"cmarkea/distilcamembert-base\")" ] }, { "cell_type": "code", "execution_count": 95, "metadata": {}, "outputs": [], "source": [ "tokenized_sentences = tokenizer(\n", " processed_sentences,\n", " is_split_into_words=True,\n", " return_offsets_mapping=True,\n", " truncation=True,\n", " padding=\"max_length\",\n", " max_length=MAX_LEN,\n", ")" ] }, { "cell_type": "code", "execution_count": 96, "metadata": {}, "outputs": [], "source": [ "def align_labels_with_tokens(encodings, labels):\n", " \"\"\"\n", " Aligns the labels to match the tokenized outputs.\n", "\n", " Args:\n", " encodings (BatchEncoding): Tokenized outputs from the Hugging Face tokenizer (must use a fast tokenizer).\n", " labels (List[List[int]]): Original labels for each sentence before tokenization. Each inner list corresponds to one sentence.\n", "\n", " Returns:\n", " List[List[int]]: Aligned labels, where each inner list corresponds to the aligned labels for the tokenized sentence.\n", " Special tokens and padding are assigned a value of -100.\n", " \"\"\"\n", " adapted_labels = []\n", "\n", " for i, label in enumerate(labels):\n", " word_ids = encodings.word_ids(\n", " batch_index=i\n", " ) # Get word IDs for the i-th sentence\n", " aligned_labels = []\n", " previous_word_id = None\n", "\n", " for word_id in word_ids:\n", " if word_id is None:\n", " # Special tokens (e.g., [CLS], [SEP], or padding)\n", " aligned_labels.append(-100)\n", " elif word_id != previous_word_id:\n", " # New word\n", " aligned_labels.append(label[word_id])\n", " else:\n", " # Subword token (same word)\n", " aligned_labels.append(\n", " label[word_id]\n", " ) # Or append -100 to ignore subwords\n", " previous_word_id = word_id\n", "\n", " adapted_labels.append(aligned_labels)\n", "\n", " return adapted_labels" ] }, { "cell_type": "code", "execution_count": 97, "metadata": {}, "outputs": [], "source": [ "readapted_labels = align_labels_with_tokens(tokenized_sentences, padded_labels)" ] }, { "cell_type": "code", "execution_count": 98, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "\n", "(\n", " train_input_ids,\n", " test_input_ids,\n", " train_attention_masks,\n", " test_attention_masks,\n", " train_labels,\n", " test_labels,\n", ") = train_test_split(\n", " tokenized_sentences[\"input_ids\"],\n", " tokenized_sentences[\"attention_mask\"],\n", " readapted_labels,\n", " test_size=0.2,\n", ")" ] }, { "cell_type": "code", "execution_count": 99, "metadata": {}, "outputs": [], "source": [ "train_dataset = tf.data.Dataset.from_tensor_slices(\n", " (\n", " {\n", " \"input_ids\": train_input_ids,\n", " \"attention_mask\": train_attention_masks,\n", " },\n", " train_labels,\n", " )\n", ")\n", "\n", "test_dataset = tf.data.Dataset.from_tensor_slices(\n", " (\n", " {\n", " \"input_ids\": test_input_ids,\n", " \"attention_mask\": test_attention_masks,\n", " },\n", " test_labels,\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": 100, "metadata": {}, "outputs": [], "source": [ "def entity_accuracy(y_true, y_pred):\n", " \"\"\"\n", " Calculate the accuracy based on the entities. Which mean that correct `O` tags will not be taken into account.\n", "\n", " Parameters:\n", " y_true (tensor): True labels.\n", " y_pred (tensor): Predicted logits.\n", "\n", " Returns:\n", " accuracy (tensor): Tag accuracy.\n", " \"\"\"\n", "\n", " y_true = tf.cast(y_true, tf.float32)\n", " # We ignore the padding and the O tag\n", " mask = y_true > 0\n", " mask = tf.cast(mask, tf.float32)\n", "\n", " y_pred_class = tf.math.argmax(y_pred, axis=-1)\n", " y_pred_class = tf.cast(y_pred_class, tf.float32)\n", "\n", " matches_true_pred = tf.equal(y_true, y_pred_class)\n", " matches_true_pred = tf.cast(matches_true_pred, tf.float32)\n", "\n", " matches_true_pred *= mask\n", "\n", " masked_acc = tf.reduce_sum(matches_true_pred) / tf.reduce_sum(mask)\n", "\n", " return masked_acc" ] }, { "cell_type": "code", "execution_count": 101, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFCamembertForTokenClassification: ['roberta.embeddings.position_ids']\n", "- This IS expected if you are initializing TFCamembertForTokenClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing TFCamembertForTokenClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n", "Some weights or buffers of the TF 2.0 model TFCamembertForTokenClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "from focal_loss import SparseCategoricalFocalLoss\n", "\n", "camembert = TFAutoModelForTokenClassification.from_pretrained(\n", " \"cmarkea/distilcamembert-base\", num_labels=len(unique_labels)\n", ")\n", "\n", "loss_func = SparseCategoricalFocalLoss(\n", " gamma=2, class_weight=[1, 10, 10], from_logits=True\n", ")\n", "\n", "camembert.compile(\n", " optimizer=tf.keras.optimizers.legacy.Adam(8e-4),\n", " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=[entity_accuracy],\n", ")" ] }, { "cell_type": "code", "execution_count": 102, "metadata": {}, "outputs": [], "source": [ "train_dataset = train_dataset.batch(64)\n", "test_dataset = test_dataset.batch(64)" ] }, { "cell_type": "code", "execution_count": 103, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/10\n" ] }, { "ename": "TypeError", "evalue": "in user code:\n\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1398, in train_function *\n return step_function(self, iterator)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1381, in step_function **\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1370, in run_step **\n outputs = model.train_step(data)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1672, in train_step\n y_pred = self(x, training=True)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py\", line 70, in error_handler\n raise e.with_traceback(filtered_tb) from None\n File \"/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py\", line 40, in tf__run_call_with_unpacked_inputs\n raise\n\n TypeError: Exception encountered when calling layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification).\n \n in user code:\n \n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1393, in run_call_with_unpacked_inputs *\n return func(self, **unpacked_inputs)\n \n TypeError: outer_factory..inner_factory..tf__call() got an unexpected keyword argument 'offset_mapping'\n \n \n Call arguments received by layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification):\n • input_ids={'input_ids': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'attention_mask': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'offset_mapping': 'tf.Tensor(shape=(None, 150, 2), dtype=int32)'}\n • attention_mask=None\n • token_type_ids=None\n • position_ids=None\n • head_mask=None\n • inputs_embeds=None\n • output_attentions=None\n • output_hidden_states=None\n • return_dict=None\n • labels=None\n • training=True\n", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[103], line 7\u001b[0m\n\u001b[1;32m 1\u001b[0m early_stopping \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mEarlyStopping(\n\u001b[1;32m 2\u001b[0m monitor\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mval_loss\u001b[39m\u001b[38;5;124m\"\u001b[39m, min_delta\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.001\u001b[39m, patience\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0\u001b[39m, restore_best_weights\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 3\u001b[0m )\n\u001b[1;32m 5\u001b[0m csv_logger \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mkeras\u001b[38;5;241m.\u001b[39mcallbacks\u001b[38;5;241m.\u001b[39mCSVLogger(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtraining.log\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 7\u001b[0m \u001b[43mcamembert\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mtrain_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidation_data\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtest_dataset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43mearly_stopping\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcsv_logger\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 12\u001b[0m \u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py:1229\u001b[0m, in \u001b[0;36mTFPreTrainedModel.fit\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1226\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(keras\u001b[38;5;241m.\u001b[39mModel\u001b[38;5;241m.\u001b[39mfit)\n\u001b[1;32m 1227\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 1228\u001b[0m args, kwargs \u001b[38;5;241m=\u001b[39m convert_batch_encoding(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m-> 1229\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback..error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 67\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 72\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n", "File \u001b[0;32m/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filelw03dryu.py:15\u001b[0m, in \u001b[0;36mouter_factory..inner_factory..tf__train_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 14\u001b[0m do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[1;32m 16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m 17\u001b[0m do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", "File \u001b[0;32m~/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py:1672\u001b[0m, in \u001b[0;36mTFPreTrainedModel.train_step\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 1670\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m(x, training\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, return_loss\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 1671\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1672\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraining\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m 1673\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_using_dummy_loss:\n\u001b[1;32m 1674\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompiled_loss(y_pred\u001b[38;5;241m.\u001b[39mloss, y_pred\u001b[38;5;241m.\u001b[39mloss, sample_weight, regularization_losses\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlosses)\n", "File \u001b[0;32m/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py:37\u001b[0m, in \u001b[0;36mouter_factory..inner_factory..tf__run_call_with_unpacked_inputs\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 36\u001b[0m do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m---> 37\u001b[0m retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(func), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m),), \u001b[38;5;28mdict\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mag__\u001b[38;5;241m.\u001b[39mld(unpacked_inputs)), fscope)\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m 39\u001b[0m do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n", "\u001b[0;31mTypeError\u001b[0m: in user code:\n\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1398, in train_function *\n return step_function(self, iterator)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1381, in step_function **\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/engine/training.py\", line 1370, in run_step **\n outputs = model.train_step(data)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1672, in train_step\n y_pred = self(x, training=True)\n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/tf_keras/src/utils/traceback_utils.py\", line 70, in error_handler\n raise e.with_traceback(filtered_tb) from None\n File \"/var/folders/3h/5n6s9rcj3sx0gpncsxbq_99m0000gn/T/__autograph_generated_filepc984rni.py\", line 40, in tf__run_call_with_unpacked_inputs\n raise\n\n TypeError: Exception encountered when calling layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification).\n \n in user code:\n \n File \"/Users/az-r-ow/Developer/TravelOrderResolver/venv/lib/python3.12/site-packages/transformers/modeling_tf_utils.py\", line 1393, in run_call_with_unpacked_inputs *\n return func(self, **unpacked_inputs)\n \n TypeError: outer_factory..inner_factory..tf__call() got an unexpected keyword argument 'offset_mapping'\n \n \n Call arguments received by layer 'tf_camembert_for_token_classification_5' (type TFCamembertForTokenClassification):\n • input_ids={'input_ids': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'attention_mask': 'tf.Tensor(shape=(None, 150), dtype=int32)', 'offset_mapping': 'tf.Tensor(shape=(None, 150, 2), dtype=int32)'}\n • attention_mask=None\n • token_type_ids=None\n • position_ids=None\n • head_mask=None\n • inputs_embeds=None\n • output_attentions=None\n • output_hidden_states=None\n • return_dict=None\n • labels=None\n • training=True\n" ] } ], "source": [ "early_stopping = tf.keras.callbacks.EarlyStopping(\n", " monitor=\"val_loss\", min_delta=0.001, patience=0, restore_best_weights=True\n", ")\n", "\n", "csv_logger = tf.keras.callbacks.CSVLogger(\"training.log\")\n", "\n", "camembert.fit(\n", " train_dataset,\n", " validation_data=test_dataset,\n", " epochs=10,\n", " callbacks=[early_stopping, csv_logger],\n", ")" ] }, { "cell_type": "code", "execution_count": 76, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 76, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from focal_loss import SparseCategoricalFocalLoss\n", "\n", "loss_func = SparseCategoricalFocalLoss(gamma=1)\n", "y_true = [0, 1, 2]\n", "y_pred = [[0.8, 0.1, 0.1], [0.2, 0.7, 0.1], [0.2, 0.2, 0.6]]\n", "loss_func(y_true, y_pred)" ] }, { "cell_type": "code", "execution_count": 79, "metadata": {}, "outputs": [], "source": [ "camembert.save_pretrained(\"./models/distilcamembert-base-ner-cross-entropy-11\")" ] }, { "cell_type": "code", "execution_count": 78, "metadata": {}, "outputs": [], "source": [ "# camembert.push_to_hub(\"CamemBERT-NER-Travel\")" ] } ], "metadata": { "kernelspec": { "display_name": "venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.8" } }, "nbformat": 4, "nbformat_minor": 2 }