feat(CamemBERT): Fine-tuned camembert model for NER

camemBERT_finetuning.ipynb

@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [
     {
@@ -19,6 +19,9 @@
      "output_type": "stream",
      "text": [
       "Requirement already satisfied: transformers in ./venv/lib/python3.12/site-packages (4.46.3)\n",
+      "Requirement already satisfied: tf-keras in ./venv/lib/python3.12/site-packages (2.18.0)\n",
+      "Collecting focal-loss\n",
+      "  Downloading focal_loss-0.0.7-py3-none-any.whl.metadata (5.1 kB)\n",
       "Requirement already satisfied: filelock in ./venv/lib/python3.12/site-packages (from transformers) (3.16.1)\n",
       "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in ./venv/lib/python3.12/site-packages (from transformers) (0.26.3)\n",
       "Requirement already satisfied: numpy>=1.17 in ./venv/lib/python3.12/site-packages (from transformers) (1.26.4)\n",
@@ -29,53 +32,44 @@
       "Requirement already satisfied: tokenizers<0.21,>=0.20 in ./venv/lib/python3.12/site-packages (from transformers) (0.20.3)\n",
       "Requirement already satisfied: safetensors>=0.4.1 in ./venv/lib/python3.12/site-packages (from transformers) (0.4.5)\n",
       "Requirement already satisfied: tqdm>=4.27 in ./venv/lib/python3.12/site-packages (from transformers) (4.66.5)\n",
+      "Requirement already satisfied: tensorflow<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tf-keras) (2.18.0)\n",
       "Requirement already satisfied: fsspec>=2023.5.0 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.10.0)\n",
       "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
+      "Requirement already satisfied: absl-py>=1.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.1.0)\n",
+      "Requirement already satisfied: astunparse>=1.6.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.6.3)\n",
+      "Requirement already satisfied: flatbuffers>=24.3.25 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (24.3.25)\n",
+      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.6.0)\n",
+      "Requirement already satisfied: google-pasta>=0.1.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.2.0)\n",
+      "Requirement already satisfied: libclang>=13.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (18.1.1)\n",
+      "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.4.0)\n",
+      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (4.25.5)\n",
+      "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (75.2.0)\n",
+      "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.16.0)\n",
+      "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.5.0)\n",
+      "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.16.0)\n",
+      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (1.67.0)\n",
+      "Requirement already satisfied: tensorboard<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n",
+      "Requirement already satisfied: keras>=3.5.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.7.0)\n",
+      "Requirement already satisfied: h5py>=3.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (3.12.1)\n",
+      "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf-keras) (0.4.1)\n",
       "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.4.0)\n",
       "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.10)\n",
       "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2.2.3)\n",
       "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2024.8.30)\n",
-      "\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
-      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
-      "Requirement already satisfied: tf_keras in ./venv/lib/python3.12/site-packages (2.18.0)\n",
-      "Requirement already satisfied: tensorflow<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tf_keras) (2.18.0)\n",
-      "Requirement already satisfied: absl-py>=1.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.1.0)\n",
-      "Requirement already satisfied: astunparse>=1.6.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.6.3)\n",
-      "Requirement already satisfied: flatbuffers>=24.3.25 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (24.3.25)\n",
-      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.6.0)\n",
-      "Requirement already satisfied: google-pasta>=0.1.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.2.0)\n",
-      "Requirement already satisfied: libclang>=13.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (18.1.1)\n",
-      "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.4.0)\n",
-      "Requirement already satisfied: packaging in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (24.1)\n",
-      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (4.25.5)\n",
-      "Requirement already satisfied: requests<3,>=2.21.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.32.3)\n",
-      "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (75.2.0)\n",
-      "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.16.0)\n",
-      "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.5.0)\n",
-      "Requirement already satisfied: typing-extensions>=3.6.6 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (4.12.2)\n",
-      "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.16.0)\n",
-      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.67.0)\n",
-      "Requirement already satisfied: tensorboard<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.18.0)\n",
-      "Requirement already satisfied: keras>=3.5.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.7.0)\n",
-      "Requirement already satisfied: numpy<2.1.0,>=1.26.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.26.4)\n",
-      "Requirement already satisfied: h5py>=3.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.12.1)\n",
-      "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.4.1)\n",
-      "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf_keras) (0.44.0)\n",
-      "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (13.9.2)\n",
-      "Requirement already satisfied: namex in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.0.8)\n",
-      "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.13.0)\n",
-      "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (3.4.0)\n",
-      "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (3.10)\n",
-      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (2.2.3)\n",
-      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (2024.8.30)\n",
-      "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.7)\n",
-      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (0.7.2)\n",
-      "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.0.4)\n",
-      "Requirement already satisfied: MarkupSafe>=2.1.1 in ./venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.0.2)\n",
-      "Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (3.0.0)\n",
-      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (2.18.0)\n",
-      "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.1.2)\n",
+      "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf-keras) (0.44.0)\n",
+      "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (13.9.2)\n",
+      "Requirement already satisfied: namex in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.0.8)\n",
+      "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.13.0)\n",
+      "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.7)\n",
+      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (0.7.2)\n",
+      "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.0.4)\n",
+      "Requirement already satisfied: MarkupSafe>=2.1.1 in ./venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf-keras) (3.0.2)\n",
+      "Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (3.0.0)\n",
+      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (2.18.0)\n",
+      "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf-keras) (0.1.2)\n",
+      "Downloading focal_loss-0.0.7-py3-none-any.whl (19 kB)\n",
+      "Installing collected packages: focal-loss\n",
+      "Successfully installed focal-loss-0.0.7\n",
       "\n",
       "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
       "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
@@ -83,12 +77,12 @@
     }
    ],
    "source": [
-    "!pip install --upgrade transformers tf-keras numpy sentencepiece"
+    "!pip install --upgrade transformers tf-keras focal-loss"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -99,7 +93,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -108,19 +102,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 23,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[nltk_data] Downloading package punkt_tab to /Users/az-r-\n",
-      "[nltk_data]     ow/nltk_data...\n",
-      "[nltk_data]   Package punkt_tab is already up-to-date!\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from app.travel_resolver.libs.nlp import data_processing as dp\n",
     "\n",
@@ -128,21 +112,22 @@
     "    \"./data/bio/fr.bio/10k_train_small_samples.bio\"\n",
     ")\n",
     "\n",
-    "lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n",
-    "    \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n",
-    ")\n",
+    "# To avoid overfitting the model on sentences that don't have any labels\n",
+    "# lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n",
+    "#     \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n",
+    "# )\n",
     "\n",
     "large_sentences, large_labels, _, __ = dp.from_bio_file_to_examples(\n",
     "    \"./data/bio/fr.bio/1k_train_large_samples.bio\"\n",
     ")\n",
     "\n",
-    "sentences = sentences + lambda_sentences + large_sentences\n",
-    "labels = labels + lambda_labels + large_labels"
+    "sentences = sentences + large_sentences\n",
+    "labels = labels + large_labels"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -155,7 +140,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -181,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -192,7 +177,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -204,7 +189,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 28,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -219,7 +204,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -242,7 +227,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 39,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -269,7 +254,65 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def entity_accuracy(y_true, y_pred):\n",
+    "    \"\"\"\n",
+    "    Calculate the accuracy based on the entities. Which mean that correct `O` tags will not be taken into account.\n",
+    "\n",
+    "    Parameters:\n",
+    "    y_true (tensor): True labels.\n",
+    "    y_pred (tensor): Predicted logits.\n",
+    "\n",
+    "    Returns:\n",
+    "    accuracy (tensor): Tag accuracy.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    y_true = tf.cast(y_true, tf.float32)\n",
+    "    # We ignore the padding and the O tag\n",
+    "    mask = y_true > 0\n",
+    "    mask = tf.cast(mask, tf.float32)\n",
+    "\n",
+    "    y_pred_class = tf.math.argmax(y_pred, axis=-1)\n",
+    "    y_pred_class = tf.cast(y_pred_class, tf.float32)\n",
+    "\n",
+    "    matches_true_pred = tf.equal(y_true, y_pred_class)\n",
+    "    matches_true_pred = tf.cast(matches_true_pred, tf.float32)\n",
+    "\n",
+    "    matches_true_pred *= mask\n",
+    "\n",
+    "    masked_acc = tf.reduce_sum(matches_true_pred) / tf.reduce_sum(mask)\n",
+    "\n",
+    "    return masked_acc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class_weights = {0: 0.1, 1: 20.0, 2: 20.0}\n",
+    "\n",
+    "\n",
+    "def weighted_loss(y_true, y_pred):\n",
+    "    weights = tf.constant(\n",
+    "        [class_weights[i] for i in range(len(class_weights))], dtype=tf.float32\n",
+    "    )\n",
+    "    weights = tf.gather(\n",
+    "        weights, tf.cast(y_true, tf.int32)\n",
+    "    )  # Get weights for true labels\n",
+    "    loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
+    "        y_true, y_pred, from_logits=True\n",
+    "    )\n",
+    "    return loss * weights"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 61,
    "metadata": {},
    "outputs": [
     {
@@ -279,57 +322,64 @@
       "All PyTorch model weights were used when initializing TFCamembertForTokenClassification.\n",
       "\n",
       "Some weights or buffers of the TF 2.0 model TFCamembertForTokenClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
-      "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy TF-Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n"
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
      ]
     }
    ],
    "source": [
+    "from focal_loss import SparseCategoricalFocalLoss\n",
+    "\n",
     "camembert = TFAutoModelForTokenClassification.from_pretrained(\n",
     "    \"camembert-base\", num_labels=len(unique_labels)\n",
     ")\n",
     "\n",
+    "loss_func = SparseCategoricalFocalLoss(\n",
+    "    gamma=2, class_weight=[0.1, 2, 2], from_logits=True\n",
+    ")\n",
+    "\n",
     "camembert.compile(\n",
-    "    optimizer=tf.keras.optimizers.Adam(5e-5),\n",
-    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
-    "    metrics=[\"accuracy\"],\n",
+    "    optimizer=tf.keras.optimizers.legacy.Adam(5e-5),\n",
+    "    loss=loss_func,\n",
+    "    metrics=[\"accuracy\", entity_accuracy],\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 46,
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_dataset = train_dataset.batch(64)\n",
-    "test_dataset = test_dataset.batch(64)"
+    "train_dataset = train_dataset.batch(32)\n",
+    "test_dataset = test_dataset.batch(32)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 62,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Epoch 1/3\n",
-      "148/148 [==============================] - 1725s 12s/step - loss: 0.2062 - accuracy: 0.9711 - val_loss: 0.0952 - val_accuracy: 0.9873\n",
-      "Epoch 2/3\n",
-      "148/148 [==============================] - 1782s 12s/step - loss: 0.0681 - accuracy: 0.9922 - val_loss: 0.0442 - val_accuracy: 0.9953\n",
-      "Epoch 3/3\n",
-      "148/148 [==============================] - 1749s 12s/step - loss: 0.0377 - accuracy: 0.9956 - val_loss: 0.0260 - val_accuracy: 0.9964\n"
+      "Epoch 1/4\n",
+      "272/272 [==============================] - 1596s 6s/step - loss: 0.0124 - accuracy: 0.9677 - entity_accuracy: 0.8099 - val_loss: 0.0038 - val_accuracy: 0.9799 - val_entity_accuracy: 0.9682\n",
+      "Epoch 2/4\n",
+      "272/272 [==============================] - 1560s 6s/step - loss: 0.0031 - accuracy: 0.9852 - entity_accuracy: 0.9684 - val_loss: 0.0019 - val_accuracy: 0.9885 - val_entity_accuracy: 0.9820\n",
+      "Epoch 3/4\n",
+      "272/272 [==============================] - 1560s 6s/step - loss: 0.0020 - accuracy: 0.9907 - entity_accuracy: 0.9767 - val_loss: 0.0016 - val_accuracy: 0.9941 - val_entity_accuracy: 0.9775\n",
+      "Epoch 4/4\n",
+      "272/272 [==============================] - 1605s 6s/step - loss: 0.0016 - accuracy: 0.9923 - entity_accuracy: 0.9789 - val_loss: 0.0017 - val_accuracy: 0.9920 - val_entity_accuracy: 0.9831\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "<tf_keras.src.callbacks.History at 0x295a015b0>"
+       "<tf_keras.src.callbacks.History at 0x2dab031a0>"
       ]
      },
-     "execution_count": 34,
+     "execution_count": 62,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -340,7 +390,7 @@
     ")\n",
     "\n",
     "camembert.fit(\n",
-    "    train_dataset, validation_data=test_dataset, epochs=3, callbacks=[callback]\n",
+    "    train_dataset, validation_data=test_dataset, epochs=4, callbacks=[callback]\n",
     ")"
    ]
   },
@@ -348,9 +398,51 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: shape=(), dtype=float32, numpy=0.1186538115143776>"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from focal_loss import SparseCategoricalFocalLoss\n",
+    "\n",
+    "loss_func = SparseCategoricalFocalLoss(gamma=1)\n",
+    "y_true = [0, 1, 2]\n",
+    "y_pred = [[0.8, 0.1, 0.1], [0.2, 0.7, 0.1], [0.2, 0.2, 0.6]]\n",
+    "loss_func(y_true, y_pred)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "metadata": {},
    "outputs": [],
    "source": [
-    "camembert.save_pretrained(\"./camembert\")"
+    "camembert.save_pretrained(\"./models/camembert\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "tf_model.h5: 100%|██████████| 440M/440M [00:20<00:00, 21.8MB/s] \n"
+     ]
+    }
+   ],
+   "source": [
+    "# camembert.push_to_hub(\"CamemBERT-NER-Travel\")"
    ]
   }
  ],

conv_tagged_file_to_bio.py

@@ -1,9 +1,21 @@
 from app.travel_resolver.libs.nlp.data_processing import from_tagged_file_to_bio_file
 
 
-INPUT_FILE = "./data/french_text/1k_unlabeled_samples.txt"
-OUTPUT_FILE = "./data/bio/fr.bio/1k_unlabeled_samples.bio"
+INPUT_FILES = [
+    "./data/scripting_lcs_1/1k_train_large_samples.txt",
+    "./data/scripting_lcs_1/10k_train_small_samples.txt",
+    "./data/scripting_lcs_1/100_eval_large_samples.txt",
+    "./data/scripting_lcs_1/800_eval_small_samples.txt",
+]
+
+OUTPUT_FILES = [
+    "./data/bio/fr.bio/1k_train_large_samples.bio",
+    "./data/bio/fr.bio/10k_train_small_samples.bio",
+    "./data/bio/fr.bio/100_eval_large_samples.bio",
+    "./data/bio/fr.bio/800_eval_small_samples.bio",
+]
 
 tag_entities_pairs = [("<Dep>", "LOC-DEP"), ("<Arr>", "LOC-ARR")]
 
-from_tagged_file_to_bio_file(INPUT_FILE, OUTPUT_FILE, tag_entities_pairs)
+for i, input_file in enumerate(INPUT_FILES):
+    from_tagged_file_to_bio_file(input_file, OUTPUT_FILES[i], tag_entities_pairs)