WIP: CamemBERT fine-tuning

camemBERT_finetuning.ipynb

@@ -0,0 +1,378 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# CamemBERT fine-tuning\n",
+    "\n",
+    "Because of dependency conflicts, we will be fine-tuning the model here and then loading it and evaluating in [deepl_ner.ipynb](./deepl_ner.ipynb).\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: transformers in ./venv/lib/python3.12/site-packages (4.46.3)\n",
+      "Requirement already satisfied: filelock in ./venv/lib/python3.12/site-packages (from transformers) (3.16.1)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in ./venv/lib/python3.12/site-packages (from transformers) (0.26.3)\n",
+      "Requirement already satisfied: numpy>=1.17 in ./venv/lib/python3.12/site-packages (from transformers) (1.26.4)\n",
+      "Requirement already satisfied: packaging>=20.0 in ./venv/lib/python3.12/site-packages (from transformers) (24.1)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in ./venv/lib/python3.12/site-packages (from transformers) (6.0.2)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in ./venv/lib/python3.12/site-packages (from transformers) (2024.9.11)\n",
+      "Requirement already satisfied: requests in ./venv/lib/python3.12/site-packages (from transformers) (2.32.3)\n",
+      "Requirement already satisfied: tokenizers<0.21,>=0.20 in ./venv/lib/python3.12/site-packages (from transformers) (0.20.3)\n",
+      "Requirement already satisfied: safetensors>=0.4.1 in ./venv/lib/python3.12/site-packages (from transformers) (0.4.5)\n",
+      "Requirement already satisfied: tqdm>=4.27 in ./venv/lib/python3.12/site-packages (from transformers) (4.66.5)\n",
+      "Requirement already satisfied: fsspec>=2023.5.0 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.10.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./venv/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.4.0)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests->transformers) (3.10)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2.2.3)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests->transformers) (2024.8.30)\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
+      "Requirement already satisfied: tf_keras in ./venv/lib/python3.12/site-packages (2.18.0)\n",
+      "Requirement already satisfied: tensorflow<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tf_keras) (2.18.0)\n",
+      "Requirement already satisfied: absl-py>=1.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.1.0)\n",
+      "Requirement already satisfied: astunparse>=1.6.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.6.3)\n",
+      "Requirement already satisfied: flatbuffers>=24.3.25 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (24.3.25)\n",
+      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.6.0)\n",
+      "Requirement already satisfied: google-pasta>=0.1.1 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.2.0)\n",
+      "Requirement already satisfied: libclang>=13.0.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (18.1.1)\n",
+      "Requirement already satisfied: opt-einsum>=2.3.2 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.4.0)\n",
+      "Requirement already satisfied: packaging in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (24.1)\n",
+      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (4.25.5)\n",
+      "Requirement already satisfied: requests<3,>=2.21.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.32.3)\n",
+      "Requirement already satisfied: setuptools in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (75.2.0)\n",
+      "Requirement already satisfied: six>=1.12.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.16.0)\n",
+      "Requirement already satisfied: termcolor>=1.1.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.5.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.6.6 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (4.12.2)\n",
+      "Requirement already satisfied: wrapt>=1.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.16.0)\n",
+      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.67.0)\n",
+      "Requirement already satisfied: tensorboard<2.19,>=2.18 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (2.18.0)\n",
+      "Requirement already satisfied: keras>=3.5.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.7.0)\n",
+      "Requirement already satisfied: numpy<2.1.0,>=1.26.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (1.26.4)\n",
+      "Requirement already satisfied: h5py>=3.11.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (3.12.1)\n",
+      "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in ./venv/lib/python3.12/site-packages (from tensorflow<2.19,>=2.18->tf_keras) (0.4.1)\n",
+      "Requirement already satisfied: wheel<1.0,>=0.23.0 in ./venv/lib/python3.12/site-packages (from astunparse>=1.6.0->tensorflow<2.19,>=2.18->tf_keras) (0.44.0)\n",
+      "Requirement already satisfied: rich in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (13.9.2)\n",
+      "Requirement already satisfied: namex in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.0.8)\n",
+      "Requirement already satisfied: optree in ./venv/lib/python3.12/site-packages (from keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.13.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (3.4.0)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (3.10)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (2.2.3)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.12/site-packages (from requests<3,>=2.21.0->tensorflow<2.19,>=2.18->tf_keras) (2024.8.30)\n",
+      "Requirement already satisfied: markdown>=2.6.8 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.7)\n",
+      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (0.7.2)\n",
+      "Requirement already satisfied: werkzeug>=1.0.1 in ./venv/lib/python3.12/site-packages (from tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.0.4)\n",
+      "Requirement already satisfied: MarkupSafe>=2.1.1 in ./venv/lib/python3.12/site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow<2.19,>=2.18->tf_keras) (3.0.2)\n",
+      "Requirement already satisfied: markdown-it-py>=2.2.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (3.0.0)\n",
+      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in ./venv/lib/python3.12/site-packages (from rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (2.18.0)\n",
+      "Requirement already satisfied: mdurl~=0.1 in ./venv/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow<2.19,>=2.18->tf_keras) (0.1.2)\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install --upgrade transformers tf-keras numpy sentencepiece"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"TF_USE_LEGACY_KERAS\"] = \"1\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package punkt_tab to /Users/az-r-\n",
+      "[nltk_data]     ow/nltk_data...\n",
+      "[nltk_data]   Package punkt_tab is already up-to-date!\n"
+     ]
+    }
+   ],
+   "source": [
+    "from app.travel_resolver.libs.nlp import data_processing as dp\n",
+    "\n",
+    "sentences, labels, vocab, unique_labels = dp.from_bio_file_to_examples(\n",
+    "    \"./data/bio/fr.bio/10k_train_small_samples.bio\"\n",
+    ")\n",
+    "\n",
+    "lambda_sentences, lambda_labels, _, __ = dp.from_bio_file_to_examples(\n",
+    "    \"./data/bio/fr.bio/1k_train_unlabeled_samples.bio\"\n",
+    ")\n",
+    "\n",
+    "large_sentences, large_labels, _, __ = dp.from_bio_file_to_examples(\n",
+    "    \"./data/bio/fr.bio/1k_train_large_samples.bio\"\n",
+    ")\n",
+    "\n",
+    "sentences = sentences + lambda_sentences + large_sentences\n",
+    "labels = labels + lambda_labels + large_labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import app.travel_resolver.libs.nlp.data_processing as dp\n",
+    "\n",
+    "processed_sentences, processed_labels = dp.process_sentences_and_labels(\n",
+    "    sentences, labels, return_tokens=True, stemming=False\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(len(processed_sentences)):\n",
+    "    for j in range(len(processed_sentences[i])):\n",
+    "        if processed_labels[i][j] > 0:\n",
+    "            processed_sentences[i][j] = processed_sentences[i][j].title()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\"\"\"\n",
+    "  This variable will control the maximum length of the sentence \n",
+    "  as well as the embedding size\n",
+    "\"\"\"\n",
+    "\n",
+    "MAX_LEN = 100"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "padded_labels = tf.keras.preprocessing.sequence.pad_sequences(\n",
+    "    processed_labels, maxlen=MAX_LEN, padding=\"post\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import TFAutoModelForTokenClassification, CamembertTokenizer\n",
+    "import numpy as np\n",
+    "\n",
+    "tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenized_sentences = tokenizer(\n",
+    "    processed_sentences,\n",
+    "    is_split_into_words=True,\n",
+    "    truncation=True,\n",
+    "    padding=\"max_length\",\n",
+    "    max_length=MAX_LEN,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "(\n",
+    "    train_input_ids,\n",
+    "    test_input_ids,\n",
+    "    train_attention_masks,\n",
+    "    test_attention_masks,\n",
+    "    train_labels,\n",
+    "    test_labels,\n",
+    ") = train_test_split(\n",
+    "    tokenized_sentences[\"input_ids\"],\n",
+    "    tokenized_sentences[\"attention_mask\"],\n",
+    "    padded_labels,\n",
+    "    test_size=0.2,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "    (\n",
+    "        {\n",
+    "            \"input_ids\": train_input_ids,\n",
+    "            \"attention_mask\": train_attention_masks,\n",
+    "        },\n",
+    "        train_labels,\n",
+    "    )\n",
+    ")\n",
+    "\n",
+    "test_dataset = tf.data.Dataset.from_tensor_slices(\n",
+    "    (\n",
+    "        {\n",
+    "            \"input_ids\": test_input_ids,\n",
+    "            \"attention_mask\": test_attention_masks,\n",
+    "        },\n",
+    "        test_labels,\n",
+    "    )\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "All PyTorch model weights were used when initializing TFCamembertForTokenClassification.\n",
+      "\n",
+      "Some weights or buffers of the TF 2.0 model TFCamembertForTokenClassification were not initialized from the PyTorch model and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy TF-Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n"
+     ]
+    }
+   ],
+   "source": [
+    "camembert = TFAutoModelForTokenClassification.from_pretrained(\n",
+    "    \"camembert-base\", num_labels=len(unique_labels)\n",
+    ")\n",
+    "\n",
+    "camembert.compile(\n",
+    "    optimizer=tf.keras.optimizers.Adam(5e-5),\n",
+    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+    "    metrics=[\"accuracy\"],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_dataset = train_dataset.batch(64)\n",
+    "test_dataset = test_dataset.batch(64)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Epoch 1/3\n",
+      "148/148 [==============================] - 1725s 12s/step - loss: 0.2062 - accuracy: 0.9711 - val_loss: 0.0952 - val_accuracy: 0.9873\n",
+      "Epoch 2/3\n",
+      "148/148 [==============================] - 1782s 12s/step - loss: 0.0681 - accuracy: 0.9922 - val_loss: 0.0442 - val_accuracy: 0.9953\n",
+      "Epoch 3/3\n",
+      "148/148 [==============================] - 1749s 12s/step - loss: 0.0377 - accuracy: 0.9956 - val_loss: 0.0260 - val_accuracy: 0.9964\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "<tf_keras.src.callbacks.History at 0x295a015b0>"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "callback = tf.keras.callbacks.EarlyStopping(\n",
+    "    monitor=\"val_loss\", patience=0, restore_best_weights=True\n",
+    ")\n",
+    "\n",
+    "camembert.fit(\n",
+    "    train_dataset, validation_data=test_dataset, epochs=3, callbacks=[callback]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "camembert.save_pretrained(\"./camembert\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -12,4 +12,5 @@ scikit-learn
 hmmlearn # for hidden markov models
 ipykernel
 tabulate
-transformers
+transformers
+sentencepiece