Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+embeddinggemma-300m/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,57 @@
+# Gemma3 Embedding Model: ONNX Conversion Demonstration
+
+This repository demonstrates the conversion and comparison of the Gemma3 embedding model from Hugging Face to ONNX format using optimum-onnx. It includes scripts for both ONNX and PyTorch inference pipelines, as well as a comparison of their outputs.
+
+## Files
+
+- `onnx_gemma3_pipeline.py`: Runs the Gemma3 embedding model using ONNXRuntime, including post-processing steps (Pooling, Dense, Normalize) with ONNX exported layers.
+- `pytorch_gemma3_pipeline.py`: Runs the original Gemma3 embedding model using PyTorch and SentenceTransformer for reference.
+- `compare_gemma3_onnx_vs_pytorch.py`: Compares the output embeddings and cosine similarities between the ONNX and PyTorch pipelines.
+- `download_missing_hf_files.py`: Downloads required files from Hugging Face and exports Dense layers to ONNX.
+- `gemma3_mean_pooling_basic.py`: The most basic implementation, running Gemma3 ONNX inference with only mean pooling (no Dense or Normalize stages).
+
+## Pipeline Differences
+
+Both pipelines use ONNXRuntime for transformer inference via `ORTModelForFeatureExtraction`. The key difference is in post-processing:
+
+- **ONNX pipeline** (`onnx_gemma3_pipeline.py`): Uses ONNXRuntime for both the transformer and Dense layers (exported to ONNX), making most of the pipeline ONNX-based except for normalization.
+- **PyTorch pipeline** (`pytorch_gemma3_pipeline.py`): Uses ONNXRuntime for the transformer, but all post-processing (Pooling, Dense, Normalize) is performed with PyTorch modules from SentenceTransformer.
+
+This demonstrates how ONNX conversion can offload more computation for faster, hardware-agnostic inference, while the PyTorch pipeline serves as the reference implementation.
+
+## Setup
+
+1. Install dependencies:
+	 ```sh
+	 pip install git+https://github.com/simondanielsson/optimum-onnx.git@feature/add-gemma3-export
+	 pip install git+https://github.com/huggingface/transformers@v4.56.0-Embedding-Gemma-preview
+	 pip install sentence-transformers onnxruntime safetensors huggingface_hub
+	 ```
+2. Export the ONNX model:
+	 ```sh
+	 optimum-cli export onnx --model google/embeddinggemma-300m-qat-q4_0-unquantized embeddinggemma-300m-onnx
+	 python download_missing_hf_files.py
+	 ```
+
+## Usage
+
+- Run the ONNX pipeline:
+	```sh
+	python onnx_gemma3_pipeline.py
+	```
+- Run the PyTorch pipeline:
+	```sh
+	python pytorch_gemma3_pipeline.py
+	```
+- Compare outputs:
+	```sh
+	python compare_gemma3_onnx_vs_pytorch.py
+	```
+
+## Results
+
+The comparison script prints cosine similarities between sample word embeddings (e.g., "apple", "banana", "car") for both ONNX and PyTorch pipelines, demonstrating the fidelity of the ONNX conversion.
+
+## References
+- [Optimum-ONNX Gemma3 PR](https://github.com/huggingface/optimum-onnx/pull/50)
+- [Gemma3 Model](https://huggingface.co/google/embeddinggemma-300m-qat-q4_0-unquantized)

compare_gemma3_onnx_vs_pytorch.py

@@ -0,0 +1,66 @@
+from sentence_transformers import SentenceTransformer
+import torch
+import numpy as np
+
+# Words to compare
+words = ["apple", "banana", "car"]
+
+# Load original SentenceTransformer (PyTorch, CUDA)
+st_model = SentenceTransformer("google/embeddinggemma-300m-qat-q4_0-unquantized")
+st_model = st_model.to("cuda" if torch.cuda.is_available() else "cpu")
+
+# Get PyTorch embeddings
+with torch.no_grad():
+    pt_embeddings = st_model.encode(words, convert_to_numpy=True, device="cuda" if torch.cuda.is_available() else "cpu")
+
+from onnx_gemma3_pipeline import onnx_st
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+
+# Basic mean pooling ONNX implementation
+def basic_mean_pooling(words):
+    tokenizer = AutoTokenizer.from_pretrained("./embeddinggemma-300m")
+    model = ORTModelForFeatureExtraction.from_pretrained("./embeddinggemma-300m")
+    embeddings = []
+    for word in words:
+        inputs = tokenizer(word, return_tensors="pt")
+        input_ids = inputs['input_ids']
+        sequence_length = input_ids.shape[1]
+        position_ids = np.arange(sequence_length)[None, :]
+        position_ids = np.tile(position_ids, (input_ids.shape[0], 1))
+        inputs['position_ids'] = torch.tensor(position_ids, dtype=torch.long)
+        outputs = model(**inputs)
+        last_hidden = outputs.last_hidden_state
+        attention_mask = inputs['attention_mask']
+        from sentence_transformers import models
+        pooling = models.Pooling(word_embedding_dimension=last_hidden.shape[-1], pooling_mode_mean_tokens=True)
+        features = {'token_embeddings': last_hidden, 'attention_mask': attention_mask}
+        pooled = pooling(features)['sentence_embedding']
+        embeddings.append(pooled[0].detach().cpu().numpy())
+    return np.stack(embeddings)
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+onnx_embeddings = onnx_st.encode(words)
+
+# Cosine similarity function
+def cosine_similarity(a, b):
+    a = a.flatten()
+    b = b.flatten()
+    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
+
+print("Safetensor Cosine similarities:")
+print(f"apple vs banana: {cosine_similarity(pt_embeddings[0], pt_embeddings[1]):.4f}")
+print(f"apple vs car: {cosine_similarity(pt_embeddings[0], pt_embeddings[2]):.4f}")
+print(f"banana vs car: {cosine_similarity(pt_embeddings[1], pt_embeddings[2]):.4f}")
+
+print("\nONNX Cosine similarities:")
+print(f"apple vs banana: {cosine_similarity(onnx_embeddings[0], onnx_embeddings[1]):.4f}")
+print(f"apple vs car: {cosine_similarity(onnx_embeddings[0], onnx_embeddings[2]):.4f}")
+print(f"banana vs car: {cosine_similarity(onnx_embeddings[1], onnx_embeddings[2]):.4f}")
+
+# Basic mean pooling ONNX pipeline
+basic_embeddings = basic_mean_pooling(words)
+print("\nBasic ONNX (mean pooling only) Cosine similarities:")
+print(f"apple vs banana: {cosine_similarity(basic_embeddings[0], basic_embeddings[1]):.4f}")
+print(f"apple vs car: {cosine_similarity(basic_embeddings[0], basic_embeddings[2]):.4f}")
+print(f"banana vs car: {cosine_similarity(basic_embeddings[1], basic_embeddings[2]):.4f}")

download_missing_hf_files.py

@@ -0,0 +1,69 @@
+from huggingface_hub import snapshot_download
+import os
+import shutil
+from sentence_transformers import SentenceTransformer
+import torch
+
+# Model repo and local directory
+repo_id = "google/embeddinggemma-300m-qat-q4_0-unquantized"
+local_dir = "embeddinggemma-300m"
+
+# Download all files except model.safetensors and those already present
+existing_files = set(os.listdir(local_dir))
+
+# Download snapshot to a temp dir
+temp_dir = "_hf_temp_download"
+os.makedirs(temp_dir, exist_ok=True)
+snapshot_download(
+    repo_id,
+    local_dir=temp_dir,
+    ignore_patterns=["model.safetensors"],
+    resume_download=True,
+    allow_patterns=None
+)
+
+# Copy missing files
+for fname in os.listdir(temp_dir):
+    if fname not in existing_files:
+        shutil.move(os.path.join(temp_dir, fname), os.path.join(local_dir, fname))
+        print(f"Downloaded: {fname}")
+    else:
+        print(f"Already exists: {fname}")
+
+# Clean up temp dir
+shutil.rmtree(temp_dir)
+print("Done.")
+
+# Export Dense layers from SentenceTransformer to ONNX
+st_model = SentenceTransformer(repo_id)
+dense1 = st_model[2].linear
+dense2 = st_model[3].linear
+
+onnx_dir = os.path.join(local_dir, "onnx")
+os.makedirs(onnx_dir, exist_ok=True)
+
+# Export Dense1
+dummy_input1 = torch.randn(1, dense1.in_features)
+dense1 = dense1.to(dummy_input1.device)
+torch.onnx.export(
+    dense1,
+    dummy_input1,
+    os.path.join(onnx_dir, "dense1.onnx"),
+    input_names=["input"],
+    output_names=["output"],
+    opset_version=14
+)
+print("Exported dense1.onnx")
+
+# Export Dense2
+dummy_input2 = torch.randn(1, dense2.in_features)
+dense2 = dense2.to(dummy_input2.device)
+torch.onnx.export(
+    dense2,
+    dummy_input2,
+    os.path.join(onnx_dir, "dense2.onnx"),
+    input_names=["input"],
+    output_names=["output"],
+    opset_version=14
+)
+print("Exported dense2.onnx")

embeddinggemma-300m/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

embeddinggemma-300m/1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+    "word_embedding_dimension": 768,
+    "pooling_mode_cls_token": false,
+    "pooling_mode_mean_tokens": true,
+    "pooling_mode_max_tokens": false,
+    "pooling_mode_mean_sqrt_len_tokens": false,
+    "pooling_mode_weightedmean_tokens": false,
+    "pooling_mode_lasttoken": false,
+    "include_prompt": true
+}

embeddinggemma-300m/2_Dense/config.json

@@ -0,0 +1,6 @@
+{
+    "in_features": 768,
+    "out_features": 3072,
+    "bias": false,
+    "activation_function": "torch.nn.modules.linear.Identity"
+}

embeddinggemma-300m/3_Dense/config.json

@@ -0,0 +1,6 @@
+{
+    "in_features": 3072,
+    "out_features": 768,
+    "bias": false,
+    "activation_function": "torch.nn.modules.linear.Identity"
+}

embeddinggemma-300m/README.md

@@ -0,0 +1,444 @@
+---
+license: gemma
+pipeline_tag: sentence-similarity
+library_name: sentence-transformers
+tags:
+- sentence-transformers
+- sentence-similarity
+- feature-extraction
+extra_gated_heading: Access EmbeddingGemma on Hugging Face
+extra_gated_prompt: To access EmbeddingGemma on Hugging Face, you’re required to review and
+  agree to Google’s usage license. To do this, please ensure you’re logged in to Hugging
+  Face and click below. Requests are processed immediately.
+extra_gated_button_content: Acknowledge license
+---
+
+# EmbeddingGemma model card
+
+**Model Page**: [EmbeddingGemma](https://ai.google.dev/gemma/docs/embeddinggemma)
+
+**Resources and Technical Documentation**:
+
+*   [Responsible Generative AI Toolkit](https://ai.google.dev/responsible)
+*   [EmbeddingGemma on Kaggle](https://www.kaggle.com/models/google/embeddinggemma/)
+*   [EmbeddingGemma on Vertex Model Garden](https://console.cloud.google.com/vertex-ai/publishers/google/model-garden/embeddinggemma)
+
+**Terms of Use**: [Terms](https://ai.google.dev/gemma/terms)
+
+**Authors**: Google DeepMind
+
+## Model Information
+
+### Description
+
+EmbeddingGemma is a 300M parameter, state-of-the-art for its size, open embedding model from Google, built from Gemma 3 (with T5Gemma initialization) and the same research and technology used to create Gemini models. EmbeddingGemma produces vector representations of text, making it well-suited for search and retrieval tasks, including classification, clustering, and semantic similarity search. This model was trained with data in 100+ spoken languages.
+
+The small size and on-device focus makes it possible to deploy in environments with limited resources such as mobile phones, laptops, or desktops, democratizing access to state of the art AI models and helping foster innovation for everyone.
+
+### Inputs and outputs
+
+-   **Input:**
+    -   Text string, such as a question, a prompt, or a document to be embedded
+    -   Maximum input context length of 2048 tokens
+
+-   **Output:**
+    -   Numerical vector representations of input text data
+    -   Output embedding dimension size of 768, with smaller options available (512, 256, or 128) via Matryoshka Representation Learning (MRL). MRL allows users to truncate the output embedding of size 768 to their desired size and then re-normalize for efficient and accurate representation.
+
+### Usage
+
+These model weights are designed to be used with [Sentence Transformers](https://www.SBERT.net), using the [Gemma 3](https://huggingface.co/docs/transformers/main/en/model_doc/gemma3) implementation from [Hugging Face Transformers](https://huggingface.co/docs/transformers/en/index) as the backbone.
+
+First install the Sentence Transformers library:
+
+```bash
+pip install -U sentence-transformers
+```
+
+Then you can load this model and run inference.
+
+```python
+from sentence_transformers import SentenceTransformer
+
+# Download from the 🤗 Hub
+model = SentenceTransformer("google/embeddinggemma-300m")
+
+# Run inference with queries and documents
+query = "Which planet is known as the Red Planet?"
+documents = [
+    "Venus is often called Earth's twin because of its similar size and proximity.",
+    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
+    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
+    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet."
+]
+query_embeddings = model.encode_query(query)
+document_embeddings = model.encode_document(documents)
+print(query_embeddings.shape, document_embeddings.shape)
+# (768,) (4, 768)
+
+# Compute similarities to determine a ranking
+similarities = model.similarity(query_embeddings, document_embeddings)
+print(similarities)
+# tensor([[0.3011, 0.6359, 0.4930, 0.4889]])
+```
+
+**NOTE**: EmbeddingGemma activations do not support `float16`. Please use `float32` or `bfloat16` as appropriate for your hardware.
+
+## Model Data
+
+### Training Dataset
+
+This model was trained on a dataset of text data that includes a wide variety of sources totaling approximately 320 billion tokens. Here are the key components:
+
+-   **Web Documents**: A diverse collection of web text ensures the model is exposed to a broad range of linguistic styles, topics, and vocabulary. The training dataset includes content in over 100 languages.
+-   **Code and Technical Documents**: Exposing the model to code and technical documentation helps it learn the structure and patterns of programming languages and specialized scientific content, which improves its understanding of code and technical questions.
+-   **Synthetic and Task-Specific Data**: Synthetically training data helps to teach the model specific skills. This includes curated data for tasks like information retrieval, classification, and sentiment analysis, which helps to fine-tune its performance for common embedding applications.
+
+The combination of these diverse data sources is crucial for training a powerful multilingual embedding model that can handle a wide variety of different tasks and data formats.
+
+### Data Preprocessing
+
+Here are the key data cleaning and filtering methods applied to the training data:
+
+-   CSAM Filtering: Rigorous CSAM (Child Sexual Abuse Material) filtering was applied at multiple stages in the data preparation process to ensure the exclusion of harmful and illegal content.
+-   Sensitive Data Filtering: As part of making Gemma pre-trained models safe and reliable, automated techniques were used to filter out certain personal information and other sensitive data from training sets.
+-   Additional methods: Filtering based on content quality and safety in line with [our policies](https://ai.google/static/documents/ai-responsibility-update-published-february-2025.pdf).
+
+## Model Development
+
+### Hardware
+
+EmbeddingGemma was trained using the latest generation of [Tensor Processing Unit (TPU)](https://cloud.google.com/tpu/docs/intro-to-tpu) hardware (TPUv5e), for more details refer to the [Gemma 3 model card](https://ai.google.dev/gemma/docs/core/model_card_3).
+
+### Software
+
+Training was done using [JAX](https://github.com/jax-ml/jax) and [ML Pathways](https://blog.google/technology/ai/introducing-pathways-next-generation-ai-architecture/). For more details refer to the [Gemma 3 model card](https://ai.google.dev/gemma/docs/core/model_card_3).
+
+## Evaluation
+
+### Benchmark Results
+
+The model was evaluated against a large collection of different datasets and metrics to cover different aspects of text understanding.
+
+#### Full Precision Checkpoint
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (Multilingual, v2)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Dimensionality</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td>768d</td>
+      <td>61.15</td>
+      <td>54.31</td>
+    </tr>
+    <tr>
+      <td>512d</td>
+      <td>60.71</td>
+      <td>53.89</td>
+    </tr>
+    <tr>
+      <td>256d</td>
+      <td>59.68</td>
+      <td>53.01</td>
+    </tr>
+    <tr>
+      <td>128d</td>
+      <td>58.23</td>
+      <td>51.77</td>
+    </tr>
+  </tbody>
+</table>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (English, v2)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Dimensionality</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td>768d</td>
+      <td>68.36</td>
+      <td>64.15</td>
+    </tr>
+    <tr>
+      <td>512d</td>
+      <td>67.80</td>
+      <td>63.59</td>
+    </tr>
+    <tr>
+      <td>256d</td>
+      <td>66.89</td>
+      <td>62.94</td>
+    </tr>
+    <tr>
+      <td>128d</td>
+      <td>65.09</td>
+      <td>61.56</td>
+    </tr>
+  </tbody>
+</table>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (Code, v1)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Dimensionality</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td>768d</td>
+      <td>68.76</td>
+      <td>68.76</td>
+    </tr>
+    <tr>
+      <td>512d</td>
+      <td>68.48</td>
+      <td>68.48</td>
+    </tr>
+    <tr>
+      <td>256d</td>
+      <td>66.74</td>
+      <td>66.74</td>
+    </tr>
+    <tr>
+      <td>128d</td>
+      <td>62.96</td>
+      <td>62.96</td>
+    </tr>
+  </tbody>
+</table>
+
+#### QAT Checkpoints
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (Multilingual, v2)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Quant config (dimensionality)</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td><em><strong>Q4_0 (768d)</strong></em></td>
+      <td><em><strong>60.62</strong></em></td>
+      <td><em><strong>53.61</strong></em></td>
+    </tr>
+    <tr>
+      <td>Q8_0 (768d)</td>
+      <td>60.93</td>
+      <td>53.95</td>
+    </tr>
+    <tr>
+      <td>Mixed Precision* (768d)</td>
+      <td>60.69</td>
+      <td>53.82</td>
+    </tr>
+  </tbody>
+</table>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (English, v2)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Quant config (dimensionality)</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td><em><strong>Q4_0 (768d)</strong></em></td>
+      <td><em><strong>67.91</strong></em></td>
+      <td><em><strong>63.64</strong></em></td>
+    </tr>
+    <tr>
+      <td>Q8_0 (768d)</td>
+      <td>68.13</td>
+      <td>63.85</td>
+    </tr>
+    <tr>
+      <td>Mixed Precision* (768d)</td>
+      <td>67.95</td>
+      <td>63.83</td>
+    </tr>
+  </tbody>
+</table>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"><strong>MTEB (Code, v1)</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><strong>Quant config (dimensionality)</strong></td>
+      <td><strong>Mean (Task)</strong></td>
+      <td><strong>Mean (TaskType)</strong></td>
+    </tr>
+    <tr>
+      <td><em><strong>Q4_0 (768d)</strong></em></td>
+      <td><em><strong>67.99</strong></em></td>
+      <td><em><strong>67.99</strong></em></td>
+    </tr>
+    <tr>
+      <td>Q8_0 (768d)</td>
+      <td>68.70</td>
+      <td>68.70</td>
+    </tr>
+    <tr>
+      <td>Mixed Precision* (768d)</td>
+      <td>68.03</td>
+      <td>68.03</td>
+    </tr>
+  </tbody>
+</table>
+
+Note: QAT models are evaluated after quantization
+
+\* Mixed Precision refers to per-channel quantization with int4 for embeddings, feedforward, and projection layers, and int8 for attention (e4_a8_f4_p4).
+
+### Prompt Instructions
+
+EmbeddingGemma can generate optimized embeddings for various use cases—such as document retrieval, question answering, and fact verification—or for specific input types—either a query or a document—using prompts that are prepended to the input strings.
+Query prompts follow the form `task: {task description} | query: ` where the task description varies by the use case, with the default task description being `search result`. Document-style prompts follow the form `title: {title | "none"} | text: ` where the title is either `none` (the default) or the actual title of the document. Note that providing a title, if available, will improve model performance for document prompts but may require manual formatting.
+
+Use the following prompts based on your use case and input data type. These may already be available in the EmbeddingGemma configuration in your modeling framework of choice.
+
+<table>
+  <thead>
+    <tr>
+      <th><br>
+<strong>Use Case (task type enum)</strong></th>
+      <th><br>
+<strong>Descriptions</strong></th>
+      <th><br>
+<strong>Recommended Prompt</strong></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><br>
+Retrieval (Query)</td>
+      <td rowspan="4"><br>
+Used to generate embeddings that are optimized for document search or information retrieval</td>
+      <td><br>
+task: search result | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Retrieval (Document)</td>
+      <td><br>
+title: {title | "none"} | text: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Question Answering</td>
+      <td><br>
+task: question answering | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Fact Verification</td>
+      <td><br>
+task: fact checking | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Classification</td>
+      <td><br>
+Used to generate embeddings that are optimized to classify texts according to preset labels</td>
+      <td><br>
+task: classification | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Clustering</td>
+      <td><br>
+Used to generate embeddings that are optimized to cluster texts based on their similarities</td>
+      <td><br>
+task: clustering | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Semantic Similarity</td>
+      <td><br>
+Used to generate embeddings that are optimized to assess text similarity. This is not intended for retrieval use cases.</td>
+      <td><br>
+task: sentence similarity | query: {content}</td>
+    </tr>
+    <tr>
+      <td><br>
+Code Retrieval</td>
+      <td><br>
+Used to retrieve a code block based on a natural language query, such as <em>sort an array</em> or <em>reverse a linked list</em>. Embeddings of the code blocks are computed using retrieval_document.</td>
+      <td><br>
+task: code retrieval | query: {content}</td>
+    </tr>
+  </tbody>
+</table>
+
+## Usage and Limitations
+
+These models have certain limitations that users should be aware of.
+
+### Intended Usage
+
+Open embedding models have a wide range of applications across various industries and domains. The following list of potential uses is not comprehensive. The purpose of this list is to provide contextual information about the possible use-cases that the model creators considered as part of model training and development.
+
+-   **Semantic Similarity**: Embeddings optimized to assess text similarity, such as recommendation systems and duplicate detection
+-   **Classification**: Embeddings optimized to classify texts according to preset labels, such as sentiment analysis and spam detection
+-  **Clustering**: Embeddings optimized to cluster texts based on their similarities, such as document organization, market research, and anomaly detection
+-   **Retrieval**
+    -   **Document**: Embeddings optimized for document search, such as indexing articles, books, or web pages for search
+    -   **Query**: Embeddings optimized for general search queries, such as custom search
+    -   **Code Query**: Embeddings optimized for retrieval of code blocks based on natural language queries, such as code suggestions and search
+
+-   **Question Answering**: Embeddings for questions in a question-answering system, optimized for finding documents that answer the question, such as chatbox.
+-   **Fact Verification**: Embeddings for statements that need to be verified, optimized for retrieving documents that contain evidence supporting or refuting the statement, such as automated fact-checking systems.
+
+### Limitations
+
+-   Training Data
+    -   The quality and diversity of the training data significantly influence the model's capabilities. Biases or gaps in the training data can lead to limitations in the model's responses.
+    -   The scope of the training dataset determines the subject areas the model can handle effectively.
+
+-   Language Ambiguity and Nuance
+    -   Natural language is inherently complex. Models might struggle to grasp subtle nuances, sarcasm, or figurative language.
+
+### Ethical Considerations and Risks
+
+Risks identified and mitigations:
+
+-   **Perpetuation of biases**: It's encouraged to perform continuous monitoring (using evaluation metrics, human review) and the exploration of de-biasing techniques during model training, fine-tuning, and other use cases.
+-   **Misuse for malicious purposes**: Technical limitations and developer and end-user education can help mitigate against malicious applications of embeddings. Educational resources and reporting mechanisms for users to flag misuse are provided. Prohibited uses of Gemma models are outlined in the [Gemma Prohibited Use Policy](https://ai.google.dev/gemma/prohibited_use_policy).
+-   **Privacy violations**: Models were trained on data filtered for removal of certain personal information and other sensitive data. Developers are encouraged to adhere to privacy regulations with privacy-preserving techniques.
+
+### Benefits
+
+At the time of release, this family of models provides high-performance open embedding model implementations designed from the ground up for responsible AI development compared to similarly sized models. Using the benchmark evaluation metrics described in this document, these models have shown superior performance to other, comparably-sized open model alternatives.

embeddinggemma-300m/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

embeddinggemma-300m/config.json

@@ -0,0 +1,61 @@
+{
+  "_sliding_window_pattern": 6,
+  "architectures": [
+    "Gemma3TextModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_logit_softcapping": null,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "final_logit_softcapping": null,
+  "head_dim": 256,
+  "hidden_activation": "gelu_pytorch_tanh",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "layer_types": [
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 2048,
+  "model_type": "gemma3_text",
+  "num_attention_heads": 3,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 1,
+  "pad_token_id": 0,
+  "query_pre_attn_scalar": 256,
+  "rms_norm_eps": 1e-06,
+  "rope_local_base_freq": 10000.0,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 512,
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.3",
+  "use_bidirectional_attention": true,
+  "use_cache": true,
+  "vocab_size": 262144
+}

embeddinggemma-300m/config_sentence_transformers.json

@@ -0,0 +1,26 @@
+{
+  "model_type": "SentenceTransformer",
+  "__version__": {
+    "sentence_transformers": "5.1.0",
+    "transformers": "4.57.0.dev0",
+    "pytorch": "2.8.0+cu128"
+  },
+  "prompts": {
+    "query": "task: search result | query: ",
+    "document": "title: none | text: ",
+    "BitextMining": "task: search result | query: ",
+    "Clustering": "task: clustering | query: ",
+    "Classification": "task: classification | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-query": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: ",
+    "STS": "task: sentence similarity | query: ",
+    "Summarization": "task: summarization | query: "
+  },
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

embeddinggemma-300m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.57.0.dev0"
+}

embeddinggemma-300m/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee985629c11dd0f70531093aeb8e8f7f5ddfb403f6c2705db340d58e4e03ffb
+size 1212541436

embeddinggemma-300m/modules.json

@@ -0,0 +1,32 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Dense",
+    "type": "sentence_transformers.models.Dense"
+  },
+  {
+    "idx": 3,
+    "name": "3",
+    "path": "3_Dense",
+    "type": "sentence_transformers.models.Dense"
+  },
+  {
+    "idx": 4,
+    "name": "4",
+    "path": "4_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

embeddinggemma-300m/onnx/dense1.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0131f901cb1759fd45439258c08d0a0bd1eea8aaae8b25556bd64550426cbf9
+size 9437360

embeddinggemma-300m/onnx/dense2.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c5394b30c83bfd957392ef027c6ce52f6771c05c7af2fc627537ef3377cde9
+size 9437360

embeddinggemma-300m/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+    "max_seq_length": 2048,
+    "do_lower_case": false
+}

embeddinggemma-300m/special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

embeddinggemma-300m/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6852f8d561078cc0cebe70ca03c5bfdd0d60a45f9d2e0e1e4cc05b68e9ec329e
+size 33385008

embeddinggemma-300m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

gemma3_mean_pooling_basic.py

@@ -0,0 +1,24 @@
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+from sentence_transformers import models
+import numpy as np
+import torch
+
+tokenizer = AutoTokenizer.from_pretrained("./embeddinggemma-300m")
+model = ORTModelForFeatureExtraction.from_pretrained("./embeddinggemma-300m")
+
+inputs = tokenizer("apple", return_tensors="pt")
+print(inputs)
+input_ids = inputs['input_ids']
+sequence_length = input_ids.shape[1]
+position_ids = np.arange(sequence_length)[None, :]
+position_ids = np.tile(position_ids, (input_ids.shape[0], 1))
+inputs['position_ids'] = torch.tensor(position_ids, dtype=torch.long)
+outputs = model(**inputs)
+last_hidden = outputs.last_hidden_state
+attention_mask = inputs['attention_mask']
+# Use SentenceTransformer's Pooling module for mean pooling
+pooling = models.Pooling(word_embedding_dimension=last_hidden.shape[-1], pooling_mode_mean_tokens=True)
+features = {'token_embeddings': last_hidden, 'attention_mask': attention_mask}
+pooled = pooling(features)['sentence_embedding']
+print("Mean pooled:", pooled[0][:5].detach().cpu().numpy())

onnx_gemma3_pipeline.py

@@ -0,0 +1,91 @@
+from sentence_transformers import models
+import torch
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+import numpy as np
+import os
+import onnxruntime
+
+# ONNX pipeline for Gemma3 embedding model
+model_dir = "embeddinggemma-300m"
+tokenizer = AutoTokenizer.from_pretrained("google/embeddinggemma-300m-qat-q4_0-unquantized")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+onnx_model = ORTModelForFeatureExtraction.from_pretrained(
+    model_dir,
+    file_name="model.onnx"
+).to(device)
+
+class ONNXTransformer:
+    def __init__(self, onnx_model, tokenizer, max_seq_length=2048):
+        self.onnx_model = onnx_model
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+    def encode(self, sentences):
+        inputs = self.tokenizer(sentences, return_tensors="pt", padding=True, truncation=True, max_length=self.max_seq_length)
+        input_ids = inputs['input_ids']
+        sequence_length = input_ids.shape[1]
+        position_ids = torch.arange(sequence_length)[None, :].expand(input_ids.shape[0], sequence_length)
+        inputs['position_ids'] = position_ids.to(input_ids.device)
+        with torch.no_grad():
+            outputs = self.onnx_model(**inputs)
+        return outputs.last_hidden_state
+
+modules = []
+onnx_transformer = ONNXTransformer(onnx_model, tokenizer, max_seq_length=2048)
+modules.append(onnx_transformer)
+for idx, name in [(1, "Pooling"), (2, "Dense"), (3, "Dense"), (4, "Normalize")]:
+    module_path = os.path.join(model_dir, f"{idx}_{name}")
+    if name == "Pooling":
+        modules.append(models.Pooling(module_path))
+    elif name == "Dense":
+        # Use ONNXRuntime for Dense layers
+        dense_onnx_path = os.path.join(model_dir, "onnx", f"dense{idx-1}.onnx")
+        modules.append(onnxruntime.InferenceSession(dense_onnx_path, providers=["CPUExecutionProvider"]))
+    elif name == "Normalize":
+        modules.append(models.Normalize())
+
+class ONNXSentenceTransformer:
+    def __init__(self, modules):
+        self.modules = modules
+    def encode(self, sentences):
+        features = self.modules[0].encode(sentences)
+        for module in self.modules[1:]:
+            if isinstance(module, models.Pooling):
+                features = module({'token_embeddings': features, 'attention_mask': torch.ones(features.shape[:2], device=features.device)})['sentence_embedding']
+            elif isinstance(module, onnxruntime.InferenceSession):
+                # ONNX Dense layer expects shape [1, in_features], so process each embedding separately
+                if isinstance(features, torch.Tensor):
+                    features = features.cpu().detach().numpy()
+                outputs = []
+                for vec in features:
+                    ort_inputs = {module.get_inputs()[0].name: vec.reshape(1, -1)}
+                    out = module.run(None, ort_inputs)[0]
+                    outputs.append(out.squeeze(0))
+                features = np.stack(outputs, axis=0)
+            elif isinstance(module, models.Normalize):
+                # Normalize still uses PyTorch
+                if not isinstance(features, torch.Tensor):
+                    features = torch.from_numpy(features)
+                features = module({'sentence_embedding': features})['sentence_embedding']
+        if isinstance(features, torch.Tensor):
+            return features.cpu().detach().numpy()
+        return features
+
+onnx_st = ONNXSentenceTransformer(modules)
+
+def cosine_similarity(a, b):
+    a = a.flatten()
+    b = b.flatten()
+    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
+
+if __name__ == "__main__":
+    words = ["apple", "banana", "car"]
+    embeddings = onnx_st.encode(words)
+    print(embeddings)
+    for idx, embedding in enumerate(embeddings):
+        print(f"Embedding {idx+1}: {embedding.shape}")
+
+    print("\nCosine similarities:")
+    print(f"apple vs banana: {cosine_similarity(embeddings[0], embeddings[1]):.4f}")
+    print(f"apple vs car: {cosine_similarity(embeddings[0], embeddings[2]):.4f}")
+    print(f"banana vs car: {cosine_similarity(embeddings[1], embeddings[2]):.4f}")

pytorch_gemma3_pipeline.py

@@ -0,0 +1,58 @@
+from sentence_transformers import models
+import torch
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+import numpy as np
+# Load tokenizer and ONNX model
+model_path = "./embeddinggemma-300m"
+tokenizer = AutoTokenizer.from_pretrained("google/embeddinggemma-300m-qat-q4_0-unquantized")
+device = "cuda" if torch.cuda.is_available() else "cpu"
+onnx_model = ORTModelForFeatureExtraction.from_pretrained(model_path).to(device)
+
+class ONNXSentenceTransformer:
+    def __init__(self, model, tokenizer):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.word_embedding_dimension = 768
+        self.pooling = models.Pooling(word_embedding_dimension=self.word_embedding_dimension, pooling_mode_mean_tokens=True)
+
+    def encode(self, sentences, batch_size=32):
+        if isinstance(sentences, str):
+            sentences = [sentences]
+        embeddings = []
+        for i in range(0, len(sentences), batch_size):
+            batch = sentences[i:i+batch_size]
+            inputs = self.tokenizer(batch, return_tensors="pt", padding=True, truncation=True)
+            input_ids = inputs['input_ids']
+            sequence_length = input_ids.shape[1]
+            position_ids = torch.arange(sequence_length)[None, :].expand(input_ids.shape[0], sequence_length)
+            inputs['position_ids'] = position_ids
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+            last_hidden = outputs.last_hidden_state
+            attention_mask = inputs['attention_mask'].to(last_hidden.device)
+            features = {'token_embeddings': last_hidden, 'attention_mask': attention_mask}
+            pooled = self.pooling(features)['sentence_embedding']
+            embeddings.append(pooled)
+        return torch.cat(embeddings, dim=0).cpu().detach().numpy()
+
+
+# Usage example
+onnx_st = ONNXSentenceTransformer(onnx_model, tokenizer)
+
+words = ["apple", "banana", "car"]
+embeddings = onnx_st.encode(words)
+print(embeddings)
+for idx, embedding in enumerate(embeddings):
+    print(f"Embedding {idx+1}: {embedding.shape}")
+
+# Cosine similarity demonstration
+def cosine_similarity(a, b):
+    a = a.flatten()
+    b = b.flatten()
+    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
+
+print("\nCosine similarities:")
+print(f"apple vs banana: {cosine_similarity(embeddings[0], embeddings[1]):.4f}")
+print(f"apple vs car: {cosine_similarity(embeddings[0], embeddings[2]):.4f}")
+print(f"banana vs car: {cosine_similarity(embeddings[1], embeddings[2]):.4f}")