Trying ONNX models on CPU.

app.py

@@ -5,7 +5,8 @@ import json
 import onnxruntime
 import time
 from datetime import datetime
-from transformers import pipeline
+from transformers import AutoTokenizer, pipeline
+from optimum.onnxruntime import ORTModelForSequenceClassification
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
@@ -24,10 +25,13 @@ print(f"Is CUDA available: {torch.cuda.is_available()}")
 if torch.cuda.is_available():
     print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")
 
-model_name = "MoritzLaurer/roberta-large-zeroshot-v2.0-c"
-tokenizer_name = "MoritzLaurer/roberta-large-zeroshot-v2.0-c"
+model_name = "MoritzLaurer/ModernBERT-large-zeroshot-v2.0"
+tokenizer_name = "MoritzLaurer/ModernBERT-large-zeroshot-v2.0"
 
-classifier_cpu = pipeline(task="zero-shot-classification", model=model_name, tokenizer=tokenizer_name)
+model_cpu = ORTModelForSequenceClassification.from_pretrained(model_id=model_name, subfolder="onnx", file_name="model_uint8.onnx")
+tokenizer_cpu = AutoTokenizer.from_pretrained(tokenizer_name)
+
+classifier_cpu = pipeline(task="zero-shot-classification", model=model_cpu, tokenizer=tokenizer_cpu)
 classifier_gpu = pipeline(task="zero-shot-classification", model=model_name, tokenizer=tokenizer_name, device="cuda:0") if torch.cuda.is_available() else classifier_cpu
 
 def classify(data_string, request: gradio.Request):

requirements.txt

@@ -5,4 +5,4 @@ json5==0.9.25
 numpy
 uvicorn
 optimum[onnxruntime-gpu]==1.24.0
-transformers==4.36
+transformers==4.48.0