optimum intel

app.py

@@ -2,14 +2,18 @@ import gradio as gr
 from transformers import pipeline
 import time
 
-sparse_qa_pipeline = pipeline(task="question-answering",model="Intel/bert-base-uncased-squadv1.1-sparse-80-1x4-block-pruneofa")
+from optimum.intel.neural_compressor import IncQuantizedModelForQuestionAnswering
+
+# model_id = "Intel/bert-base-uncased-squadv1.1-sparse-80-1x4-block-pruneofa"
+model_id = "Intel/distilbert-base-uncased-squadv1.1-sparse-80-1x4-block-pruneofa-int8"
+int8_model = IncQuantizedModelForQuestionAnswering.from_pretrained(model_id)
+
+# sparse_qa_pipeline = pipeline(task="question-answering",model="Intel/bert-base-uncased-squadv1.1-sparse-80-1x4-block-pruneofa")
 # sparse_qa_pipeline = pipeline(task="question-answering",model="Intel/distilbert-base-uncased-squadv1.1-sparse-80-1x4-block-pruneofa-int8")
 
-dense_qa_pipeline = pipeline(task="question-answering",model="csarron/bert-base-uncased-squad-v1")
-# dense_qa_pipeline = pipeline(task="question-answering",model="distilbert-base-uncased-distilled-squad")
+# dense_qa_pipeline = pipeline(task="question-answering",model="csarron/bert-base-uncased-squad-v1")
+dense_qa_pipeline = pipeline(task="question-answering",model="distilbert-base-uncased-distilled-squad")
 
-def greet(name):
-    return "Hello " + name + "!!"
 
 def predict(context,question):
     '''
@@ -22,7 +26,7 @@ def predict(context,question):
     '''
     
     sparse_start_time = time.perf_counter()
-    sparse_predictions = sparse_qa_pipeline(context=context,question=question)
+    sparse_predictions = int8_model(context=context,question=question)
     sparse_end_time = time.perf_counter()
     sparse_duration = (sparse_end_time - sparse_start_time) * 1000
     sparse_answer = sparse_predictions['answer']

requirements.txt

@@ -1,3 +1,4 @@
 transformers
 torch
-tensorflow
+tensorflow
+optimum.intel