multiword

app.py

@@ -1,5 +1,13 @@
 import streamlit as st
 from transformers import pipeline
+import unicodedata
+
+def normalize(text):
+    return unicodedata.normalize('NFC', text
+        ).replace('\u05ba', '\u05b9'
+        ).replace('\u05be', '-'
+        ).replace('״', '"'
+        ).replace("׳", "'")
 
 with st.spinner('Loading TaatikNet framework...'):
     pipe = pipeline("text2text-generation", model='malper/taatiknet', device_map="auto")
@@ -9,5 +17,8 @@ st.success('Loaded!')
 text = st.text_area('Enter text and press ctrl/command+enter:')
 
 if text:
-    output = pipe(text, max_length=40)
-    st.write(output[0]['generated_text'])
+
+    words = [normalize(x) for x in text.split()]
+    outputs = pipe(words, max_length=40)
+    output_text = ' '.join([x['generated_text'] for x in outputs])
+    st.write(output_text)