asr

app.py

@@ -13,13 +13,16 @@ mms_transcribe = gr.Interface(
             label="Select Model for ASR",
             value="ixxan/wav2vec2-large-mms-1b-uyghur-latin",
             interactive=True
-        )
+        ),
+    ],
+    outputs=[
+        gr.Textbox(label="Uyghur Arabic Transcription"),
+        gr.Textbox(label="Uyghur Latin Transcription"),
     ],
-    outputs="text",
     #examples=ASR_EXAMPLES,
     title="Speech-to-text",
     description=(
-        "Transcribe audio from a microphone or input file."
+        "Transcribe Uyghur speech audio from a microphone or input file."
     ),
     #article=ASR_NOTE,
     allow_flagging="never",
@@ -29,7 +32,7 @@ mms_synthesize = gr.Interface(
     fn=tts.synthesize,
     inputs=[
         gr.Text(label="Input text"),
-         gr.Dropdown(
+        gr.Dropdown(
             choices=[model for model in tts.models_info],
             label="Select Model for TTS",
             value="Meta-MMS",

asr.py

@@ -9,49 +9,55 @@ from transformers import (
     Wav2Vec2ForCTC
 )
 import numpy as np
+import util
 
 # Load processor and model
 models_info = {
     "OpenAI-Whisper-Uzbek": {
         "processor": WhisperProcessor.from_pretrained("openai/whisper-small", language="uzbek", task="transcribe"),
         "model": AutoModelForSpeechSeq2Seq.from_pretrained("openai/whisper-small"),
-        "ctc_model": False
+        "ctc_model": False,
+        "arabic_script": False
     },
     "ixxan/whisper-small-thugy20": {
         "processor": AutoProcessor.from_pretrained("ixxan/whisper-small-thugy20"),
         "model": AutoModelForSpeechSeq2Seq.from_pretrained("ixxan/whisper-small-thugy20"),
-        "ctc_model": False
+        "ctc_model": False,
+        "arabic_script": False
     },
     "ixxan/whisper-small-uyghur-common-voice": {
         "processor": AutoProcessor.from_pretrained("ixxan/whisper-small-uyghur-common-voice"),
         "model": AutoModelForSpeechSeq2Seq.from_pretrained("ixxan/whisper-small-uyghur-common-voice"),
-        "ctc_model": False
+        "ctc_model": False,
+        "arabic_script": False
     },
     "Meta-MMS": {
         "processor": AutoProcessor.from_pretrained("facebook/mms-1b-all", target_lang='uig-script_arabic'),
         "model": AutoModelForCTC.from_pretrained("facebook/mms-1b-all", target_lang='uig-script_arabic', ignore_mismatched_sizes=True),
-        "ctc_model": True
+        "ctc_model": True,
+        "arabic_script": True
     },
     "ixxan/wav2vec2-large-mms-1b-uyghur-latin": {
         "processor": Wav2Vec2Processor.from_pretrained("ixxan/wav2vec2-large-mms-1b-uyghur-latin", target_lang='uig-script_latin'),
         "model": Wav2Vec2ForCTC.from_pretrained("ixxan/wav2vec2-large-mms-1b-uyghur-latin", target_lang='uig-script_latin'),
-        "ctc_model": True
+        "ctc_model": True,
+        "arabic_script": False
     },
 }
 
-def transcribe(audio_data, model_id) -> str:
-    if model_id == "Compare All Models":
-        return transcribe_all_models(audio_data)
-    else:
-        return transcribe_with_model(audio_data, model_id)
+# def transcribe(audio_data, model_id) -> str:
+#     if model_id == "Compare All Models":
+#         return transcribe_all_models(audio_data)
+#     else:
+#         return transcribe_with_model(audio_data, model_id)
 
-def transcribe_all_models(audio_data) -> dict:
-    transcriptions = {}
-    for model_id in models_info.keys():
-        transcriptions[model_id] = transcribe_with_model(audio_data, model_id)
-    return transcriptions
+# def transcribe_all_models(audio_data) -> dict:
+#     transcriptions = {}
+#     for model_id in models_info.keys():
+#         transcriptions[model_id] = transcribe_with_model(audio_data, model_id)
+#     return transcriptions
 
-def transcribe_with_model(audio_data, model_id) -> str:
+def transcribe(audio_data, model_id) -> str:
     # Load audio file
     if not audio_data:
         return "<<ERROR: Empty Audio Input>>"
@@ -97,4 +103,10 @@ def transcribe_with_model(audio_data, model_id) -> str:
             generated_ids = model.generate(inputs["input_features"], max_length=225)
             transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
     
-    return transcription
+    if models_info[model_id]["arabic_script"]:
+        transcription_arabic = transcription
+        transcription_latin = util.ug_arab_to_latn(transcription)
+    else: # Latin script output
+        transcription_arabic = util.ug_latn_to_arab(transcription)
+        transcription_latin = transcription
+    return transcription_arabic, transcription_latin

tts.py

@@ -1,17 +1,21 @@
 from transformers import VitsModel, AutoTokenizer
 import torch
 import scipy.io.wavfile
+import util
 
 # Load processor and model
 models_info = {
     "Meta-MMS": {
         "processor": AutoTokenizer.from_pretrained("facebook/mms-tts-uig-script_arabic"),
         "model": VitsModel.from_pretrained("facebook/mms-tts-uig-script_arabic"),
+        "arabic_script": True
     },
 }
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 def synthesize(text, model_id):
+    if models_info[model_id]["arabic_script"]:
+        text = util.ug_latn_to_arab(text)
     processor = models_info[model_id]["processor"]
     model = models_info[model_id]["model"].to(device)
     inputs = processor(text, return_tensors="pt").to(device)
@@ -20,7 +24,7 @@ def synthesize(text, model_id):
         output = model(**inputs).waveform.cpu()  # Move output back to CPU for saving
     
     output_path = "tts_output.wav"
-    sample_rate = 16000
+    sample_rate = model.config.sample_rate
     scipy.io.wavfile.write(output_path, rate=sample_rate, data=output.numpy()[0])
 
     return output_path

util.py

@@ -0,0 +1,5 @@
+from umsc import UgMultiScriptConverter
+
+# Initialize uyghur script converter 
+ug_arab_to_latn = UgMultiScriptConverter('UAS', 'ULS')
+ug_latn_to_arab = UgMultiScriptConverter('ULS', 'UAS')