Update app.py

app.py

@@ -23,78 +23,27 @@ import tempfile
 
 #     return sanskrit_text, audio_path
 # Load model and tokenizer
-import os
-import sys
-import transformers
-import tensorflow as tf
-from datasets import load_dataset
-from transformers import AutoTokenizer
-from transformers import TFAutoModelForSeq2SeqLM, DataCollatorForSeq2Seq
-from transformers import AdamWeightDecay
-from transformers import AutoTokenizer, TFAutoModelForSeq2SeqLM
+# import os
+# import sys
+# import transformers
+# import tensorflow as tf
+# from datasets import load_dataset
+# from transformers import AutoTokenizer
+# from transformers import TFAutoModelForSeq2SeqLM, DataCollatorForSeq2Seq
+# from transformers import AdamWeightDecay
+# from transformers import AutoTokenizer, TFAutoModelForSeq2SeqLM
 
-model_checkpoint = "Helsinki-NLP/opus-mt-en-hi"
+# model_checkpoint = "Helsinki-NLP/opus-mt-en-hi"
 
-from datasets import load_dataset
+# from datasets import load_dataset
 
-raw_datasets = load_dataset("rahular/itihasa", download_mode="force_redownload")
+# raw_datasets = load_dataset("rahular/itihasa", download_mode="force_redownload")
 
-import torch
-from transformers import MarianMTModel, MarianTokenizer, Trainer, TrainingArguments
-from datasets import load_dataset
+# import torch
+# from transformers import MarianMTModel, MarianTokenizer, Trainer, TrainingArguments
+# from datasets import load_dataset
 
 # Load the pre-trained English to Hindi model
-model_checkpoint = "Helsinki-NLP/opus-mt-en-hi"
-model = MarianMTModel.from_pretrained(model_checkpoint)
-tokenizer = MarianTokenizer.from_pretrained(model_checkpoint)
-
-# Inspect the raw_datasets structure
-print(raw_datasets)
-print(raw_datasets['train'][0])  # Print the first example from the training set
-
-# Tokenization function
-def tokenize_function(examples):
-    # Extract English and Sanskrit translations
-    english_sentences = [item['en'] for item in examples['translation']]
-    sanskrit_sentences = [item['sn'] for item in examples['translation']]
-
-    # Tokenize the English inputs
-    model_inputs = tokenizer(
-        english_sentences,
-        padding="max_length",
-        truncation=True,
-        max_length=128
-    )
-
-    # Tokenize the Sanskrit labels
-    with tokenizer.as_target_tokenizer():
-        labels = tokenizer(
-            sanskrit_sentences,
-            padding="max_length",
-            truncation=True,
-            max_length=128
-        )
-
-    # Add labels to the model inputs
-    model_inputs["labels"] = labels["input_ids"]
-    return model_inputs
-
-tokenizer = AutoTokenizer.from_pretrained(get_model_name())
-
-model = M2M100ForConditionalGeneration.from_pretrained(get_model_name())
-# I dont know wheter this will be of use or not
-
-tokenized_train = raw_datasets['train'].map(tokenize_function, batched=True)
-
-
-
-tokenized_validation = raw_datasets['validation'].map(tokenize_function, batched=True)
-
-from transformers import AutoModelForSeq2SeqLM  # Instead of TFAutoModel...
-
-model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)
-
-
 # from transformers import M2M100ForConditionalGeneration, AutoModelForCausalLM
 
 # # Load appropriate model based on phase
@@ -259,28 +208,6 @@ model___name = "SweUmaVarsh/m2m100-en-sa-translation"
 #     shuffle=False,
 #     batch_size=8,
 #     collate_fn=data_collator,
-# )
-
-# from transformers import create_optimizer
-
-# steps_per_epoch = len(train_dataset)
-# num_train_steps = steps_per_epoch * 1  # 1 epoch in your case
-# num_warmup_steps = int(0.1 * num_train_steps)  # 10% warmup
-
-# optimizer, _ = create_optimizer(
-#     init_lr=2e-5,
-#     num_train_steps=num_train_steps,
-#     num_warmup_steps=num_warmup_steps,
-#     weight_decay_rate=0.01
-# )
-
-# model.compile(optimizer=optimizer)
-# model.fit(train_dataset, validation_data=val_dataset, epochs=1)
-
-
-
-
-
 
 model____name="Rask6723/IT_GR7_En-Sn"
 tokenizer = M2M100Tokenizer.from_pretrained(model___name)