Upload 2 files

database.py

@@ -0,0 +1,175 @@
+from datasets import load_dataset
+import pandas as pd
+import torch
+from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import BertTokenizer, BertForQuestionAnswering, BertConfig,AutoModelForCausalLM
+from pymongo import MongoClient
+import torchtext
+torchtext.disable_torchtext_deprecation_warning()
+from torchtext.data import get_tokenizer
+from yeni_tokenize import TokenizerProcessor
+
+
+class Database:
+
+    # MongoDB connection settings
+
+    def get_mongodb(database_name='yeniDatabase', collection_name='test', host='localhost', port=27017):
+        """
+        MongoDB connection and collection selection
+        """
+        client = MongoClient(f'mongodb://{host}:{port}/')
+        db = client[database_name]
+        collection = db[collection_name]
+        return collection
+    
+    @staticmethod
+    def get_mongodb():
+        # MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmalıdır.
+        return 'mongodb://localhost:27017/', 'yeniDatabase', 'train'
+
+    @staticmethod
+    def get_input_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Prompt": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Prompt": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        input_texts_from_db =  [doc['Prompt'] for doc in cursor] 
+        # Input text'leri döndürme
+        # Düz metin listesine dönüştürme
+        return input_texts_from_db
+    input_text= get_input_texts()
+    print("metinler yazılıyor:")
+    for text in input_text:
+        print(text)
+
+    
+    @staticmethod
+    def get_output_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Response": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Response": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        output_texts_from_db = [doc['Response'] for doc in cursor]
+        #output metin listesine çevirme 
+        return output_texts_from_db
+
+    @staticmethod
+    def get_average_prompt_token_length():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Tüm dökümanları çekme ve 'prompt_token_length' alanını alma
+        docs = collection.find({}, {'Prompt_token_length': 1})
+        # 'prompt_token_length' değerlerini toplama ve sayma
+        total_length = 0
+        count = 0
+        for doc in docs:
+            if 'Prompt_token_length' in doc:
+                total_length += doc['Prompt_token_length']
+                count += 1
+        # Ortalama hesaplama
+        average_length = total_length / count if count > 0 else 0
+        return int(average_length)
+
+
+# Tokenizer ve Modeli yükleme
+"""
+class TokenizerProcessor:
+    def __init__(self, tokenizer_name='bert-base-uncased'):
+        self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name)
+
+    def tokenize_and_encode(self, input_texts, output_texts, max_length=100):
+        encoded = self.tokenizer.batch_encode_plus(
+            text_pair=list(zip(input_texts, output_texts)),
+            padding='max_length',
+            truncation=True,
+            max_length=max_length,
+            return_attention_mask=True,
+            return_tensors='pt'
+        )
+        return encoded
+
+    paraphrase = tokenizer.encode_plus(sequence_0, sequence_2, return_tensors="pt")
+not_paraphrase = tokenizer.encode_plus(sequence_0, sequence_1, return_tensors="pt")
+
+paraphrase_classification_logits = model(**paraphrase)[0]
+not_paraphrase_classification_logits = model(**not_paraphrase)[0]
+    def custom_padding(self, input_ids_list, max_length=100, pad_token_id=0):
+        padded_inputs = []
+        for ids in input_ids_list:
+            if len(ids) < max_length:
+                padded_ids = ids + [pad_token_id] * (max_length - len(ids))
+            else:
+                padded_ids = ids[:max_length]
+            padded_inputs.append(padded_ids)
+        return padded_inputs
+
+    def pad_and_truncate_pairs(self, input_texts, output_texts, max_length=100):
+
+        #input ve output verilerinin uzunluğunu eşitleme 
+        inputs = self.tokenizer(input_texts, padding=False, truncation=False, return_tensors=None)
+        outputs = self.tokenizer(output_texts, padding=False, truncation=False, return_tensors=None)
+        
+        input_ids = self.custom_padding(inputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        output_ids = self.custom_padding(outputs['input_ids'], max_length, self.tokenizer.pad_token_id)
+        
+        input_ids_tensor = torch.tensor(input_ids)
+        output_ids_tensor = torch.tensor(output_ids)
+        
+        input_attention_mask = (input_ids_tensor != self.tokenizer.pad_token_id).long()
+        output_attention_mask = (output_ids_tensor != self.tokenizer.pad_token_id).long()
+        
+        return {
+            'input_ids': input_ids_tensor,
+            'input_attention_mask': input_attention_mask,
+            'output_ids': output_ids_tensor,
+            'output_attention_mask': output_attention_mask
+        }
+    
+"""
+    #cümleleri teker teker input ve output verilerinden çekmem gerekiyor 
+    #def tokenize_and_pad_sequences(sequence_1,sequence2,)
+
+
+"""class DataPipeline:
+    def __init__(self, tokenizer_name='bert-base-uncased', max_length=100):
+        self.tokenizer_processor = TokenizerProcessor(tokenizer_name)
+        self.max_length = max_length
+
+    def prepare_data(self):
+        input_texts = Database.get_input_texts()
+        output_texts = Database.get_output_texts()
+        encoded_data = self.tokenizer_processor.pad_and_truncate_pairs(input_texts, output_texts, self.max_length)
+        return encoded_data
+    
+    def tokenize_texts(self, texts):
+        return [self.tokenize(text) for text in texts]
+    
+    def encode_texts(self, texts):
+        return [self.encode(text, self.max_length) for text in texts]
+
+# Example Usage
+if __name__ == "__main__":
+    data_pipeline = DataPipeline()
+    encoded_data = data_pipeline.prepare_data()
+    print(encoded_data)
+"""
+

t5.py

@@ -0,0 +1,13 @@
+
+# pip install accelerate
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+tokenizer = T5Tokenizer.from_pretrained("dbmdz/bert-base-turkish-cased")
+model = T5ForConditionalGeneration.from_pretrained("dbmdz/bert-base-turkish-cased", device_map="auto")
+
+
+input_text = "Başlık: Dijital Pazarlama, Alt başlıklar: dijital pazarlama nasıl yapılır?, dijital pazarlama ve gelişimi, dijital pazarlamanın iş hayatındaki yeri , anahtar kelimeler : pazarlama, ticaret, dijtalleşme, müşteri, ürün "
+input_ids = tokenizer(input_text, return_tensors="pt").input_ids.to("cuda")
+
+outputs = model.generate(input_ids)
+print(tokenizer.decode(outputs[0]))