File size: 4,393 Bytes
a244ac5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 |
# Transform data for model training and evaluation.
# Python Libraries
import os
import sys
import pandas as pd
import numpy as np
from dataclasses import dataclass
# local Imports
from src.components.data_ingestion import DataIngestion
from src.exception import CustomException
from src.logger import logging
from src.utils import save_object
from src.components.model_trainer import ModelTrainer
# Sklearn Imports
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler
@dataclass
class DataTransformationConfig:
preprocessor_obj_file_path = os.path.join('artifacts', 'preprocessor.pkl')
class DataTransformation:
def __init__(self):
self.data_transformation_config = DataTransformationConfig()
def get_data_transformer_object(self):
try:
logging.info("Data Transformation initiated")
numerical_columns = ['reading score', 'writing score']
categorical_columns = ['gender', 'race/ethnicity', 'parental level of education', 'lunch', 'test preparation course']
num_pipeline = Pipeline(
steps=[
('imputer', SimpleImputer(strategy='median')),
('scaler', StandardScaler())
]
)
cat_pipeline = Pipeline(
steps=[
('imputer', SimpleImputer(strategy='most_frequent')),
('one_hot_encoder', OneHotEncoder()),
('scaler', StandardScaler(with_mean=False))
]
)
logging.info("Numerical and Categorical pipelines created")
preprocessor = ColumnTransformer(
[
('num_pipeline', num_pipeline, numerical_columns),
('cat_pipeline', cat_pipeline, categorical_columns)
]
)
return preprocessor
except Exception as e:
logging.error("Error in Data Transformation {0}".format(e))
raise CustomException(e, sys)
def initiate_data_transformation(self, train_path, test_path):
try:
# Reading train and test data
train_df = pd.read_csv(train_path)
test_df = pd.read_csv(test_path)
logging.info("Read train and test data completed")
logging.info("Obtaining preprocessor object")
preprocessing_obj = self.get_data_transformer_object()
target_column_name = "math score"
input_feature_train_df = train_df.drop(columns=[target_column_name], axis=1)
target_feature_train_df = train_df[target_column_name]
input_feature_test_df = test_df.drop(columns=[target_column_name], axis=1)
target_feature_test_df = test_df[target_column_name]
logging.info("Applying preprocessing object on training and testing dataframes")
input_feature_train_arr = preprocessing_obj.fit_transform(input_feature_train_df)
input_feature_test_arr = preprocessing_obj.transform(input_feature_test_df)
logging.info("Preprocessing completed successfully")
train_arr = np.c_[input_feature_train_arr, np.array(target_feature_train_df)]
test_arr = np.c_[input_feature_test_arr, np.array(target_feature_test_df)]
logging.info("Saved preprocessing object")
save_object(
file_path=self.data_transformation_config.preprocessor_obj_file_path,
obj=preprocessing_obj
)
return (
train_arr,
test_arr,
self.data_transformation_config.preprocessor_obj_file_path,
)
except Exception as e:
logging.error("Error in initiate_data_transformation {0}".format(e))
raise CustomException(e, sys)
if __name__ == "__main__":
obj = DataIngestion()
train_data, test_data = obj.initiate_data_ingestion()
data_transformation = DataTransformation()
train_arr , test_arr , preProcessorPath = data_transformation.initiate_data_transformation(train_data, test_data)
model_trainer = ModelTrainer()
print(model_trainer.initiate_model_trainer(train_arr , test_arr)) |