Adding etl process and change fixed numbers into variables

src/config/optimization_config.py

@@ -45,24 +45,91 @@ def get_shift_list():
         return streamlit_shift_list
     except Exception as e:
         print(f"using default value for shift list")
-        shift_list = extract.read_shift_data()
-        shift_list = shift_list["shift"].unique()
+        shift_list = extract.read_shift_cost_data()
+        shift_list = shift_list["id"].unique()
         return shift_list
 SHIFT_LIST = get_shift_list()
 print(SHIFT_LIST)
 
 
-LINE_LIST = ["long", "short"]  # WH_Workforce_Hourly_Pay_Scale.csv -> This is default setting. User should be able to manipulate
+def get_line_list():
+    try:
+        streamlit_line_list = dashboard.line_list
+        return streamlit_line_list
+    except Exception as e:
+        print(f"using default value for line list")
+        line_df = extract.read_packaging_line_data()
+        line_list = line_df["id"].unique().tolist()
+        return line_list
+
+LINE_LIST = get_line_list()
+print(LINE_LIST)
+
+def get_line_cnt_per_type():
+    try:
+        streamlit_line_cnt_per_type = dashboard.line_cnt_per_type
+        return streamlit_line_cnt_per_type
+    except Exception as e:
+        print(f"using default value for line cnt per type")
+        line_df = extract.read_packaging_line_data()
+        line_cnt_per_type = line_df.set_index("id")["line_count"].to_dict()
+        print(line_cnt_per_type)
+        return line_cnt_per_type
+
+LINE_CNT_PER_TYPE = get_line_cnt_per_type()
+print(LINE_CNT_PER_TYPE)
+
+def get_demand_dictionary():
+    try:
+        streamlit_demand_dictionary = dashboard.demand_dictionary
+        return streamlit_demand_dictionary
+    except Exception as e:
+        print(f"using default value for demand dictionary")
+        demand_df = extract.read_demand_data()
+        print(demand_df)
+        demand_dictionary = demand_df.groupby('Material Number')["Order quantity (GMEIN)"].sum().to_dict()
+        return demand_dictionary
+
+DEMAND_DICTIONARY = get_demand_dictionary()
+print(DEMAND_DICTIONARY)
+
+def get_cost_list_per_emp_shift():
+    try:
+        streamlit_cost_list_per_emp_shift = dashboard.cost_list_per_emp_shift
+        return streamlit_cost_list_per_emp_shift
+    except Exception as e:
+        print(f"using default value for cost list per emp shift")
+        shift_cost_df = extract.read_shift_cost_data()
+        #question - Important : there is multiple type of employment type in terms of the cost
+        #1 -. unicef 2 - humanizer
+        return {1:{1:43,2:43,6:64},2:{1:27,2:27,6:41}}
+
+COST_LIST_PER_EMP_SHIFT = get_cost_list_per_emp_shift()
+print(COST_LIST_PER_EMP_SHIFT)
+        
+
+
+# COST_LIST_PER_EMP_SHIFT = {  # WH_Workforce_Hourly_Pay_Scale
+#     "Fixed": {1: 0, 2: 22, 3: 18},
+#     "Humanizer": {1: 10, 2: 10, 3: 10},
+# }
+
+def get_productivity_list_per_emp_product():
+    try:
+        streamlit_productivity_list_per_emp_product = dashboard.productivity_list_per_emp_product
+        return streamlit_productivity_list_per_emp_product
+    except Exception as e:
+        print(f"using default value for productivity list per emp product")
+        emp_product_df = extract.read_employee_productivity_data()
+        print(emp_product_df)
+        productivity_list_per_emp_product = emp_product_df.groupby("employment_type_id")["productivity"].sum().to_dict()
+        print(productivity_list_per_emp_product)
+        return 
+    
+
+PRODUCTIVITY_LIST_PER_EMP_PRODUCT = get_productivity_list_per_emp_product()
+print(PRODUCTIVITY_LIST_PER_EMP_PRODUCT)
 
-LINE_LIST_PER_TYPE = {
-    "long": 2,
-    "short": 3,
-}  # Work_centre_capacity but not enough on the number of line and exact number of capacity
-DEMAND_LIST = {"a": 1000, "b": 6000, "c": 4000}  # COOIS_Planned_and_Released.csv
-COST_LIST_PER_EMP_SHIFT = {  # WH_Workforce_Hourly_Pay_Scale
-    "Fixed": {1: 0, 2: 22, 3: 18},
-    "Humanizer": {1: 10, 2: 10, 3: 10},
-}
 PRODUCTIVITY_LIST_PER_EMP_PRODUCT = {  # Kits_Calculation
     "Fixed": {
         1: {"a": 1000, "b": 1000, "c": 1000},

src/etl/extract.py

@@ -29,8 +29,8 @@ def read_employee_data(
     return pd.read_csv(path)
 
 
-def read_shift_data(
-    path="data/real_data_excel/converted_csv/Work_Shifts_timing.csv",
+def read_shift_cost_data(
+    path="data/real_data_excel/converted_csv/work_shift.csv",
 ) -> pd.DataFrame:
     return pd.read_csv(path)
 
@@ -52,6 +52,14 @@ def read_material_master(
 ) -> pd.DataFrame:
     return pd.read_csv(path)
 
+def read_packaging_line_data(
+    path="data/real_data_excel/converted_csv/Work_Centre_Capacity_processed.csv",
+) -> pd.DataFrame:
+    df = pd.read_csv(path)
+    # Filter for packaging lines only
+    df = df[df["line_for_packaging"] == True]
+    return df
+
 
 def read_released_orders_data(
     path="data/real_data_excel/converted_csv/COOIS_Released_Prod_Orders.csv",