Upload 40 files

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+db/imp_db.db filter=lfs diff=lfs merge=lfs -text

.streamlit/config.toml

@@ -0,0 +1,7 @@
+
+[theme]
+primaryColor="#f6ad51"
+backgroundColor="#FFFFFF"
+secondaryBackgroundColor="#F0F2F6"
+textColor="#31333F"
+font="sans serif"

Home.py

@@ -0,0 +1,1614 @@
+# importing pacakages
+import streamlit as st
+
+st.set_page_config(layout="wide")
+from utilities import (
+    load_local_css,
+    set_header,
+    ensure_project_dct_structure,
+    store_hashed_password,
+    verify_password,
+    is_pswrd_flag_set,
+    set_pswrd_flag,
+)
+import os
+from datetime import datetime
+import pandas as pd
+import pickle
+import psycopg2
+
+#
+import numbers
+from collections import OrderedDict
+import re
+from ppt_utils import create_ppt
+from constants import default_dct
+import time
+from log_application import log_message, delete_old_log_files
+import sqlite3
+
+# setting page config
+load_local_css("styles.css")
+set_header()
+db_cred = None
+
+# --------------Functions----------------------#
+
+# # schema = db_cred["schema"]
+
+
+##API DATA#######################
+# Function to load gold layer data
+# @st.cache_data(show_spinner=False)
+def load_gold_layer_data(table_name):
+    # Fetch Table
+    query = f"""
+    SELECT * FROM {table_name};
+    """
+
+    # Execute the query and get the results
+    results = query_excecuter_postgres(
+        query, db_cred, insert=False, return_dataframe=True
+    )
+
+    if results is not None and not results.empty:
+        # Create a DataFrame
+        gold_layer_df = results
+
+    else:
+        st.warning("No data found for the selected table.")
+        st.stop()
+
+    # Columns to be removed
+    columns_to_remove = [
+        "clnt_nam",
+        "crte_dt_tm",
+        "crte_by_uid",
+        "updt_dt_tm",
+        "updt_by_uid",
+        "campgn_id",
+        "campgn_nam",
+        "ad_id",
+        "ad_nam",
+        "tctc_id",
+        "tctc_nam",
+        "campgn_grp_id",
+        "campgn_grp_nam",
+        "ad_grp_id",
+        "ad_grp_nam",
+    ]
+
+    # TEMP CODE
+    gold_layer_df = gold_layer_df.rename(
+        columns={
+            "imprssns_cnt": "mda_imprssns_cnt",
+            "clcks_cnt": "mda_clcks_cnt",
+            "vd_vws_cnt": "mda_vd_vws_cnt",
+        }
+    )
+
+    # Remove specific columns
+    gold_layer_df = gold_layer_df.drop(columns=columns_to_remove, errors="ignore")
+
+    # Convert columns to numeric or datetime as appropriate
+    for col in gold_layer_df.columns:
+        if (
+            col.startswith("rspns_mtrc_")
+            or col.startswith("mda_")
+            or col.startswith("exogenous_")
+            or col.startswith("internal_")
+            or col in ["spnd_amt"]
+        ):
+            gold_layer_df[col] = pd.to_numeric(gold_layer_df[col], errors="coerce")
+        elif col == "rcrd_dt":
+            gold_layer_df[col] = pd.to_datetime(gold_layer_df[col], errors="coerce")
+
+    # Replace columns starting with 'mda_' to 'media_'
+    gold_layer_df.columns = [
+        (col.replace("mda_", "media_") if col.startswith("mda_") else col)
+        for col in gold_layer_df.columns
+    ]
+
+    # Identify non-numeric columns
+    non_numeric_columns = gold_layer_df.select_dtypes(exclude=["number"]).columns
+    allow_non_numeric_columns = ["rcrd_dt", "aggrgtn_lvl", "sub_chnnl_nam", "panl_nam"]
+
+    # Remove non-numeric columns except for allowed non-numeric columns
+    non_numeric_columns_to_remove = [
+        col for col in non_numeric_columns if col not in allow_non_numeric_columns
+    ]
+    gold_layer_df = gold_layer_df.drop(
+        columns=non_numeric_columns_to_remove, errors="ignore"
+    )
+
+    # Remove specific columns
+    allow_columns = ["rcrd_dt", "aggrgtn_lvl", "sub_chnnl_nam", "panl_nam", "spnd_amt"]
+    for col in gold_layer_df.columns:
+        if (
+            col.startswith("rspns_mtrc_")
+            or col.startswith("media_")
+            or col.startswith("exogenous_")
+            or col.startswith("internal_")
+        ):
+            allow_columns.append(col)
+    gold_layer_df = gold_layer_df[allow_columns]
+
+    # Rename columns
+    gold_layer_df = gold_layer_df.rename(
+        columns={
+            "rcrd_dt": "date",
+            "sub_chnnl_nam": "channels",
+            "panl_nam": "panel",
+            "spnd_amt": "spends",
+        }
+    )
+
+    # Clean column values
+    gold_layer_df["panel"] = (
+        gold_layer_df["panel"].astype(str).str.lower().str.strip().str.replace(" ", "_")
+    )
+    gold_layer_df["channels"] = (
+        gold_layer_df["channels"]
+        .astype(str)
+        .str.lower()
+        .str.strip()
+        .str.replace(" ", "_")
+    )
+
+    # Replace columns starting with 'rspns_mtrc_' to 'response_metric_'
+    gold_layer_df.columns = [
+        (
+            col.replace("rspns_mtrc_", "response_metric_")
+            if col.startswith("rspns_mtrc_")
+            else col
+        )
+        for col in gold_layer_df.columns
+    ]
+
+    # Get the minimum date from the main dataframe
+    min_date = gold_layer_df["date"].min()
+
+    # Get maximum dates for daily and weekly data
+    max_date_daily = None
+    max_date_weekly = None
+
+    if not gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "daily"].empty:
+        max_date_daily = gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "daily"][
+            "date"
+        ].max()
+
+    if not gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "weekly"].empty:
+        max_date_weekly = gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "weekly"][
+            "date"
+        ].max() + pd.DateOffset(days=6)
+
+    # Determine final maximum date
+    if max_date_daily is not None and max_date_weekly is not None:
+        final_max_date = max(max_date_daily, max_date_weekly)
+    elif max_date_daily is not None:
+        final_max_date = max_date_daily
+    elif max_date_weekly is not None:
+        final_max_date = max_date_weekly
+
+    # Create a date range with daily frequency
+    date_range = pd.date_range(start=min_date, end=final_max_date, freq="D")
+
+    # Create a base DataFrame with all channels and all panels for each channel
+    unique_channels = gold_layer_df["channels"].unique()
+    unique_panels = gold_layer_df["panel"].unique()
+    base_data = [
+        (channel, panel, date)
+        for channel in unique_channels
+        for panel in unique_panels
+        for date in date_range
+    ]
+    base_df = pd.DataFrame(base_data, columns=["channels", "panel", "date"])
+
+    # Process weekly data to convert it to daily
+    if not gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "weekly"].empty:
+        weekly_data = gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "weekly"].copy()
+        daily_data = []
+
+        for index, row in weekly_data.iterrows():
+            week_start = pd.to_datetime(row["date"]) - pd.to_timedelta(
+                pd.to_datetime(row["date"]).weekday(), unit="D"
+            )
+            for i in range(7):
+                daily_date = week_start + pd.DateOffset(days=i)
+                new_row = row.copy()
+                new_row["date"] = daily_date
+                for col in new_row.index:
+                    if isinstance(new_row[col], numbers.Number):
+                        new_row[col] = new_row[col] / 7
+                daily_data.append(new_row)
+
+        daily_data_df = pd.DataFrame(daily_data)
+        daily_data_df["aggrgtn_lvl"] = "daily"
+        gold_layer_df = pd.concat(
+            [gold_layer_df[gold_layer_df["aggrgtn_lvl"] != "weekly"], daily_data_df],
+            ignore_index=True,
+        )
+
+    # Process monthly data to convert it to daily
+    if not gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "monthly"].empty:
+        monthly_data = gold_layer_df[gold_layer_df["aggrgtn_lvl"] == "monthly"].copy()
+        daily_data = []
+
+        for index, row in monthly_data.iterrows():
+            month_start = pd.to_datetime(row["date"]).replace(day=1)
+            next_month_start = (month_start + pd.DateOffset(months=1)).replace(day=1)
+            days_in_month = (next_month_start - month_start).days
+
+            for i in range(days_in_month):
+                daily_date = month_start + pd.DateOffset(days=i)
+                new_row = row.copy()
+                new_row["date"] = daily_date
+                for col in new_row.index:
+                    if isinstance(new_row[col], numbers.Number):
+                        new_row[col] = new_row[col] / days_in_month
+                daily_data.append(new_row)
+
+        daily_data_df = pd.DataFrame(daily_data)
+        daily_data_df["aggrgtn_lvl"] = "daily"
+        gold_layer_df = pd.concat(
+            [gold_layer_df[gold_layer_df["aggrgtn_lvl"] != "monthly"], daily_data_df],
+            ignore_index=True,
+        )
+
+    # Remove aggrgtn_lvl column
+    gold_layer_df = gold_layer_df.drop(columns=["aggrgtn_lvl"], errors="ignore")
+
+    # Group by 'panel', and 'date'
+    gold_layer_df = gold_layer_df.groupby(["channels", "panel", "date"]).sum()
+
+    # Merge gold_layer_df to base_df on channels, panel and date
+    gold_layer_df_cleaned = pd.merge(
+        base_df, gold_layer_df, on=["channels", "panel", "date"], how="left"
+    )
+
+    # Pivot the dataframe and rename columns
+    pivot_columns = [
+        col
+        for col in gold_layer_df_cleaned.columns
+        if col not in ["channels", "panel", "date"]
+    ]
+    gold_layer_df_cleaned = gold_layer_df_cleaned.pivot_table(
+        index=["date", "panel"], columns="channels", values=pivot_columns, aggfunc="sum"
+    ).reset_index()
+
+    # Flatten the columns
+    gold_layer_df_cleaned.columns = [
+        "_".join(col).strip() if col[1] else col[0]
+        for col in gold_layer_df_cleaned.columns.values
+    ]
+
+    # Replace columns ending with '_all' to '_total'
+    gold_layer_df_cleaned.columns = [
+        col.replace("_all", "_total") if col.endswith("_all") else col
+        for col in gold_layer_df_cleaned.columns
+    ]
+
+    # Clean panel column values
+    gold_layer_df_cleaned["panel"] = (
+        gold_layer_df_cleaned["panel"]
+        .astype(str)
+        .str.lower()
+        .str.strip()
+        .str.replace(" ", "_")
+    )
+
+    # Drop all columns that end with '_total' except those starting with 'response_metric_'
+    cols_to_drop = [
+        col
+        for col in gold_layer_df_cleaned.columns
+        if col.endswith("_total") and not col.startswith("response_metric_")
+    ]
+    gold_layer_df_cleaned.drop(columns=cols_to_drop, inplace=True)
+
+    return gold_layer_df_cleaned
+
+
+def check_valid_name():
+    if (
+        not st.session_state["project_name_box"]
+        .lower()
+        .startswith(defualt_project_prefix)
+    ):
+        st.session_state["disable_create_project"] = True
+        with warning_box:
+            st.warning("Project Name should follow naming conventions")
+            st.session_state["warning"] = (
+                "Project Name should follow naming conventions!"
+            )
+
+        with warning_box:
+            st.warning("Project Name should follow naming conventions")
+            st.button("Reset Name", on_click=reset_project_text_box, key="2")
+
+    if st.session_state["project_name_box"] == defualt_project_prefix:
+        with warning_box:
+            st.warning("Cannot Name only with Prefix")
+            st.session_state["warning"] = "Cannot Name only with Prefix"
+        st.session_state["disable_create_project"] = True
+
+    if st.session_state["project_name_box"] in user_projects:
+        with warning_box:
+            st.warning("Project already exists please enter new name")
+            st.session_state["warning"] = "Project already exists please enter new name"
+        st.session_state["disable_create_project"] = True
+    else:
+        st.session_state["disable_create_project"] = False
+
+
+def query_excecuter_postgres(
+    query,
+    db_path=None,
+    params=None,
+    insert=True,
+    insert_retrieve=False,
+    db_cred=None,
+):
+    """
+    Executes a SQL query on a SQLite database, handling both insert and select operations.
+
+    Parameters:
+    query (str): The SQL query to be executed.
+    db_path (str): Path to the SQLite database file.
+    params (tuple, optional): Parameters to pass into the SQL query for parameterized execution.
+    insert (bool, default=True): Flag to determine if the query is an insert operation (default) or a select operation.
+    insert_retrieve (bool, default=False): Flag to determine if the query should insert and then return the inserted ID.
+
+    """
+    try:
+        # Construct a cross-platform path to the database
+        db_dir = os.path.join("db")
+        os.makedirs(db_dir, exist_ok=True)  # Make sure the directory exists
+        db_path = os.path.join(db_dir, "imp_db.db")
+
+        # Establish connection to the SQLite database
+        conn = sqlite3.connect(db_path)
+    except sqlite3.Error as e:
+        st.warning(f"Unable to connect to the SQLite database: {e}")
+        st.stop()
+
+    # Create a cursor object to interact with the database
+    c = conn.cursor()
+
+    try:
+        # Execute the query with or without parameters
+        if params:
+            params = tuple(params)
+            query = query.replace("IN (?)", f"IN ({','.join(['?' for _ in params])})")
+            c.execute(query, params)
+        else:
+            c.execute(query)
+
+        if not insert:
+            # If not an insert operation, fetch and return the results
+            results = c.fetchall()
+            return results
+        elif insert_retrieve:
+            # If insert and retrieve operation, commit and return the last inserted row ID
+            conn.commit()
+            return c.lastrowid
+        else:
+            # For standard insert operations, commit the transaction
+            conn.commit()
+
+    except Exception as e:
+        st.write(f"Error executing query: {e}")
+    finally:
+        conn.close()
+
+
+# Function to check if the input contains any SQL keywords
+def contains_sql_keywords_check(user_input):
+
+    sql_keywords = [
+        "SELECT",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "DROP",
+        "ALTER",
+        "CREATE",
+        "GRANT",
+        "REVOKE",
+        "UNION",
+        "JOIN",
+        "WHERE",
+        "HAVING",
+        "EXEC",
+        "TRUNCATE",
+        "REPLACE",
+        "MERGE",
+        "DECLARE",
+        "SHOW",
+        "FROM",
+    ]
+
+    pattern = "|".join(re.escape(keyword) for keyword in sql_keywords)
+    return re.search(pattern, user_input, re.IGNORECASE)
+
+
+# def get_table_names(schema):
+#     query = f"""
+#     SELECT table_name
+#     FROM information_schema.tables
+#     WHERE table_schema = '{schema}'
+#     AND table_type = 'BASE TABLE'
+#     AND table_name LIKE '%_mmo_gold';
+#     """
+#     table_names = query_excecuter_postgres(query, db_cred, insert=False)
+#     table_names = [table[0] for table in table_names]
+
+#     return table_names
+
+
+def update_summary_df():
+    """
+    Updates the 'project_summary_df' in the session state with the latest project
+    summary information based on the most recent updates.
+
+    This function executes a SQL query to retrieve project metadata from a database
+    and stores the result in the session state.
+
+    Uses:
+    - query_excecuter_postgres(query, params=params, insert=False): A function that
+      executes the provided SQL query on a PostgreSQL database.
+
+    Modifies:
+    - st.session_state['project_summary_df']: Updates the dataframe with columns:
+      'Project Number', 'Project Name', 'Last Modified Page', 'Last Modified Time'.
+    """
+
+    query = f"""
+            WITH LatestUpdates AS (
+                SELECT
+                    prj_id,
+                    page_nam,
+                    updt_dt_tm,
+                    ROW_NUMBER() OVER (PARTITION BY prj_id ORDER BY updt_dt_tm DESC) AS rn
+                FROM
+                    mmo_project_meta_data
+            )
+            SELECT
+                p.prj_id,
+                p.prj_nam AS prj_nam,
+                lu.page_nam,
+                lu.updt_dt_tm
+            FROM
+                LatestUpdates lu
+            RIGHT JOIN
+                mmo_projects p ON lu.prj_id = p.prj_id
+            WHERE
+                p.prj_ownr_id = ? AND lu.rn = 1
+            """
+
+    params = (st.session_state["emp_id"],)  # Parameters for the SQL query
+
+    # Execute the query and retrieve project summary data
+    project_summary = query_excecuter_postgres(
+        query, db_cred, params=params, insert=False
+    )
+
+    # Update the session state with the project summary dataframe
+    st.session_state["project_summary_df"] = pd.DataFrame(
+        project_summary,
+        columns=[
+            "Project Number",
+            "Project Name",
+            "Last Modified Page",
+            "Last Modified Time",
+        ],
+    )
+
+    st.session_state["project_summary_df"] = st.session_state[
+        "project_summary_df"
+    ].sort_values(by=["Last Modified Time"], ascending=False)
+
+
+def reset_project_text_box():
+    st.session_state["project_name_box"] = defualt_project_prefix
+    st.session_state["disable_create_project"] = True
+
+
+def query_excecuter_sqlite(
+    insert_projects_query,
+    insert_meta_data_query,
+    db_path=None,
+    params_projects=None,
+    params_meta=None,
+):
+    """
+    Executes the project insert and associated metadata insert in an SQLite database.
+
+    Parameters:
+    insert_projects_query (str): SQL query for inserting into the mmo_projects table.
+    insert_meta_data_query (str): SQL query for inserting into the mmo_project_meta_data table.
+    db_path (str): Path to the SQLite database file.
+    params_projects (tuple, optional): Parameters for the mmo_projects table insert.
+    params_meta (tuple, optional): Parameters for the mmo_project_meta_data table insert.
+
+    Returns:
+    bool: True if successful, False otherwise.
+    """
+    try:
+        # Construct a cross-platform path to the database
+        db_dir = os.path.join("db")
+        os.makedirs(db_dir, exist_ok=True)  # Make sure the directory exists
+        db_path = os.path.join(db_dir, "imp_db.db")
+
+        # Establish connection to the SQLite database
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        # Execute the first insert query into the mmo_projects table
+        cursor.execute(insert_projects_query, params_projects)
+
+        # Get the last inserted project ID
+        prj_id = cursor.lastrowid
+
+        # Modify the parameters for the metadata table with the inserted prj_id
+        params_meta = (prj_id,) + params_meta
+
+        # Execute the second insert query into the mmo_project_meta_data table
+        cursor.execute(insert_meta_data_query, params_meta)
+
+        # Commit the transaction
+        conn.commit()
+
+    except sqlite3.Error as e:
+        st.warning(f"Error executing query: {e}")
+        return False
+    finally:
+        # Close the connection
+        conn.close()
+
+    return True
+
+
+def new_project():
+    """
+    Cleans the project name input and inserts project data into the SQLite database,
+    updating session state and triggering UI rerun if successful.
+    """
+
+    # Define a dictionary containing project data
+    project_dct = default_dct.copy()
+
+    gold_layer_df = pd.DataFrame()
+    if str(api_name).strip().lower() != "na":
+        try:
+            gold_layer_df = load_gold_layer_data(api_name)
+        except Exception as e:
+            st.toast(
+                "Failed to load gold layer data. Please check the gold layer structure and connection.",
+                icon="⚠️",
+            )
+            log_message(
+                "error",
+                f"Error loading gold layer data: {str(e)}",
+                "Home",
+            )
+
+    project_dct["data_import"]["gold_layer_df"] = gold_layer_df
+
+    # Get current time for database insertion
+    inserted_time = datetime.now().isoformat()
+
+    # Define SQL queries for inserting project and metadata into the SQLite database
+    insert_projects_query = """
+        INSERT INTO mmo_projects (prj_ownr_id, prj_nam, alwd_emp_id, crte_dt_tm, crte_by_uid)
+        VALUES (?, ?, ?, ?, ?);
+    """
+    insert_meta_data_query = """
+        INSERT INTO mmo_project_meta_data (prj_id, page_nam, file_nam, pkl_obj, crte_dt_tm, crte_by_uid, updt_dt_tm)
+        VALUES (?, ?, ?, ?, ?, ?, ?);
+    """
+
+    # Get current time for metadata update
+    updt_dt_tm = datetime.now().isoformat()
+
+    # Serialize project_dct using pickle
+    project_pkl = pickle.dumps(project_dct)
+
+    # Prepare data for database insertion
+    projects_data = (
+        st.session_state["emp_id"],  # prj_ownr_id
+        project_name,  # prj_nam
+        ",".join(matching_user_id),  # alwd_emp_id
+        inserted_time,  # crte_dt_tm
+        st.session_state["emp_id"],  # crte_by_uid
+    )
+
+    project_meta_data = (
+        "Home",  # page_nam
+        "project_dct",  # file_nam
+        project_pkl,  # pkl_obj
+        inserted_time,  # crte_dt_tm
+        st.session_state["emp_id"],  # crte_by_uid
+        updt_dt_tm,  # updt_dt_tm
+    )
+
+    # Execute the insertion query for SQLite
+    success = query_excecuter_sqlite(
+        insert_projects_query,
+        insert_meta_data_query,
+        params_projects=projects_data,
+        params_meta=project_meta_data,
+    )
+
+    if success:
+        st.success("Project Created")
+        update_summary_df()
+    else:
+        st.error("Failed to create project.")
+
+
+def validate_password(user_input):
+    # List of SQL keywords to check for
+    sql_keywords = [
+        "SELECT",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "DROP",
+        "ALTER",
+        "CREATE",
+        "GRANT",
+        "REVOKE",
+        "UNION",
+        "JOIN",
+        "WHERE",
+        "HAVING",
+        "EXEC",
+        "TRUNCATE",
+        "REPLACE",
+        "MERGE",
+        "DECLARE",
+        "SHOW",
+        "FROM",
+    ]
+
+    # Create a regex pattern for SQL keywords
+    pattern = "|".join(re.escape(keyword) for keyword in sql_keywords)
+
+    # Check if input contains any SQL keywords
+    if re.search(pattern, user_input, re.IGNORECASE):
+        return "SQL keyword detected."
+
+    # Password validation criteria
+    if len(user_input) < 8:
+        return "Password should be at least 8 characters long."
+    if not re.search(r"[A-Z]", user_input):
+        return "Password should contain at least one uppercase letter."
+    if not re.search(r"[0-9]", user_input):
+        return "Password should contain at least one digit."
+    if not re.search(r"[a-z]", user_input):
+        return "Password should contain at least one lowercase letter."
+    if not re.search(r'[!@#$%^&*(),.?":{}|<>]', user_input):
+        return "Password should contain at least one special character."
+
+    # If all checks pass
+    return "Valid input."
+
+
+def fetch_and_process_projects(emp_id):
+    query = f"""
+    WITH ProjectAccess AS (
+        SELECT 
+            p.prj_id,
+            p.prj_nam,
+            p.alwd_emp_id,
+            u.emp_nam AS project_owner
+        FROM mmo_projects p
+        JOIN mmo_users u ON p.prj_ownr_id = u.emp_id
+    )
+    SELECT
+        pa.prj_id,
+        pa.prj_nam,
+        pa.project_owner
+    FROM
+        ProjectAccess pa
+    WHERE
+        pa.alwd_emp_id LIKE ?
+    ORDER BY
+        pa.prj_id;
+    """
+
+    params = (f"%{emp_id}%",)
+    results = query_excecuter_postgres(query, db_cred, params=params, insert=False)
+
+    # Process the results to create the desired dictionary structure
+    clone_project_dict = {}
+    for row in results:
+        project_id, project_name, project_owner = row
+
+        if project_owner not in clone_project_dict:
+            clone_project_dict[project_owner] = []
+
+        clone_project_dict[project_owner].append(
+            {"project_name": project_name, "project_id": project_id}
+        )
+
+    return clone_project_dict
+
+
+def get_project_id_from_dict(projects_dict, owner_name, project_name):
+    if owner_name in projects_dict:
+        for project in projects_dict[owner_name]:
+            if project["project_name"] == project_name:
+                return project["project_id"]
+    return None
+
+
+# def fetch_project_metadata(prj_id):
+#     query = f"""
+#     SELECT
+#         prj_id, page_nam, file_nam, pkl_obj, dshbrd_ts
+#     FROM
+#         mmo_project_meta_data
+#     WHERE
+#         prj_id = ?;
+#     """
+
+#     params = (prj_id,)
+#     return query_excecuter_postgres(query, db_cred, params=params, insert=False)
+
+
+def fetch_project_metadata(prj_id):
+    # Query to select project metadata
+    query = """
+    SELECT
+        prj_id, page_nam, file_nam, pkl_obj, dshbrd_ts
+    FROM
+        mmo_project_meta_data
+    WHERE
+        prj_id = ?;
+    """
+
+    params = (prj_id,)
+    return query_excecuter_postgres(query, db_cred, params=params, insert=False)
+
+
+# def create_new_project(prj_ownr_id, prj_nam, alwd_emp_id, emp_id):
+#     query = f"""
+#     INSERT INTO mmo_projects (prj_ownr_id, prj_nam, alwd_emp_id, crte_by_uid, crte_dt_tm)
+#     VALUES (?, ?, ?, ?, NOW())
+#     RETURNING prj_id;
+#     """
+
+#     params = (prj_ownr_id, prj_nam, alwd_emp_id, emp_id)
+#     result = query_excecuter_postgres(
+#         query, db_cred, params=params, insert=True, insert_retrieve=True
+#     )
+#     return result[0][0]
+
+
+# def create_new_project(prj_ownr_id, prj_nam, alwd_emp_id, emp_id):
+#     # Query to insert a new project
+#     insert_query = """
+#     INSERT INTO mmo_projects (prj_ownr_id, prj_nam, alwd_emp_id, crte_by_uid, crte_dt_tm)
+#     VALUES (?, ?, ?, ?, DATETIME('now'));
+#     """
+
+#     params = (prj_ownr_id, prj_nam, alwd_emp_id, emp_id)
+
+#     # Execute the insert query
+#     query_excecuter_postgres(insert_query, db_cred, params=params, insert=True)
+
+#     # Retrieve the last inserted prj_id
+#     retrieve_id_query = "SELECT last_insert_rowid();"
+#     result = query_excecuter_postgres(retrieve_id_query, db_cred, insert_retrieve=True)
+
+#     return result[0][0]
+
+
+def create_new_project(prj_ownr_id, prj_nam, alwd_emp_id, emp_id):
+    # Query to insert a new project
+    insert_query = """
+    INSERT INTO mmo_projects (prj_ownr_id, prj_nam, alwd_emp_id, crte_by_uid, crte_dt_tm)
+    VALUES (?, ?, ?, ?, DATETIME('now'));
+    """
+    params = (prj_ownr_id, prj_nam, alwd_emp_id, emp_id)
+
+    # Execute the insert query and retrieve the last inserted prj_id directly
+    last_inserted_id = query_excecuter_postgres(
+        insert_query, params=params, insert_retrieve=True
+    )
+
+    return last_inserted_id
+
+
+def insert_project_metadata(new_prj_id, metadata, created_emp_id):
+    # query = f"""
+    # INSERT INTO mmo_project_meta_data (
+    #     prj_id, page_nam, crte_dt_tm, file_nam, pkl_obj, dshbrd_ts, crte_by_uid
+    # )
+    # VALUES (?, ?, NOW(), ?, ?, ?, ?);
+    # """
+
+    query = """
+    INSERT INTO mmo_project_meta_data (
+        prj_id, page_nam, crte_dt_tm, file_nam, pkl_obj, dshbrd_ts, crte_by_uid
+    )
+    VALUES (?, ?, DATETIME('now'), ?, ?, ?, ?);
+    """
+
+    for row in metadata:
+        params = (new_prj_id, row[1], row[2], row[3], row[4], created_emp_id)
+        query_excecuter_postgres(query, db_cred, params=params, insert=True)
+
+
+# def delete_projects_by_ids(prj_ids):
+#     # Ensure prj_ids is a tuple to use with the IN clause
+#     prj_ids_tuple = tuple(prj_ids)
+
+#     # Query to delete project metadata
+#     delete_metadata_query = f"""
+#     DELETE FROM mmo_project_meta_data
+#     WHERE prj_id IN ?;
+#     """
+
+#     delete_projects_query = f"""
+#     DELETE FROM mmo_projects
+#     WHERE prj_id IN ?;
+#     """
+
+#     try:
+#         # Delete from metadata table
+#         query_excecuter_postgres(
+#             delete_metadata_query, db_cred, params=(prj_ids_tuple,), insert=True
+#         )
+
+#         # Delete from projects table
+#         query_excecuter_postgres(
+#             delete_projects_query, db_cred, params=(prj_ids_tuple,), insert=True
+#         )
+
+#     except Exception as e:
+#         st.write(f"Error deleting projects: {e}")
+
+
+def delete_projects_by_ids(prj_ids):
+    # Ensure prj_ids is a tuple to use with the IN clause
+    prj_ids_tuple = tuple(prj_ids)
+
+    # Dynamically generate placeholders for SQLite
+    placeholders = ", ".join(["?"] * len(prj_ids_tuple))
+
+    # Query to delete project metadata with dynamic placeholders
+    delete_metadata_query = f"""
+    DELETE FROM mmo_project_meta_data
+    WHERE prj_id IN ({placeholders});
+    """
+
+    delete_projects_query = f"""
+    DELETE FROM mmo_projects
+    WHERE prj_id IN ({placeholders});
+    """
+
+    try:
+        # Delete from metadata table
+        query_excecuter_postgres(
+            delete_metadata_query, db_cred, params=prj_ids_tuple, insert=True
+        )
+
+        # Delete from projects table
+        query_excecuter_postgres(
+            delete_projects_query, db_cred, params=prj_ids_tuple, insert=True
+        )
+
+    except Exception as e:
+        st.write(f"Error deleting projects: {e}")
+
+
+def fetch_users_with_access(prj_id):
+    # Query to get allowed employee IDs for the project
+    get_allowed_emps_query = """
+    SELECT alwd_emp_id
+    FROM mmo_projects
+    WHERE prj_id = ?;
+    """
+
+    # Fetch the allowed employee IDs
+    allowed_emp_ids_result = query_excecuter_postgres(
+        get_allowed_emps_query, db_cred, params=(prj_id,), insert=False
+    )
+
+    if not allowed_emp_ids_result:
+        return []
+
+    # Extract the allowed employee IDs (Assuming alwd_emp_id is a comma-separated string)
+    allowed_emp_ids_str = allowed_emp_ids_result[0][0]
+    allowed_emp_ids = allowed_emp_ids_str.split(",")
+
+    # Convert to tuple for the IN clause
+    allowed_emp_ids_tuple = tuple(allowed_emp_ids)
+
+    # Query to get user details for the allowed employee IDs
+    get_users_query = """
+    SELECT emp_id, emp_nam, emp_typ
+    FROM mmo_users
+    WHERE emp_id IN ({});
+    """.format(
+        ",".join("?" * len(allowed_emp_ids_tuple))
+    )  # Dynamically construct the placeholder list
+
+    # Fetch user details
+    user_details = query_excecuter_postgres(
+        get_users_query, db_cred, params=allowed_emp_ids_tuple, insert=False
+    )
+
+    return user_details
+
+
+# def update_project_access(prj_id, user_names, new_user_ids):
+#     # Convert the list of new user IDs to a comma-separated string
+#     new_user_ids_str = ",".join(new_user_ids)
+
+#     # Query to update the alwd_emp_id for the specified project
+#     update_access_query = f"""
+#     UPDATE mmo_projects
+#     SET alwd_emp_id = ?
+#     WHERE prj_id = ?;
+#     """
+
+#     # Execute the update query
+#     query_excecuter_postgres(
+#         update_access_query, db_cred, params=(new_user_ids_str, prj_id), insert=True
+#     )
+#     st.success(f"Project {prj_id} access updated successfully")
+
+
+def fetch_user_ids_from_dict(user_dict, user_names):
+    user_ids = []
+    # Iterate over the user_dict to find matching user names
+    for user_id, details in user_dict.items():
+        if details[0] in user_names:
+            user_ids.append(user_id)
+    return user_ids
+
+
+def update_project_access(prj_id, user_names, user_dict):
+    # Fetch the new user IDs based on the provided user names from the dictionary
+    new_user_ids = fetch_user_ids_from_dict(user_dict, user_names)
+
+    # Convert the list of new user IDs to a comma-separated string
+    new_user_ids_str = ",".join(new_user_ids)
+
+    # Query to update the alwd_emp_id for the specified project
+    update_access_query = f"""
+    UPDATE mmo_projects
+    SET alwd_emp_id = ?
+    WHERE prj_id = ?;
+    """
+
+    # Execute the update query
+    query_excecuter_postgres(
+        update_access_query, db_cred, params=(new_user_ids_str, prj_id), insert=True
+    )
+    st.write(f"Project {prj_id} access updated successfully")
+
+
+def validate_emp_id():
+
+    if st.session_state.sign_up not in st.session_state["unique_ids"].keys():
+        st.warning("You dont have access to the tool please contact admin")
+
+
+# -------------------Front END-------------------------#
+
+st.header("Manage Projects")
+
+unique_users_query = f"""
+                    SELECT DISTINCT emp_id, emp_nam, emp_typ
+                    FROM mmo_users;
+                    """
+
+if "unique_ids" not in st.session_state:
+
+    unique_users_result = query_excecuter_postgres(
+        unique_users_query, db_cred, insert=False
+    )  # retrieves all the users who has access to MMO TOOL
+
+    if len(unique_users_result) == 0:
+        st.warning("No users data present in db, please contact admin!")
+        st.stop()
+
+    st.session_state["unique_ids"] = {
+        emp_id: (emp_nam, emp_type) for emp_id, emp_nam, emp_type in unique_users_result
+    }
+
+
+if "toggle" not in st.session_state:
+    st.session_state["toggle"] = 0
+
+
+if "emp_id" not in st.session_state:
+    reset_password = st.radio(
+        "Select An Option",
+        options=["Login", "Reset Password"],
+        index=st.session_state["toggle"],
+        horizontal=True,
+    )
+
+    if reset_password == "Login":
+        emp_id = st.text_input("Employee id").lower()  # emp id
+        password = st.text_input("Password", max_chars=15, type="password")
+        login_button = st.button("Login", use_container_width=True)
+
+    else:
+
+        emp_id = st.text_input(
+            "Employee id", key="sign_up", on_change=validate_emp_id
+        ).lower()
+
+        current_password = st.text_input(
+            "Enter Current Password  and Press Enter to Validate",
+            max_chars=15,
+            type="password",
+            key="current_password",
+        )
+        if emp_id:
+
+            if emp_id not in st.session_state["unique_ids"].keys():
+                st.write("Invalid id!")
+                st.stop()
+            else:
+                if not is_pswrd_flag_set(emp_id):
+
+                    if verify_password(emp_id, current_password):
+                        st.success("Your password key has been successfully validated!")
+
+                    elif (
+                        not verify_password(emp_id, current_password)
+                        and len(current_password) > 1
+                    ):
+                        st.write("Wrong Password Key Please Try Again")
+                        st.stop()
+
+                elif verify_password(emp_id, current_password):
+                    st.success("Your password  has been successfully validated!")
+
+                elif (
+                    not verify_password(emp_id, current_password)
+                    and len(current_password) > 1
+                ):
+                    st.write("Wrong Password Please Try Again")
+                    st.stop()
+
+        new_password = st.text_input(
+            "Enter New Password", max_chars=15, type="password", key="new_password"
+        )
+
+        st.markdown(
+            "**Password must be at least 8 to 15 characters long and contain at least one uppercase letter, one lowercase letter, one digit, and one special character. No SQL commands allowed.**"
+        )
+
+        validation_result = validate_password(new_password)
+
+        confirm_new_password = st.text_input(
+            "Confirm New Password",
+            max_chars=15,
+            type="password",
+            key="confirm_new_password",
+        )
+
+        reset_button = st.button("Reset Password", use_container_width=True)
+
+        if reset_button:
+
+            validation_result = validate_password(new_password)
+
+            if validation_result != "Valid input.":
+                st.warning(validation_result)
+                st.stop()
+            elif new_password != confirm_new_password:
+                st.warning(
+                    "The new password and confirmation password do not match. Please try again."
+                )
+                st.stop()
+            else:
+                store_hashed_password(emp_id, confirm_new_password)
+                set_pswrd_flag(emp_id)
+                st.success("Password Reset Successful!")
+
+                with st.spinner("Redirecting to Login"):
+                    time.sleep(3)
+                st.session_state["toggle"] = 0
+                st.rerun()
+
+        st.stop()
+
+    if login_button:
+
+        if emp_id not in st.session_state["unique_ids"].keys() or len(password) == 0:
+            st.warning("invalid id or password!")
+
+            st.stop()
+
+        if not is_pswrd_flag_set(emp_id):
+            st.warning("Reset password to continue")
+            with st.spinner("Redirecting"):
+                st.session_state["toggle"] = 1
+                time.sleep(2)
+            st.rerun()
+            st.stop()
+
+        elif verify_password(emp_id, password):
+            with st.spinner("Loading Saved Projects"):
+                st.session_state["emp_id"] = emp_id
+
+                update_summary_df()  # function call to fetch user saved projects
+
+                st.session_state["clone_project_dict"] = fetch_and_process_projects(
+                    st.session_state["emp_id"]
+                )
+                if "project_dct" in st.session_state:
+                    del st.session_state["project_dct"]
+
+                st.session_state["project_name"] = None
+
+            delete_old_log_files()
+
+            st.rerun()
+
+            if (
+                len(st.session_state["emp_id"]) == 0
+                or st.session_state["emp_id"]
+                not in st.session_state["unique_ids"].keys()
+            ):
+                st.stop()
+        else:
+            st.warning("Invalid user name or password")
+
+    st.stop()
+
+if st.button("Logout"):
+    if "emp_id" in st.session_state:
+        del st.session_state["emp_id"]
+    st.rerun()
+
+if st.session_state["emp_id"] in st.session_state["unique_ids"].keys():
+
+    if "project_name" not in st.session_state:
+        st.session_state["project_name"] = None
+
+    cols1 = st.columns([2, 1])
+
+    st.session_state["username"] = st.session_state["unique_ids"][
+        st.session_state["emp_id"]
+    ][0]
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+    st.markdown(
+        """
+                Enter project number in the text below and click on load project to load the project.
+
+                """
+    )
+
+    st.markdown("Select Project")
+
+    # st.write(type(st.session_state.keys))
+
+    if len(st.session_state["project_summary_df"]) != 0:
+
+        # Display an editable data table using Streamlit's data editor component
+
+        table = st.dataframe(
+            st.session_state["project_summary_df"],
+            use_container_width=True,
+            hide_index=True,
+        )
+
+        project_number = st.selectbox(
+            "Enter Project number",
+            options=st.session_state["project_summary_df"]["Project Number"],
+        )
+
+        log_message(
+            "info",
+            f"Project number {project_number} selected by employee {st.session_state['emp_id']}.",
+            "Home",
+        )
+
+        project_col = st.columns(2)
+
+        # if "load_project_key" not in st.session_state:
+        #     st.session_state["load_project_key"] = None\
+
+        def load_project_fun():
+            st.session_state["project_name"] = (
+                st.session_state["project_summary_df"]
+                .loc[
+                    st.session_state["project_summary_df"]["Project Number"]
+                    == project_number,
+                    "Project Name",
+                ]
+                .values[0]
+            )  # fetching project name from project number stored in summary df
+
+            project_dct_query = f"""
+            SELECT pkl_obj 
+            FROM mmo_project_meta_data 
+            WHERE prj_id = ? AND file_nam = ?;
+            """
+            # Execute the query and retrieve the result
+            project_dct_retrieved = query_excecuter_postgres(
+                project_dct_query,
+                db_cred,
+                params=(project_number, "project_dct"),
+                insert=False,
+            )
+            # retrieves project dict (meta data)  stored in db
+
+            st.session_state["project_dct"] = pickle.loads(
+                project_dct_retrieved[0][0]
+            )  # converting bytes data to original objet using pickle
+            st.session_state["project_number"] = project_number
+
+            keys_to_keep = [
+                "unique_ids",
+                "emp_id",
+                "project_dct",
+                "project_name",
+                "project_number",
+                "username",
+                "project_summary_df",
+                "clone_project_dict",
+            ]
+
+            # Clear all keys in st.session_state except the ones to keep
+            for key in list(st.session_state.keys()):
+                if key not in keys_to_keep:
+                    del st.session_state[key]
+
+            ensure_project_dct_structure(st.session_state["project_dct"], default_dct)
+
+        if st.button(
+            "Load Project",
+            use_container_width=True,
+            key="load_project_key",
+            on_click=load_project_fun,
+        ):
+            st.success("Project Loded")
+
+            # st.rerun()  # refresh the page
+
+            # st.write(st.session_state['project_dct'])
+    if "radio_box_index" not in st.session_state:
+        st.session_state["radio_box_index"] = 0
+
+    projct_radio = st.radio(
+        "Select Options",
+        [
+            "Create New Project",
+            "Modify Project Access",
+            "Clone Saved Projects",
+            "Delete Projects",
+        ],
+        horizontal=True,
+        index=st.session_state["radio_box_index"],
+    )
+
+    if projct_radio == "Modify Project Access":
+
+        with st.expander("Modify Project Access"):
+            project_number_for_access = st.selectbox(
+                "Select Project Number",
+                st.session_state["project_summary_df"]["Project Number"],
+            )
+
+            with st.spinner("Loading"):
+                users_who_has_access = fetch_users_with_access(
+                    project_number_for_access
+                )
+
+            users_name_who_has_access = [user[1] for user in users_who_has_access]
+            modified_users_for_access_options = [
+                details[0]
+                for user_id, details in st.session_state["unique_ids"].items()
+                if user_id != st.session_state["emp_id"]
+            ]
+
+            users_name_who_has_access = [
+                name
+                for name in users_name_who_has_access
+                if name in modified_users_for_access_options
+            ]
+
+            modified_users_for_access = st.multiselect(
+                "Select or deselect users to grant or revoke access, then click the 'Modify Access' button to submit changes.",
+                options=modified_users_for_access_options,
+                default=users_name_who_has_access,
+            )
+
+            if st.button("Modify Access", use_container_width=True):
+                with st.spinner("Modifying Access"):
+                    update_project_access(
+                        project_number_for_access,
+                        modified_users_for_access,
+                        st.session_state["unique_ids"],
+                    )
+
+    if projct_radio == "Create New Project":
+
+        with st.expander("Create New Project", expanded=False):
+
+            st.session_state["is_create_project_open"] = True
+
+            unique_users = [
+                user[0] for user in st.session_state["unique_ids"].values()
+            ]  # fetching unique users who has access to the tool
+
+            user_projects = list(
+                set(st.session_state["project_summary_df"]["Project Name"])
+            )  # fetching corressponding user's projects
+            st.markdown(
+                """
+                To create a new project, follow the instructions below:
+                
+                1. **Project Name**:
+                    - It should start with the client name, followed by the username.
+                    - It should not contain special characters except for underscores (`_`) and should not contain spaces.
+                    - Example format: `<client_name>_<username>_<project_name>`
+                
+                2. **Select User**: Select the user you want to give access to this project.
+                
+                3. **Create New Project**: Click **Create New Project** once the above details are entered.
+                
+                **Example**:
+                
+                - For a client named "ClientA" and a user named "UserX" with a project named "NewCampaign", the project name should be:
+                `ClientA_UserX_NewCampaign`
+                """
+            )
+
+            project_col1 = st.columns(3)
+
+            with project_col1[0]:
+
+                # API_tables = get_table_names(schema)  # load API files
+
+                slection_tables = ["NA"]
+
+                api_name = st.selectbox("Select API data", slection_tables, index=0)
+
+                # data availabe through API
+                # api_path = API_path_dict[api_name]
+
+            with project_col1[1]:
+                defualt_project_prefix = f"{api_name.split('_mmo_')[0]}_{st.session_state['unique_ids'][st.session_state['emp_id']][0]}_".replace(
+                    " ", "_"
+                ).lower()
+
+                if "project_name_box" not in st.session_state:
+                    st.session_state["project_name_box"] = defualt_project_prefix
+
+                project_name = st.text_input(
+                    "Enter Project Name", key="project_name_box"
+                )
+                warning_box = st.empty()
+
+            with project_col1[2]:
+
+                allowed_users = st.multiselect(
+                    "Select Users who can access to this Project",
+                    [val for val in unique_users],
+                )
+
+                allowed_users = list(allowed_users)
+
+                matching_user_id = []
+
+                if len(allowed_users) > 0:
+
+                    # converting the selection to comma seperated values to store in db
+
+                    for emp_id, details in st.session_state["unique_ids"].items():
+                        for name in allowed_users:
+                            if name in details:
+                                matching_user_id.append(emp_id)
+                                break
+
+            st.button(
+                "Reset Project Name",
+                on_click=reset_project_text_box,
+                help="",
+                use_container_width=True,
+            )
+
+            create = st.button(
+                "Create New Project",
+                use_container_width=True,
+                help="Project Name should follow naming convention",
+            )
+
+            if create:
+                if not project_name.lower().startswith(defualt_project_prefix):
+                    with warning_box:
+                        st.warning("Project Name should follow naming convention")
+                        st.stop()
+
+                if project_name == defualt_project_prefix:
+                    with warning_box:
+                        st.warning("Cannot name only with prefix")
+                        st.stop()
+
+                if project_name in user_projects:
+                    with warning_box:
+                        st.warning("Project already exists please enter new name")
+                        st.stop()
+
+                if not (
+                    2 <= len(project_name) <= 50
+                    and bool(re.match("^[A-Za-z0-9_]*$", project_name))
+                ):
+                    # Store the warning message details in session state
+
+                    with warning_box:
+                        st.warning(
+                            "Please provide a valid project name (2-50 characters, only A-Z, a-z, 0-9, and _)."
+                        )
+                        st.stop()
+
+                if contains_sql_keywords_check(project_name):
+                    with warning_box:
+                        st.warning(
+                            "Input contains SQL keywords. Please avoid using SQL commands."
+                        )
+                        st.stop()
+                else:
+                    pass
+
+                with st.spinner("Creating Project"):
+                    new_project()
+
+                with warning_box:
+                    st.write("Project Created")
+
+                st.session_state["radio_box_index"] = 1
+
+                log_message(
+                    "info",
+                    f"Employee {st.session_state['emp_id']} created new project {project_name}.",
+                    "Home",
+                )
+
+                st.rerun()
+
+    if projct_radio == "Clone Saved Projects":
+
+        with st.expander("Clone Saved Projects", expanded=False):
+
+            if len(st.session_state["clone_project_dict"]) == 0:
+                st.warning("You dont have access to any saved projects")
+                st.stop()
+
+            cols = st.columns(2)
+
+            with cols[0]:
+                owners = list(st.session_state["clone_project_dict"].keys())
+                owner_name = st.selectbox("Select Owner", owners)
+
+            with cols[1]:
+
+                project_names = [
+                    project["project_name"]
+                    for project in st.session_state["clone_project_dict"][owner_name]
+                ]
+                project_name_owner = st.selectbox(
+                    "Select a saved Project available for you",
+                    project_names,
+                )
+
+                defualt_project_prefix = f"{project_name_owner.split('_')[0]}_{st.session_state['unique_ids'][st.session_state['emp_id']][0]}_".replace(
+                    " ", "_"
+                ).lower()
+                user_projects = list(
+                    set(st.session_state["project_summary_df"]["Project Name"])
+                )
+
+            cloned_project_name = st.text_input(
+                "Enter Project Name",
+                value=defualt_project_prefix,
+            )
+            warning_box = st.empty()
+
+            if st.button(
+                "Load Project", use_container_width=True, key="load_project_button_key"
+            ):
+
+                if not cloned_project_name.lower().startswith(defualt_project_prefix):
+                    with warning_box:
+                        st.warning("Project Name should follow naming conventions")
+                        st.stop()
+
+                if cloned_project_name == defualt_project_prefix:
+                    with warning_box:
+                        st.warning("Cannot Name only with Prefix")
+                        st.stop()
+
+                if cloned_project_name in user_projects:
+                    with warning_box:
+                        st.warning("Project already exists please enter new name")
+                        st.stop()
+
+                with st.spinner("Cloning Project"):
+                    old_prj_id = get_project_id_from_dict(
+                        st.session_state["clone_project_dict"],
+                        owner_name,
+                        project_name_owner,
+                    )
+                    old_metadata = fetch_project_metadata(old_prj_id)
+
+                    new_prj_id = create_new_project(
+                        st.session_state["emp_id"],
+                        cloned_project_name,
+                        "",
+                        st.session_state["emp_id"],
+                    )
+
+                    insert_project_metadata(
+                        new_prj_id, old_metadata, st.session_state["emp_id"]
+                    )
+                    update_summary_df()
+                    st.success("Project Cloned")
+                    st.rerun()
+
+    if projct_radio == "Delete Projects":
+        if len(st.session_state["project_summary_df"]) != 0:
+
+            with st.expander("Delete Projects", expanded=True):
+
+                delete_projects = st.multiselect(
+                    "Select all the projects number who want to delete",
+                    st.session_state["project_summary_df"]["Project Number"],
+                )
+                st.warning(
+                    "Projects will be permanently deleted. Other users will not be able to clone them if they have not already done so."
+                )
+                if st.button("Delete Projects", use_container_width=True):
+                    if len(delete_projects) > 0:
+                        with st.spinner("Deleting Projects"):
+                            delete_projects_by_ids(delete_projects)
+                            update_summary_df()
+                            st.success("Projects Deleted")
+                            st.rerun()
+
+                    else:
+                        st.warning("Please select atleast one project number to delete")
+
+    if projct_radio == "Download Project PPT":
+
+        try:
+            ppt = create_ppt(
+                st.session_state["project_name"],
+                st.session_state["username"],
+                "panel",  # new
+            )
+
+            if ppt is not False:
+                st.download_button(
+                    "Download",
+                    data=ppt.getvalue(),
+                    file_name=st.session_state["project_name"]
+                    + " Project Summary.pptx",
+                    use_container_width=True,
+                )
+            else:
+                st.warning("Please make some progress before downloading PPT.")
+
+        except Exception as e:
+            st.warning("PPT Download Faild ")
+            # new
+            log_message(
+                log_type="error", message=f"Error in PPT build: {e}", page_name="Home"
+            )

README.md

@@ -1,13 +1,13 @@
----
-title: MediaMixOptimization
-emoji: 🦀
-colorFrom: red
-colorTo: gray
-sdk: streamlit
-sdk_version: 1.44.1
-app_file: app.py
-pinned: false
-short_description: This tool helps in optimizing media spends
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: MediaMixOptimization
+emoji: 📊
+colorFrom: red
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.28.0
+app_file: Home.py
+pinned: false
+short_description: This tool helps in optimizing media spends
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

constants.py

@@ -0,0 +1,188 @@
+###########################################################################################################
+# Default
+###########################################################################################################
+
+import pandas as pd
+from collections import OrderedDict
+
+# default_dct
+default_dct = {
+    "data_import": {
+        "gold_layer_df": pd.DataFrame(),
+        "granularity_selection": "daily",
+        "dashboard_df": None,
+        "tool_df": None,
+        "unique_panels": [],
+        "imputation_df": None,
+        "imputed_tool_df": None,
+        "group_dict": {},
+        "category_dict": {},
+    },
+    "data_validation": {
+        "target_column": 0,
+        "selected_panels": None,
+        "selected_feature": 0,
+        "validated_variables": [],
+        "Non_media_variables": 0,
+        "correlation": [],
+    },
+    "transformations": {
+        "final_df": None,
+        "summary_string": None,
+        "Media": {},
+        "Exogenous": {},
+        "Internal": {},
+        "Specific": {},
+        "correlation_plot_selection": [],
+    },
+    "model_build": {
+        "sel_target_col": None,
+        "all_iters_check": False,
+        "iterations": 0,
+        "build_button": False,
+        "show_results_check": False,
+        "session_state_saved": {},
+    },
+    "model_tuning": {
+        "sel_target_col": None,
+        "sel_model": {},
+        "flag_expander": False,
+        "start_date_default": None,
+        "end_date_default": None,
+        "repeat_default": "No",
+        "flags": {},
+        "select_all_flags_check": {},
+        "selected_flags": {},
+        "trend_check": False,
+        "week_num_check": False,
+        "sine_cosine_check": False,
+        "session_state_saved": {},
+    },
+    "saved_model_results": {
+        "selected_options": None,
+        "model_grid_sel": [1],
+    },
+    "current_media_performance": {
+        "model_outputs": {},
+    },
+    "media_performance": {"start_date": None, "end_date": None},
+    "response_curves": {
+        "original_metadata_file": None,
+        "modified_metadata_file": None,
+    },
+    "scenario_planner": {
+        "original_metadata_file": None,
+        "modified_metadata_file": None,
+    },
+    "saved_scenarios": {"saved_scenarios_dict": OrderedDict()},
+    "optimized_result_analysis": {
+        "selected_scenario_selectbox_visualize": 0,
+        "metric_selectbox_visualize": 0,
+    },
+}
+
+###########################################################################################################
+# Data Import
+###########################################################################################################
+
+# Constants Data Import
+upload_rows_limit = 1000000  # Maximum number of rows allowed for upload
+upload_column_limit = 1000  # Maximum number of columns allowed for upload
+word_length_limit_lower = 2  # Minimum allowed length for words
+word_length_limit_upper = 100  # Maximum allowed length for words
+minimum_percent_overlap = (
+    1  # Minimum required percentage of overlap with the reference data
+)
+minimum_row_req = 30  # Minimum number of rows required
+percent_drop_col_threshold = 50  # Percentage threshold above which columns are automatically categorized for drop imputation
+
+###########################################################################################################
+# Transfromations
+###########################################################################################################
+
+# Constants Transformations
+predefined_defaults = {
+    "Lag": (1, 2),
+    "Lead": (1, 2),
+    "Moving Average": (1, 2),
+    "Saturation": (10, 20),
+    "Power": (2, 4),
+    "Adstock": (0.5, 0.7),
+}  # Pre-defined default values of every transformation
+
+# Transfromations min, max and step
+lead_min_value = 1
+lead_max_value = 10
+lead_step = 1
+lag_min_value = 1
+lag_max_value = 10
+lag_step = 1
+moving_average_min_value = 1
+moving_average_max_value = 10
+moving_average_step = 1
+saturation_min_value = 0
+saturation_max_value = 100
+saturation_step = 1
+power_min_value = 1
+power_max_value = 5
+power_step = 1
+adstock_min_value = 0.0
+adstock_max_value = 1.0
+adstock_step = 0.05
+display_max_col = 500  # Maximum columns to display
+
+###########################################################################################################
+# Model Build
+###########################################################################################################
+
+MAX_COMBINATIONS = 50000 # Max number of model combinations possible
+MIN_MODEL_NAME_LENGTH = 0  # model can only be saved if len(model_name) is greater than this value
+MIN_P_VALUE_THRESHOLD = 0.06  # coefficients with p values less than this value are considered valid
+MODEL_POS_COEFF_RATIO_THRESHOLD = 0  # ratio of positive coefficients/total coefficients for model validity
+MODEL_P_VALUE_RATIO_THRESHOLD = 0  # ratio of coefficients with p value/total coefficients for model validity
+MAX_TOP_FEATURES = 5  # max number of top features selected per variable
+MAX_NUM_FILTERS = 10  # maximum number of filters allowed
+DEFAULT_FILTER_VALUE = 0.0  # default value of a new filter
+VIF_LOW_THRESHOLD = 3  # Threshold for VIF to be colored green
+VIF_HIGH_THRESHOLD = 10  # Threshold for VIF to be colored red
+DEFAULT_TRAIN_RATIO = 3/4  # default train set ratio (75%)
+
+###########################################################################################################
+# Model Tuning
+###########################################################################################################
+
+import numpy as np
+
+NUM_FLAG_COLS_TO_DISPLAY = 4  # Number of columns to be created on UI to display flags
+HALF_YEAR_THRESHOLD = 6  # Threshold of months to create quarter frequency sine-cosine waves
+FULL_YEAR_THRESHOLD = 12  # Threshold of months to create quarter annual sine-cosine waves
+TREND_MIN = 5  # Starting value of trend line
+ANNUAL_FREQUENCY = 2 * np.pi / 365  # annual frequency
+QTR_FREQUENCY_FACTOR = 4  # multiplication factor to get quarterly frequency
+HALF_YEARLY_FREQUENCY_FACTOR = 2  # multiplication factor to get semi-annual frequency
+
+###########################################################################################################
+# Scenario Planner
+###########################################################################################################
+
+# Constants Scenario Planner
+xtol_tolerance_per = 1  # Percenatge of tolerance
+mroi_threshold = 0.05  # mROI threshold
+word_length_limit_lower = 2  # Minimum allowed length for words
+word_length_limit_upper = 100  # Maximum allowed length for words
+
+###########################################################################################################
+# PPT utils
+###########################################################################################################
+
+TITLE_FONT_SIZE = 20
+AXIS_LABEL_FONT_SIZE = 8
+CHART_TITLE_FONT_SIZE = 14
+AXIS_TITLE_FONT_SIZE = 12
+DATA_LABEL_FONT_SIZE = 8
+LEGEND_FONT_SIZE = 10
+PIE_LEGEND_FONT_SIZE = 7
+
+###########################################################################################################
+# Page Name
+###########################################################################################################

data_analysis.py

@@ -0,0 +1,267 @@
+import streamlit as st
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+from sklearn.metrics import r2_score
+from collections import OrderedDict
+import plotly.express as px
+import plotly.graph_objects as go
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import streamlit as st
+import re
+from matplotlib.colors import ListedColormap
+# from st_aggrid import AgGrid, GridOptionsBuilder
+# from src.agstyler import PINLEFT, PRECISION_TWO, draw_grid
+
+
+def format_numbers(x):
+    if abs(x) >= 1e6:
+        # Format as millions with one decimal place and commas
+        return f'{x/1e6:,.1f}M'
+    elif abs(x) >= 1e3:
+        # Format as thousands with one decimal place and commas
+        return f'{x/1e3:,.1f}K'
+    else:
+        # Format with one decimal place and commas for values less than 1000
+        return f'{x:,.1f}'
+
+    
+
+def line_plot(data, x_col, y1_cols, y2_cols, title):
+    """
+    Create a line plot with two sets of y-axis data.
+
+    Parameters:
+    data (DataFrame): The data containing the columns to be plotted.
+    x_col (str): The column name for the x-axis.
+    y1_cols (list): List of column names for the primary y-axis.
+    y2_cols (list): List of column names for the secondary y-axis.
+    title (str): The title of the plot.
+
+    Returns:
+    fig (Figure): The Plotly figure object with the line plot.
+    """
+    fig = go.Figure()
+
+    # Add traces for the primary y-axis
+    for y1_col in y1_cols:
+        fig.add_trace(go.Scatter(x=data[x_col], y=data[y1_col], mode='lines', name=y1_col, line=dict(color='#11B6BD')))
+
+    # Add traces for the secondary y-axis
+    for y2_col in y2_cols:
+        fig.add_trace(go.Scatter(x=data[x_col], y=data[y2_col], mode='lines', name=y2_col, yaxis='y2', line=dict(color='#739FAE')))
+
+    # Configure the layout for the secondary y-axis if needed
+    if len(y2_cols) != 0:
+        fig.update_layout(yaxis=dict(), yaxis2=dict(overlaying='y', side='right'))
+    else:
+        fig.update_layout(yaxis=dict(), yaxis2=dict(overlaying='y', side='right'))
+
+    # Add title if provided
+    if title:
+        fig.update_layout(title=title)
+
+    # Customize axes and legend
+    fig.update_xaxes(showgrid=False)
+    fig.update_yaxes(showgrid=False)
+    fig.update_layout(legend=dict(
+        orientation="h",
+        yanchor="top",
+        y=1.1,
+        xanchor="center",
+        x=0.5
+    ))
+
+    return fig
+
+
+
+def line_plot_target(df, target, title):
+    """
+    Create a line plot with a trendline for a target column.
+
+    Parameters:
+    df (DataFrame): The data containing the columns to be plotted.
+    target (str): The column name for the y-axis.
+    title (str): The title of the plot.
+
+    Returns:
+    fig (Figure): The Plotly figure object with the line plot and trendline.
+    """
+    # Calculate the trendline coefficients
+    coefficients = np.polyfit(df['date'].view('int64'), df[target], 1)
+    trendline = np.poly1d(coefficients)
+    fig = go.Figure()
+
+    # Add the target line plot
+    fig.add_trace(go.Scatter(x=df['date'], y=df[target], mode='lines', name=target, line=dict(color='#11B6BD')))
+    
+    # Calculate and add the trendline plot
+    trendline_x = df['date']
+    trendline_y = trendline(df['date'].view('int64'))
+    fig.add_trace(go.Scatter(x=trendline_x, y=trendline_y, mode='lines', name='Trendline', line=dict(color='#739FAE')))
+
+    # Update layout with title and x-axis type
+    fig.update_layout(
+        title=title,
+        xaxis=dict(type='date')
+    )
+
+    # Add vertical lines at the start of each year
+    for year in df['date'].dt.year.unique()[1:]:
+        january_1 = pd.Timestamp(year=year, month=1, day=1)
+        fig.add_shape(
+            go.layout.Shape(
+                type="line",
+                x0=january_1,
+                x1=january_1,
+                y0=0,
+                y1=1,
+                xref="x",
+                yref="paper",
+                line=dict(color="grey", width=1.5, dash="dash"),
+            )
+        )
+    
+    # Customize the legend
+    fig.update_layout(legend=dict(
+        orientation="h",
+        yanchor="top",
+        y=1.1,
+        xanchor="center",
+        x=0.5
+    ))
+    
+    return fig
+
+
+def correlation_plot(df, selected_features, target):
+    """
+    Create a correlation heatmap plot for selected features and target column.
+
+    Parameters:
+    df (DataFrame): The data containing the columns to be plotted.
+    selected_features (list): List of column names to be included in the correlation plot.
+    target (str): The target column name to be included in the correlation plot.
+
+    Returns:
+    fig (Figure): The Matplotlib figure object with the correlation heatmap plot.
+    """
+    # Define custom colormap
+    custom_cmap = ListedColormap(['#08083B', "#11B6BD"])  
+    
+    # Select the relevant columns for correlation calculation
+    corr_df = df[selected_features]
+    corr_df = pd.concat([corr_df, df[target]], axis=1)
+    
+    # Create a matplotlib figure and axis
+    fig, ax = plt.subplots(figsize=(16, 12))
+    
+    # Generate the heatmap with correlation coefficients
+    sns.heatmap(corr_df.corr(), annot=True, cmap='Blues', fmt=".2f", linewidths=0.5, mask=np.triu(corr_df.corr()))
+    
+    # Customize the plot
+    plt.xticks(rotation=45)
+    plt.yticks(rotation=0)
+    
+    return fig
+
+
+def summary(data, selected_feature, spends, Target=None):
+    """
+    Create a summary table of selected features and optionally a target column.
+
+    Parameters:
+    data (DataFrame): The data containing the columns to be summarized.
+    selected_feature (list): List of column names to be included in the summary.
+    spends (str): The column name for the spends data.
+    Target (str, optional): The target column name for additional summary calculations. Default is None.
+
+    Returns:
+    sum_df (DataFrame): The summary DataFrame with formatted values.
+    """
+    if Target:
+        # Summarize data for the target column
+        sum_df = data[selected_feature]
+        sum_df['Year'] = data['date'].dt.year
+        sum_df = sum_df.groupby('Year')[selected_feature].sum().reset_index()
+        
+        # Calculate total sum and append to the DataFrame
+        total_sum = sum_df.sum(numeric_only=True)
+        total_sum['Year'] = 'Total'
+        sum_df = pd.concat([sum_df, total_sum.to_frame().T], axis=0, ignore_index=True).copy()
+        
+        # Set 'Year' as index and format numbers
+        sum_df.set_index(['Year'], inplace=True)
+        sum_df = sum_df.applymap(format_numbers)
+        
+        # Format spends columns as currency
+        spends_col = [col for col in sum_df.columns if any(keyword in col for keyword in ['spends', 'cost'])]
+        for col in spends_col:
+            sum_df[col] = sum_df[col].map(lambda x: f'${x}')
+        
+        return sum_df
+    else:
+        # Include spends in the selected features
+        selected_feature.append(spends)
+        
+        # Ensure unique features
+        selected_feature = list(set(selected_feature))
+        
+        if len(selected_feature) > 1:
+            imp_clicks = selected_feature[1]
+            spends_col = selected_feature[0]
+            
+            # Summarize data for the selected features
+            sum_df = data[selected_feature]
+            sum_df['Year'] = data['date'].dt.year
+            sum_df = sum_df.groupby('Year')[selected_feature].agg('sum')
+            
+            # Calculate CPM/CPC
+            sum_df['CPM/CPC'] = (sum_df[spends_col] / sum_df[imp_clicks]) * 1000
+            
+            # Calculate grand total and append to the DataFrame
+            sum_df.loc['Grand Total'] = sum_df.sum()
+            
+            # Format numbers and replace NaNs
+            sum_df = sum_df.applymap(format_numbers)
+            sum_df.fillna('-', inplace=True)
+            sum_df = sum_df.replace({"0.0": '-', 'nan': '-'})
+            
+            # Format spends columns as currency
+            sum_df[spends_col] = sum_df[spends_col].map(lambda x: f'${x}')
+            
+            return sum_df
+        else:
+            # Summarize data for a single selected feature
+            sum_df = data[selected_feature]
+            sum_df['Year'] = data['date'].dt.year
+            sum_df = sum_df.groupby('Year')[selected_feature].agg('sum')
+            
+            # Calculate grand total and append to the DataFrame
+            sum_df.loc['Grand Total'] = sum_df.sum()
+            
+            # Format numbers and replace NaNs
+            sum_df = sum_df.applymap(format_numbers)
+            sum_df.fillna('-', inplace=True)
+            sum_df = sum_df.replace({"0.0": '-', 'nan': '-'})
+            
+            # Format spends columns as currency
+            spends_col = [col for col in sum_df.columns if any(keyword in col for keyword in ['spends', 'cost'])]
+            for col in spends_col:
+                sum_df[col] = sum_df[col].map(lambda x: f'${x}')
+            
+            return sum_df
+
+
+
+def sanitize_key(key, prefix=""):
+    # Use regular expressions to remove non-alphanumeric characters and spaces
+    key = re.sub(r'[^a-zA-Z0-9]', '', key)
+    return f"{prefix}{key}"
+
+
+
+

data_prep.py

@@ -0,0 +1,185 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import statsmodels.api as sm
+from sklearn.metrics import mean_absolute_error, r2_score,mean_absolute_percentage_error  
+from sklearn.preprocessing import MinMaxScaler
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+from plotly.subplots import make_subplots
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+from datetime import datetime
+import seaborn as sns
+
+
+def plot_actual_vs_predicted(date, y, predicted_values, model, target_column=None, flag=None, repeat_all_years=False, is_panel=False):
+    """
+    Plots actual vs predicted values with optional flags and aggregation for panel data.
+
+    Parameters:
+    date (pd.Series): Series of dates for x-axis.
+    y (pd.Series): Actual values.
+    predicted_values (pd.Series): Predicted values from the model.
+    model (object): Trained model object.
+    target_column (str, optional): Name of the target column.
+    flag (tuple, optional): Start and end dates for flagging periods.
+    repeat_all_years (bool, optional): Whether to repeat flags for all years.
+    is_panel (bool, optional): Whether the data is panel data requiring aggregation.
+
+    Returns:
+    metrics_table (pd.DataFrame): DataFrame containing MAPE, R-squared, and Adjusted R-squared.
+    line_values (list): List of flag values for plotting.
+    fig (go.Figure): Plotly figure object.
+    """
+    if flag is not None:
+        fig = make_subplots(specs=[[{"secondary_y": True}]])
+    else:
+        fig = go.Figure()
+
+    if is_panel:
+        df = pd.DataFrame()
+        df['date'] = date
+        df['Actual'] = y
+        df['Predicted'] = predicted_values
+        df_agg = df.groupby('date').agg({'Actual': 'sum', 'Predicted': 'sum'}).reset_index()
+        df_agg.columns = ['date', 'Actual', 'Predicted']
+        assert len(df_agg) == pd.Series(date).nunique()
+
+        fig.add_trace(go.Scatter(x=df_agg['date'], y=df_agg['Actual'], mode='lines', name='Actual', line=dict(color='#08083B')))
+        fig.add_trace(go.Scatter(x=df_agg['date'], y=df_agg['Predicted'], mode='lines', name='Predicted', line=dict(color='#11B6BD')))
+    else:
+        fig.add_trace(go.Scatter(x=date, y=y, mode='lines', name='Actual', line=dict(color='#08083B')))
+        fig.add_trace(go.Scatter(x=date, y=predicted_values, mode='lines', name='Predicted', line=dict(color='#11B6BD')))
+
+    line_values = []
+    if flag:
+        min_date, max_date = flag[0], flag[1]
+        min_week = datetime.strptime(str(min_date), "%Y-%m-%d").strftime("%U")
+        max_week = datetime.strptime(str(max_date), "%Y-%m-%d").strftime("%U")
+
+        if repeat_all_years:
+            line_values = list(pd.Series(date).map(lambda x: 1 if (pd.Timestamp(x).week >= int(min_week)) & (pd.Timestamp(x).week <= int(max_week)) else 0))
+            assert len(line_values) == len(date)
+            fig.add_trace(go.Scatter(x=date, y=line_values, mode='lines', name='Flag', line=dict(color='#FF5733')), secondary_y=True)
+        else:
+            line_values = list(pd.Series(date).map(lambda x: 1 if (pd.Timestamp(x) >= pd.Timestamp(min_date)) and (pd.Timestamp(x) <= pd.Timestamp(max_date)) else 0))
+            fig.add_trace(go.Scatter(x=date, y=line_values, mode='lines', name='Flag', line=dict(color='#FF5733')), secondary_y=True)
+
+    mape = mean_absolute_percentage_error(y, predicted_values)
+    r2 = r2_score(y, predicted_values)
+    adjr2 = 1 - (1 - r2) * (len(y) - 1) / (len(y) - len(model.params) - 1)
+
+    metrics_table = pd.DataFrame({
+        'Metric': ['MAPE', 'R-squared', 'AdjR-squared'],
+        'Value': [mape, r2, adjr2]
+    })
+
+    fig.update_layout(
+        xaxis=dict(title='Date'),
+        yaxis=dict(title=target_column),
+        xaxis_tickangle=-30
+    )
+    fig.add_annotation(
+        text=f"MAPE: {mape * 100:0.1f}%,  Adj. R-squared: {adjr2 * 100:.1f}%",
+        xref="paper",
+        yref="paper",
+        x=0.95,
+        y=1.2,
+        showarrow=False,
+    )
+
+    return metrics_table, line_values, fig
+
+
+def plot_residual_predicted(actual, predicted, df):
+    """
+    Plots standardized residuals against predicted values.
+
+    Parameters:
+    actual (pd.Series): Actual values.
+    predicted (pd.Series): Predicted values.
+    df (pd.DataFrame): DataFrame containing the data.
+
+    Returns:
+    fig (go.Figure): Plotly figure object.
+    """
+    df_ = df.copy()
+    df_['Residuals'] = actual - pd.Series(predicted)
+    df_['StdResidual'] = (df_['Residuals'] - df_['Residuals'].mean()) / df_['Residuals'].std()
+
+    fig = px.scatter(df_, x=predicted, y='StdResidual', opacity=0.5, color_discrete_sequence=["#11B6BD"])
+
+    fig.add_hline(y=0, line_dash="dash", line_color="darkorange")
+    fig.add_hline(y=2, line_color="red")
+    fig.add_hline(y=-2, line_color="red")
+
+    fig.update_xaxes(title='Predicted')
+    fig.update_yaxes(title='Standardized Residuals (Actual - Predicted)')
+
+    fig.update_layout(title='2.3.1 Residuals over Predicted Values', autosize=False, width=600, height=400)
+
+    return fig
+
+
+def residual_distribution(actual, predicted):
+    """
+    Plots the distribution of residuals.
+
+    Parameters:
+    actual (pd.Series): Actual values.
+    predicted (pd.Series): Predicted values.
+
+    Returns:
+    plt (matplotlib.pyplot): Matplotlib plot object.
+    """
+    Residuals = actual - pd.Series(predicted)
+
+    sns.set(style="whitegrid")
+    plt.figure(figsize=(6, 4))
+    sns.histplot(Residuals, kde=True, color="#11B6BD")
+
+    plt.title('2.3.3 Distribution of Residuals')
+    plt.xlabel('Residuals')
+    plt.ylabel('Probability Density')
+
+    return plt
+
+
+def qqplot(actual, predicted):
+    """
+    Creates a QQ plot of the residuals.
+
+    Parameters:
+    actual (pd.Series): Actual values.
+    predicted (pd.Series): Predicted values.
+
+    Returns:
+    fig (go.Figure): Plotly figure object.
+    """
+    Residuals = actual - pd.Series(predicted)
+    Residuals = pd.Series(Residuals)
+    Resud_std = (Residuals - Residuals.mean()) / Residuals.std()
+
+    fig = go.Figure()
+    fig.add_trace(go.Scatter(x=sm.ProbPlot(Resud_std).theoretical_quantiles,
+                             y=sm.ProbPlot(Resud_std).sample_quantiles,
+                             mode='markers',
+                             marker=dict(size=5, color="#11B6BD"),
+                             name='QQ Plot'))
+
+    diagonal_line = go.Scatter(
+        x=[-2, 2],
+        y=[-2, 2],
+        mode='lines',
+        line=dict(color='red'),
+        name=' '
+    )
+    fig.add_trace(diagonal_line)
+
+    fig.update_layout(title='2.3.2 QQ Plot of Residuals', title_x=0.5, autosize=False, width=600, height=400,
+                      xaxis_title='Theoretical Quantiles', yaxis_title='Sample Quantiles')
+
+    return fig
+

db/imp_db.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d945da7748723c6078c2d2bee4aaae5ca0838dcee0e2032a836341dcf7794e
+size 15618048

db_creation.py

@@ -0,0 +1,114 @@
+# import sqlite3
+
+# # Connect to SQLite database (or create it if it doesn't exist)
+# conn = sqlite3.connect('imp_db.db')
+
+# # Enable foreign key support
+# conn.execute('PRAGMA foreign_keys = ON;')
+
+# # Create a cursor object
+# c = conn.cursor()
+
+# # SQL queries to create tables
+# create_mmo_users_table = """
+# CREATE TABLE IF NOT EXISTS mmo_users (
+#     emp_id TEXT PRIMARY KEY ,
+#     emp_nam TEXT NOT NULL,
+#     emp_typ TEXT NOT NULL,
+#     pswrd_key TEXT NOT NULL,
+#     pswrd_flag INTEGER NOT NULL DEFAULT 0,
+#     crte_dt_tm TEXT DEFAULT (datetime('now')),
+#     crte_by_uid TEXT NOT NULL,
+#     updt_dt_tm TEXT DEFAULT (datetime('now')),
+#     updt_by_uid TEXT
+# );
+# """
+
+# create_mmo_projects_table = """
+# CREATE TABLE IF NOT EXISTS mmo_projects (
+#     prj_id INTEGER PRIMARY KEY AUTOINCREMENT,
+#     prj_ownr_id TEXT NOT NULL,
+#     prj_nam TEXT NOT NULL,
+#     alwd_emp_id TEXT,
+#     meta_data_agrgt TEXT,
+#     crte_dt_tm TEXT DEFAULT (datetime('now')),
+#     crte_by_uid TEXT NOT NULL,
+#     updt_dt_tm TEXT DEFAULT (datetime('now')),
+#     updt_by_uid TEXT,
+#     FOREIGN KEY (prj_ownr_id) REFERENCES mmo_users(emp_id)
+# );
+# """
+
+# create_mmo_project_meta_data_table = """
+# CREATE TABLE IF NOT EXISTS mmo_project_meta_data (
+#     prj_guid INTEGER PRIMARY KEY AUTOINCREMENT,
+#     prj_id INTEGER NOT NULL,
+#     page_nam TEXT NOT NULL,
+#     file_nam TEXT NOT NULL,
+#     pkl_obj BLOB,
+#     dshbrd_ts TEXT,
+#     crte_dt_tm TEXT DEFAULT (datetime('now')),
+#     crte_by_uid TEXT NOT NULL,
+#     updt_dt_tm TEXT DEFAULT (datetime('now')),
+#     updt_by_uid TEXT,
+#     FOREIGN KEY (prj_id) REFERENCES mmo_projects(prj_id)
+# );
+# """
+
+# # Execute the queries to create tables
+# c.execute(create_mmo_users_table)
+# c.execute(create_mmo_projects_table)
+# c.execute(create_mmo_project_meta_data_table)
+
+# # Commit changes and close the connection
+# conn.commit()
+# conn.close()
+import sqlite3
+
+def add_user_to_db(db_path, user_id, name, user_type, pswrd_key):
+    """
+    Adds a user to the mmo_users table in the SQLite database.
+
+    Parameters:
+    - db_path (str): The path to the SQLite database file.
+    - user_id (str): The ID of the user.
+    - name (str): The name of the user.
+    - user_type (str): The type of the user.
+    - pswrd_key (str): The password key for the user.
+    """
+    
+    try:
+        # Connect to the SQLite database
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        
+        # SQL query to insert a new user
+        insert_query = """
+        INSERT INTO mmo_users (emp_id, emp_nam, emp_typ, pswrd_key,crte_by_uid)
+        VALUES (?, ?, ?, ?,?)
+        """
+        
+        # Execute the query with parameters
+        cursor.execute(insert_query, (user_id, name, user_type, pswrd_key, user_id))
+        
+        # Commit the transaction
+        conn.commit()
+        
+        print(f"User {name} added successfully.")
+        
+    except sqlite3.Error as e:
+        print(f"Error adding user to the database: {e}")
+    
+    finally:
+        # Close the database connection
+        conn.close()
+
+# Define the database path and user details
+db_path = r'db\imp_db.db'  # Update this path to your actual database path
+user_id = 'e162284'
+name = 'admin'
+user_type = 'admin'
+pswrd_key = '$2b$12$wP7R0usvKWtr4X06qwGWvOFQCkzOZAzSVRAoDv/68x6GS4rHK5mDm'
+
+# Add the user to the database
+add_user_to_db(db_path, user_id, name, user_type, pswrd_key)

log_application.py

@@ -0,0 +1,157 @@
+"""
+Contains logging functions to initialize and configure loggers, facilitating efficient tracking of ETL processes. 
+Also Provides methods to log messages with different log levels and additional context information such as channel name and client name
+"""
+
+import os
+import streamlit as st
+import logging, logging.handlers
+from datetime import datetime, timezone
+
+
+def delete_old_log_files():
+    """
+    Deletes all log files in the 'logs' directory that are older than the current month.
+
+    Returns:
+        None
+    """
+    # Set the logs directory to 'logs' within the current working directory
+    logs_directory = os.path.join(os.getcwd(), "logs")
+
+    # Ensure the logs directory exists
+    if not os.path.exists(logs_directory):
+        print(f"Directory {logs_directory} does not exist.")
+        return
+
+    # Get the current year and month in UTC
+    current_utc_time = datetime.now(timezone.utc)
+    current_year_month = current_utc_time.strftime("%Y%m")
+
+    # Iterate through all files in the logs directory
+    for filename in os.listdir(logs_directory):
+        # We assume the log files are in the format "eid_YYYYMM.log"
+        if filename.endswith(".log"):
+            # Extract the date portion from the filename
+            try:
+                file_date = filename.split("_")[-1].replace(".log", "")
+                # Compare the file date with the current date
+                if file_date < current_year_month:
+                    # Construct the full file path
+                    file_path = os.path.join(logs_directory, filename)
+                    # Delete the file
+                    os.remove(file_path)
+
+            except IndexError:
+                # If the filename doesn't match the expected pattern, skip it
+                pass
+
+
+def create_log_file_name(emp_id, project_name):
+    """
+    Generates a log file name using the format eid_YYYYMMDD.log based on UTC time.
+
+    Returns:
+        str: The generated log file name.
+    """
+    # Get the current UTC time
+    utc_now = datetime.now(timezone.utc)
+    # Format the date as YYYYMMDD
+    date_str = utc_now.strftime("%Y%m%d")
+    # Create the file name with the format eid_YYYYMMDD.log
+    log_file_name = f"{emp_id}_{project_name}_{date_str}.log"
+    return log_file_name
+
+
+def get_logger(log_file):
+    """
+    Initializes and configures a logger. If the log file already exists, it appends to it.
+    Returns the same logger instance if called multiple times.
+
+    Args:
+        log_file (str): The path to the log file where logs will be written.
+
+    Returns:
+        logging.Logger: Configured logger instance.
+    """
+
+    # Define the logger name
+    logger_name = os.path.basename(log_file)
+    logger = logging.getLogger(logger_name)
+
+    # If the logger already has handlers, return it to prevent duplicate handlers
+    if logger.hasHandlers():
+        return logger
+
+    # Set the logging level
+    logging_level = logging.INFO
+    logger.setLevel(logging_level)
+
+    # Ensure the logs directory exists and is writable
+    logs_dir = os.path.dirname(log_file)
+    if not os.path.exists(logs_dir):
+        os.makedirs(logs_dir, exist_ok=True)
+
+    # Change the directory permissions to 0777 (0o777 in Python 3)
+    os.chmod(logs_dir, 0o777)
+
+    # Create a file handler to append to the log file
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(
+        logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S")
+    )
+    logger.addHandler(file_handler)
+
+    # Create a stream handler to print to console
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(
+        logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S")
+    )
+    logger.addHandler(stream_handler)
+
+    return logger
+
+
+def log_message(log_type, message, page_name):
+    """
+    Logs a message with the specified log type and additional context information.
+
+    Args:
+        log_type (str): The type of log ('info', 'error', 'warning', 'debug').
+        message (str): The message to log.
+        page_name (str): The name of the page associated with the message.
+
+    Returns:
+        None
+    """
+
+    # Retrieve the employee ID and project name from session state
+    emp_id = st.session_state["emp_id"]
+    project_name = st.session_state["project_name"]
+
+    # Create log file name using a function that generates a name based on the current date and EID
+    log_file_name = create_log_file_name(emp_id, project_name)
+
+    # Construct the full path to the log file within the "logs" within the current working directory
+    file_path = os.path.join(os.getcwd(), "logs", log_file_name)
+
+    # Generate and store the logger instance in session state
+    logger = get_logger(file_path)
+
+    # Construct the log message with all required context information
+    log_message = (
+        f"USER_EID: {emp_id} ; "
+        f"PROJECT_NAME: {project_name} ; "
+        f"PAGE_NAME: {page_name} ; "
+        f"MESSAGE: {message}"
+    )
+
+    # Log the message with the appropriate log level based on the log_type argument
+    if log_type == "info":
+        logger.info(log_message)
+    elif log_type == "error":
+        logger.error(log_message)
+    elif log_type == "warning":
+        logger.warning(log_message)
+    else:
+        logger.debug(log_message)

logs/e111111_None_20250408.log

@@ -0,0 +1 @@
+2025-04-08 13:32:28 INFO USER_EID: e111111 ; PROJECT_NAME: None ; PAGE_NAME: Home ; MESSAGE: Project number 1 selected by employee e111111.

logs/e111111_na_demo_user_demo_project_20250407.log

@@ -0,0 +1 @@
+2025-04-07 18:10:49 INFO USER_EID: e111111 ; PROJECT_NAME: na_demo_user_demo_project ; PAGE_NAME: Home ; MESSAGE: Project number 642 selected by employee e111111.

logs/e111111_na_demo_user_demo_project_20250408.log

@@ -0,0 +1,2 @@
+2025-04-08 13:32:31 INFO USER_EID: e111111 ; PROJECT_NAME: na_demo_user_demo_project ; PAGE_NAME: Home ; MESSAGE: Project number 1 selected by employee e111111.
+2025-04-08 13:32:53 INFO USER_EID: e111111 ; PROJECT_NAME: na_demo_user_demo_project ; PAGE_NAME: Home ; MESSAGE: Project number 1 selected by employee e111111.

pages/10_Saved_Scenarios.py

@@ -0,0 +1,344 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="Saved Scenarios",
+    page_icon="⚖️",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+import io
+import sys
+import json
+import pickle
+import zipfile
+import traceback
+import numpy as np
+import pandas as pd
+from scenario import numerize
+from openpyxl import Workbook
+from post_gres_cred import db_cred
+from log_application import log_message
+from utilities import (
+    project_selection,
+    update_db,
+    set_header,
+    load_local_css,
+    name_formating,
+)
+
+schema = db_cred["schema"]
+load_local_css("styles.css")
+set_header()
+
+# Initialize project name session state
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+# Fetch project dictionary
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+# Display Username and Project Name
+if "username" in st.session_state and st.session_state["username"] is not None:
+
+    cols1 = st.columns([2, 1])
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+    # Function to get saved scenarios dictionary
+    def get_saved_scenarios_dict():
+        return st.session_state["project_dct"]["saved_scenarios"][
+            "saved_scenarios_dict"
+        ]
+
+
+# Function to format values based on their size
+def format_value(value):
+    return round(value, 4) if value < 1 else round(value, 1)
+
+
+# Function to recursively convert non-serializable types to serializable ones
+def convert_to_serializable(obj):
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_to_serializable(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_to_serializable(element) for element in obj]
+    elif isinstance(obj, (int, float, str, bool, type(None))):
+        return obj
+    else:
+        # Fallback: convert the object to a string
+        return str(obj)
+
+
+# Function to generate zip file of current scenario
+@st.cache_data(show_spinner=False)
+def download_as_zip(
+    df,
+    scenario_data,
+    excel_name="optimization_results.xlsx",
+    json_name="scenario_params.json",
+):
+    # Create an in-memory bytes buffer for the ZIP file
+    buffer = io.BytesIO()
+
+    # Create a ZipFile object in memory
+    with zipfile.ZipFile(buffer, "w") as zip_file:
+        # Save the DataFrame to an Excel file in the zip using openpyxl
+        excel_buffer = io.BytesIO()
+        workbook = Workbook()
+        sheet = workbook.active
+        sheet.title = "Results"
+
+        # Write DataFrame headers
+        for col_num, column_title in enumerate(df.columns, 1):
+            sheet.cell(row=1, column=col_num, value=column_title)
+
+        # Write DataFrame data
+        for row_num, row_data in enumerate(df.values, 2):
+            for col_num, cell_value in enumerate(row_data, 1):
+                sheet.cell(row=row_num, column=col_num, value=cell_value)
+
+        # Save the workbook to the in-memory buffer
+        workbook.save(excel_buffer)
+        excel_buffer.seek(0)  # Rewind the buffer to the beginning
+        zip_file.writestr(excel_name, excel_buffer.getvalue())
+
+        # Save the dictionary to a JSON file in the zip
+        json_buffer = io.BytesIO()
+        json_buffer.write(
+            json.dumps(convert_to_serializable(scenario_data), indent=4).encode("utf-8")
+        )
+        json_buffer.seek(0)  # Rewind the buffer to the beginning
+        zip_file.writestr(json_name, json_buffer.getvalue())
+
+    buffer.seek(0)  # Rewind the buffer to the beginning
+
+    return buffer
+
+
+# Function to delete the selected scenario from the saved scenarios dictionary
+def delete_selected_scenarios(selected_scenario):
+    if (
+        selected_scenario
+        in st.session_state["project_dct"]["saved_scenarios"]["saved_scenarios_dict"]
+    ):
+        del st.session_state["project_dct"]["saved_scenarios"]["saved_scenarios_dict"][
+            selected_scenario
+        ]
+
+
+try:
+    # Page Title
+    st.title("Saved Scenarios")
+
+    # Placeholder to display scenarios name
+    scenarios_name_placeholder = st.empty()
+
+    # Get saved scenarios dictionary and scenario name list
+    saved_scenarios_dict = get_saved_scenarios_dict()
+    scenarios_list = list(saved_scenarios_dict.keys())
+
+    # Check if the list of saved scenarios is empty
+    if len(scenarios_list) == 0:
+        # Display a warning message if no scenarios are saved
+        st.warning("No scenarios saved. Please save a scenario to load.", icon="⚠️")
+
+        # Log message
+        log_message(
+            "warning",
+            "No scenarios saved. Please save a scenario to load.",
+            "Saved Scenarios",
+        )
+
+        st.stop()
+
+    # Columns for scenario selection and save progress
+    select_scenario_col, save_progress_col = st.columns(2)
+    save_message_display_placeholder = st.container()
+
+    # Display a dropdown saved scenario list
+    selected_scenario = select_scenario_col.selectbox(
+        "Pick a Scenario", sorted(scenarios_list), key="selected_scenario"
+    )
+
+    # Save page progress
+    with save_progress_col:
+        st.write("###")
+        with save_message_display_placeholder, st.spinner("Saving Progress ..."):
+            if save_progress_col.button("Save Progress", use_container_width=True):
+                # Update DB
+                update_db(
+                    prj_id=st.session_state["project_number"],
+                    page_nam="Saved Scenarios",
+                    file_nam="project_dct",
+                    pkl_obj=pickle.dumps(st.session_state["project_dct"]),
+                    schema=schema,
+                )
+
+                # Display success message
+                st.success("Progress saved successfully!", icon="💾")
+                st.toast("Progress saved successfully!", icon="💾")
+
+                # Log message
+                log_message("info", "Progress saved successfully!", "Saved Scenarios")
+
+    selected_scenario_data = saved_scenarios_dict[selected_scenario]
+
+    # Scenarios Name
+    metrics_name = selected_scenario_data["metrics_selected"]
+    panel_name = selected_scenario_data["panel_selected"]
+    optimization_name = selected_scenario_data["optimization"]
+    multiplier = selected_scenario_data["multiplier"]
+    timeframe = selected_scenario_data["timeframe"]
+
+    # Display the scenario details with bold "Metric," "Panel," and "Optimization"
+    scenarios_name_placeholder.markdown(
+        f"**Metric**: {name_formating(metrics_name)}; **Panel**: {name_formating(panel_name)}; **Fix**: {name_formating(optimization_name)}; **Timeframe**: {name_formating(timeframe)}"
+    )
+
+    # Create columns for download and delete buttons
+    download_col, delete_col = st.columns(2)
+    save_message_display_placeholder = st.container()
+
+    # Channel List
+    channels_list = list(selected_scenario_data["channels"].keys())
+
+    # List to hold data for all channels
+    channels_data = []
+
+    # Iterate through each channel and gather required data
+    for channel in channels_list:
+        channel_conversion_rate = selected_scenario_data["channels"][channel][
+            "conversion_rate"
+        ]
+        channel_actual_spends = (
+            selected_scenario_data["channels"][channel]["actual_total_spends"]
+            * channel_conversion_rate
+        )
+        channel_optimized_spends = (
+            selected_scenario_data["channels"][channel]["modified_total_spends"]
+            * channel_conversion_rate
+        )
+
+        channel_actual_metrics = selected_scenario_data["channels"][channel][
+            "actual_total_sales"
+        ]
+        channel_optimized_metrics = selected_scenario_data["channels"][channel][
+            "modified_total_sales"
+        ]
+
+        channel_roi_mroi_data = selected_scenario_data["channel_roi_mroi"][channel]
+
+        # Extract the ROI and MROI data
+        actual_roi = channel_roi_mroi_data["actual_roi"]
+        optimized_roi = channel_roi_mroi_data["optimized_roi"]
+        actual_mroi = channel_roi_mroi_data["actual_mroi"]
+        optimized_mroi = channel_roi_mroi_data["optimized_mroi"]
+
+        # Calculate spends per metric
+        spends_per_metrics_actual = channel_actual_spends / channel_actual_metrics
+        spends_per_metrics_optimized = (
+            channel_optimized_spends / channel_optimized_metrics
+        )
+
+        # Append the collected data as a dictionary to the list
+        channels_data.append(
+            {
+                "Channel Name": channel,
+                "Spends Actual": numerize(channel_actual_spends / multiplier),
+                "Spends Optimized": numerize(channel_optimized_spends / multiplier),
+                f"{name_formating(metrics_name)} Actual": numerize(
+                    channel_actual_metrics / multiplier
+                ),
+                f"{name_formating(metrics_name)} Optimized": numerize(
+                    channel_optimized_metrics / multiplier
+                ),
+                "ROI Actual": format_value(actual_roi),
+                "ROI Optimized": format_value(optimized_roi),
+                "MROI Actual": format_value(actual_mroi),
+                "MROI Optimized": format_value(optimized_mroi),
+                f"Spends per {name_formating(metrics_name)} Actual": round(
+                    spends_per_metrics_actual, 2
+                ),
+                f"Spends per {name_formating(metrics_name)} Optimized": round(
+                    spends_per_metrics_optimized, 2
+                ),
+            }
+        )
+
+    # Create a DataFrame from the collected data
+    df = pd.DataFrame(channels_data)
+
+    # Display the DataFrame
+    st.dataframe(df, hide_index=True)
+
+    # Generate download able data for selected scenario
+    buffer = download_as_zip(
+        df,
+        selected_scenario_data,
+        excel_name="optimization_results.xlsx",
+        json_name="scenario_params.json",
+    )
+
+    # Provide the buffer as a downloadable ZIP file
+    file_name = f"{selected_scenario}_scenario_data.zip"
+    if download_col.download_button(
+        label="Download",
+        data=buffer,
+        file_name=file_name,
+        mime="application/zip",
+        use_container_width=True,
+    ):
+        # Log message
+        log_message(
+            "info",
+            f"FILE_NAME: {file_name} has been successfully downloaded.",
+            "Saved Scenarios",
+        )
+
+    # Button to trigger the deletion of the selected scenario
+    if delete_col.button(
+        "Delete",
+        use_container_width=True,
+        on_click=delete_selected_scenarios,
+        args=(selected_scenario,),
+    ):
+        # Display success message
+        with save_message_display_placeholder:
+            st.success(
+                "Selected scenario successfully deleted. Click the 'Save Progress' button to ensure your changes are updated!",
+                icon="🗑️",
+            )
+            st.toast(
+                "Selected scenario successfully deleted. Click the 'Save Progress' button to ensure your changes are updated!",
+                icon="🗑️",
+            )
+
+            # Log message
+            log_message(
+                "info", "Selected scenario successfully deleted.", "Saved Scenarios"
+            )
+
+except Exception as e:
+    # Capture the error details
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+
+    # Log message
+    log_message("error", f"An error occurred: {error_message}.", "Saved Scenarios")
+
+    # Display a warning message
+    st.warning(
+        "Oops! Something went wrong. Please try refreshing the tool or creating a new project.",
+        icon="⚠️",
+    )

pages/11_AI_Model_Media_Recommendation.py

@@ -0,0 +1,695 @@
+import streamlit as st
+from scenario import numerize
+import pandas as pd
+from utilities import (
+    format_numbers,
+    load_local_css,
+    set_header,
+    name_formating,
+    project_selection,
+)
+import pickle
+import yaml
+from yaml import SafeLoader
+from scenario import class_from_dict
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+import pandas as pd
+from plotly.subplots import make_subplots
+import sqlite3
+from utilities import update_db
+from collections import OrderedDict
+import os
+
+st.set_page_config(layout="wide")
+load_local_css("styles.css")
+set_header()
+
+st.empty()
+st.header("AI Model Media Recommendation")
+
+# def get_saved_scenarios_dict():
+#     # Path to the saved scenarios file
+#     saved_scenarios_dict_path = os.path.join(
+#         st.session_state["project_path"], "saved_scenarios.pkl"
+#     )
+
+#     # Load existing scenarios if the file exists
+#     if os.path.exists(saved_scenarios_dict_path):
+#         with open(saved_scenarios_dict_path, "rb") as f:
+#             saved_scenarios_dict = pickle.load(f)
+#     else:
+#         saved_scenarios_dict = OrderedDict()
+
+#     return saved_scenarios_dict
+
+
+# # Function to format values based on their size
+# def format_value(value):
+#     return round(value, 4) if value < 1 else round(value, 1)
+
+
+# # Function to recursively convert non-serializable types to serializable ones
+# def convert_to_serializable(obj):
+#     if isinstance(obj, np.ndarray):
+#         return obj.tolist()
+#     elif isinstance(obj, dict):
+#         return {key: convert_to_serializable(value) for key, value in obj.items()}
+#     elif isinstance(obj, list):
+#         return [convert_to_serializable(element) for element in obj]
+#     elif isinstance(obj, (int, float, str, bool, type(None))):
+#         return obj
+#     else:
+#         # Fallback: convert the object to a string
+#         return str(obj)
+
+
+if "username" not in st.session_state:
+    st.session_state["username"] = None
+
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+# if   "project_path" not in st.session_state:
+#     st.stop()
+# if 'username' in st.session_state and st.session_state['username'] is not None:
+
+#     data_path = os.path.join(st.session_state["project_path"], "data_import.pkl")
+
+#     try:
+#         with open(data_path, "rb") as f:
+#             data = pickle.load(f)
+#     except Exception as e:
+#         st.error(f"Please import data from the Data Import Page")
+#         st.stop()
+# # Get saved scenarios dictionary and scenario name list
+# saved_scenarios_dict = get_saved_scenarios_dict()
+# scenarios_list = list(saved_scenarios_dict.keys())
+
+# #st.write(saved_scenarios_dict)
+# # Check if the list of saved scenarios is empty
+# if len(scenarios_list) == 0:
+#     # Display a warning message if no scenarios are saved
+#     st.warning("No scenarios saved. Please save a scenario to load.", icon="⚠️")
+#     st.stop()
+
+# # Display a dropdown saved scenario list
+# selected_scenario = st.selectbox(
+#     "Pick a Scenario", sorted(scenarios_list), key="selected_scenario"
+# )
+# selected_scenario_data = saved_scenarios_dict[selected_scenario]
+
+# # Scenarios Name
+# metrics_name = selected_scenario_data["metrics_selected"]
+# panel_name = selected_scenario_data["panel_selected"]
+# optimization_name = selected_scenario_data["optimization"]
+
+# # Display the scenario details with bold "Metric," "Panel," and "Optimization"
+
+# # Create columns for download and delete buttons
+# download_col, delete_col = st.columns(2)
+
+
+# channels_list = list(selected_scenario_data["channels"].keys())
+
+# # List to hold data for all channels
+# channels_data = []
+
+# # Iterate through each channel and gather required data
+# for channel in channels_list:
+#     channel_conversion_rate = selected_scenario_data["channels"][channel][
+#         "conversion_rate"
+#     ]
+#     channel_actual_spends = (
+#         selected_scenario_data["channels"][channel]["actual_total_spends"]
+#         * channel_conversion_rate
+#     )
+#     channel_optimized_spends = (
+#         selected_scenario_data["channels"][channel]["modified_total_spends"]
+#         * channel_conversion_rate
+#     )
+
+#     channel_actual_metrics = selected_scenario_data["channels"][channel][
+#         "actual_total_sales"
+#     ]
+#     channel_optimized_metrics = selected_scenario_data["channels"][channel][
+#         "modified_total_sales"
+#     ]
+
+#     channel_roi_mroi_data = selected_scenario_data["channel_roi_mroi"][channel]
+
+#     # Extract the ROI and MROI data
+#     actual_roi = channel_roi_mroi_data["actual_roi"]
+#     optimized_roi = channel_roi_mroi_data["optimized_roi"]
+#     actual_mroi = channel_roi_mroi_data["actual_mroi"]
+#     optimized_mroi = channel_roi_mroi_data["optimized_mroi"]
+
+#     # Calculate spends per metric
+#     spends_per_metrics_actual = channel_actual_spends / channel_actual_metrics
+#     spends_per_metrics_optimized = channel_optimized_spends / channel_optimized_metrics
+
+#     # Append the collected data as a dictionary to the list
+#     channels_data.append(
+#         {
+#             "Channel Name": channel,
+#             "Spends Actual": channel_actual_spends,
+#             "Spends Optimized": channel_optimized_spends,
+#             f"{metrics_name} Actual": channel_actual_metrics,
+#             f"{name_formating(metrics_name)} Optimized": numerize(
+#                 channel_optimized_metrics
+#             ),
+#             "ROI Actual": format_value(actual_roi),
+#             "ROI Optimized": format_value(optimized_roi),
+#             "MROI Actual": format_value(actual_mroi),
+#             "MROI Optimized": format_value(optimized_mroi),
+#             f"Spends per {name_formating(metrics_name)} Actual": numerize(
+#                 spends_per_metrics_actual
+#             ),
+#             f"Spends per {name_formating(metrics_name)} Optimized": numerize(
+#                 spends_per_metrics_optimized
+#             ),
+#         }
+#     )
+
+# # Create a DataFrame from the collected data
+
+##NEW CODE##########
+
+scenarios_name_placeholder = st.empty()
+
+
+# Function to get saved scenarios dictionary
+def get_saved_scenarios_dict():
+    return st.session_state["project_dct"]["saved_scenarios"]["saved_scenarios_dict"]
+
+
+# Function to format values based on their size
+def format_value(value):
+    return round(value, 4) if value < 1 else round(value, 1)
+
+
+# Function to recursively convert non-serializable types to serializable ones
+def convert_to_serializable(obj):
+    if isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_to_serializable(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_to_serializable(element) for element in obj]
+    elif isinstance(obj, (int, float, str, bool, type(None))):
+        return obj
+    else:
+        # Fallback: convert the object to a string
+        return str(obj)
+
+
+# Get saved scenarios dictionary and scenario name list
+saved_scenarios_dict = get_saved_scenarios_dict()
+scenarios_list = list(saved_scenarios_dict.keys())
+
+# Check if the list of saved scenarios is empty
+if len(scenarios_list) == 0:
+    # Display a warning message if no scenarios are saved
+    st.warning("No scenarios saved. Please save a scenario to load.", icon="⚠️")
+    st.stop()
+
+# Display a dropdown saved scenario list
+selected_scenario = st.selectbox(
+    "Pick a Scenario", sorted(scenarios_list), key="selected_scenario"
+)
+selected_scenario_data = saved_scenarios_dict[selected_scenario]
+
+# Scenarios Name
+metrics_name = selected_scenario_data["metrics_selected"]
+panel_name = selected_scenario_data["panel_selected"]
+optimization_name = selected_scenario_data["optimization"]
+multiplier = selected_scenario_data["multiplier"]
+timeframe = selected_scenario_data["timeframe"]
+
+# Display the scenario details with bold "Metric," "Panel," and "Optimization"
+scenarios_name_placeholder.markdown(
+    f"**Metric**: {name_formating(metrics_name)}; **Panel**: {name_formating(panel_name)}; **Fix**: {name_formating(optimization_name)}; **Timeframe**: {name_formating(timeframe)}"
+)
+
+# Create columns for download and delete buttons
+download_col, delete_col = st.columns(2)
+
+# Channel List
+channels_list = list(selected_scenario_data["channels"].keys())
+
+# List to hold data for all channels
+channels_data = []
+
+# Iterate through each channel and gather required data
+for channel in channels_list:
+    channel_conversion_rate = selected_scenario_data["channels"][channel][
+        "conversion_rate"
+    ]
+    channel_actual_spends = (
+        selected_scenario_data["channels"][channel]["actual_total_spends"]
+        * channel_conversion_rate
+    )
+    channel_optimized_spends = (
+        selected_scenario_data["channels"][channel]["modified_total_spends"]
+        * channel_conversion_rate
+    )
+
+    channel_actual_metrics = selected_scenario_data["channels"][channel][
+        "actual_total_sales"
+    ]
+    channel_optimized_metrics = selected_scenario_data["channels"][channel][
+        "modified_total_sales"
+    ]
+
+    channel_roi_mroi_data = selected_scenario_data["channel_roi_mroi"][channel]
+
+    # Extract the ROI and MROI data
+    actual_roi = channel_roi_mroi_data["actual_roi"]
+    optimized_roi = channel_roi_mroi_data["optimized_roi"]
+    actual_mroi = channel_roi_mroi_data["actual_mroi"]
+    optimized_mroi = channel_roi_mroi_data["optimized_mroi"]
+
+    # Calculate spends per metric
+    spends_per_metrics_actual = channel_actual_spends / channel_actual_metrics
+    spends_per_metrics_optimized = channel_optimized_spends / channel_optimized_metrics
+
+    # Append the collected data as a dictionary to the list
+    channels_data.append(
+        {
+            "Channel Name": channel,
+            "Spends Actual": (channel_actual_spends / multiplier),
+            "Spends Optimized": (channel_optimized_spends / multiplier),
+            f"{name_formating(metrics_name)} Actual": (
+                channel_actual_metrics / multiplier
+            ),
+            f"{name_formating(metrics_name)} Optimized": (
+                channel_optimized_metrics / multiplier
+            ),
+            "ROI Actual": format_value(actual_roi),
+            "ROI Optimized": format_value(optimized_roi),
+            "MROI Actual": format_value(actual_mroi),
+            "MROI Optimized": format_value(optimized_mroi),
+            f"Spends per {name_formating(metrics_name)} Actual": round(
+                spends_per_metrics_actual, 2
+            ),
+            f"Spends per {name_formating(metrics_name)} Optimized": round(
+                spends_per_metrics_optimized, 2
+            ),
+        }
+    )
+
+# Create a DataFrame from the collected data
+# df = pd.DataFrame(channels_data)
+
+# # Display the DataFrame
+# st.dataframe(df, hide_index=True)
+
+summary_df_sorted = pd.DataFrame(channels_data).sort_values(by=["Spends Optimized"])
+
+
+summary_df_sorted["Delta"] = (
+    summary_df_sorted["Spends Optimized"] - summary_df_sorted["Spends Actual"]
+)
+
+
+summary_df_sorted["Delta_percent"] = np.round(
+    (summary_df_sorted["Delta"]) / summary_df_sorted["Spends Actual"] * 100, 2
+)
+
+# spends_data = pd.read_excel("Overview_data_test.xlsx")
+
+
+st.header("Optimized Media Spend Overview")
+
+channel_colors = px.colors.qualitative.Plotly
+
+fig = make_subplots(
+    rows=1,
+    cols=3,
+    subplot_titles=("Actual Spend", "Spends Optimized", "Delta"),
+    horizontal_spacing=0.05,
+)
+
+for i, channel in enumerate(summary_df_sorted["Channel Name"].unique()):
+    channel_df = summary_df_sorted[summary_df_sorted["Channel Name"] == channel]
+    channel_color = channel_colors[i % len(channel_colors)]
+
+    fig.add_trace(
+        go.Bar(
+            x=channel_df["Spends Actual"],
+            y=channel_df["Channel Name"],
+            text=channel_df["Spends Actual"].apply(format_numbers),
+            marker_color=channel_color,
+            orientation="h",
+        ),
+        row=1,
+        col=1,
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=channel_df["Spends Optimized"],
+            y=channel_df["Channel Name"],
+            text=channel_df["Spends Optimized"].apply(format_numbers),
+            marker_color=channel_color,
+            orientation="h",
+            showlegend=False,
+        ),
+        row=1,
+        col=2,
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=channel_df["Delta_percent"],
+            y=channel_df["Channel Name"],
+            text=channel_df["Delta_percent"].apply(lambda x: f"{x:.0f}%"),
+            marker_color=channel_color,
+            orientation="h",
+            showlegend=False,
+        ),
+        row=1,
+        col=3,
+    )
+fig.update_layout(height=600, width=900, title="", showlegend=False)
+
+fig.update_yaxes(showticklabels=False, row=1, col=2)
+fig.update_yaxes(showticklabels=False, row=1, col=3)
+
+fig.update_xaxes(showticklabels=False, row=1, col=1)
+fig.update_xaxes(showticklabels=False, row=1, col=2)
+fig.update_xaxes(showticklabels=False, row=1, col=3)
+
+
+st.plotly_chart(fig, use_container_width=True)
+
+
+summary_df_sorted["Perc_alloted"] = np.round(
+    summary_df_sorted["Spends Optimized"] / summary_df_sorted["Spends Optimized"].sum(),
+    2,
+)
+st.header("Optimized Media Spend Allocation")
+
+fig = make_subplots(
+    rows=1,
+    cols=2,
+    subplot_titles=("Spends Optimized", "% Split"),
+    horizontal_spacing=0.05,
+)
+
+for i, channel in enumerate(summary_df_sorted["Channel Name"].unique()):
+    channel_df = summary_df_sorted[summary_df_sorted["Channel Name"] == channel]
+    channel_color = channel_colors[i % len(channel_colors)]
+
+    fig.add_trace(
+        go.Bar(
+            x=channel_df["Spends Optimized"],
+            y=channel_df["Channel Name"],
+            text=channel_df["Spends Optimized"].apply(format_numbers),
+            marker_color=channel_color,
+            orientation="h",
+        ),
+        row=1,
+        col=1,
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=channel_df["Perc_alloted"],
+            y=channel_df["Channel Name"],
+            text=channel_df["Perc_alloted"].apply(lambda x: f"{100*x:.0f}%"),
+            marker_color=channel_color,
+            orientation="h",
+            showlegend=False,
+        ),
+        row=1,
+        col=2,
+    )
+
+fig.update_layout(height=600, width=900, title="", showlegend=False)
+
+fig.update_yaxes(showticklabels=False, row=1, col=2)
+fig.update_yaxes(showticklabels=False, row=1, col=3)
+
+fig.update_xaxes(showticklabels=False, row=1, col=1)
+fig.update_xaxes(showticklabels=False, row=1, col=2)
+fig.update_xaxes(showticklabels=False, row=1, col=3)
+
+
+st.plotly_chart(fig, use_container_width=True)
+
+
+st.session_state["cleaned_data"] = st.session_state["project_dct"]["data_import"][
+    "imputed_tool_df"
+]
+st.session_state["category_dict"] = st.session_state["project_dct"]["data_import"][
+    "category_dict"
+]
+
+effectiveness_overall = pd.DataFrame()
+
+response_metrics = list(
+    *[
+        st.session_state["category_dict"][key]
+        for key in st.session_state["category_dict"].keys()
+        if key == "Response Metrics"
+    ]
+)
+
+effectiveness_overall = (
+    st.session_state["cleaned_data"][response_metrics].sum().reset_index()
+)
+
+effectiveness_overall.columns = ["ResponseMetricName", "ResponseMetricValue"]
+
+
+effectiveness_overall["Efficiency"] = effectiveness_overall["ResponseMetricValue"].map(
+    lambda x: x / summary_df_sorted["Spends Optimized"].sum()
+)
+
+
+columns6 = st.columns(3)
+
+effectiveness_overall.sort_values(
+    by=["ResponseMetricValue"], ascending=False, inplace=True
+)
+effectiveness_overall = np.round(effectiveness_overall, 2)
+
+columns4 = st.columns([0.55, 0.45])
+
+# effectiveness_overall = effectiveness_overall.sort_values(by=["ResponseMetricValue"])
+
+# with columns4[0]:
+#     fig = px.funnel(
+#         effectiveness_overall,
+#         x="ResponseMetricValue",
+#         y="ResponseMetricName",
+#         color="ResponseMetricName",
+#         title="Effectiveness",
+#     )
+#     fig.update_layout(
+#         showlegend=False,
+#         yaxis=dict(tickmode="array"),
+#     )
+#     fig.update_traces(
+#         textinfo="value",
+#         textposition="inside",
+#         texttemplate="%{x:.2s} ",
+#         hoverinfo="y+x+percent initial",
+#     )
+#     fig.update_traces(
+#         marker=dict(line=dict(color="black", width=2)),
+#         selector=dict(marker=dict(color="blue")),
+#     )
+
+#     st.plotly_chart(fig, use_container_width=True)
+
+# with columns4[1]:
+#     fig1 = px.bar(
+#         effectiveness_overall.sort_values(by=["ResponseMetricValue"], ascending=False),
+#         x="Efficiency",
+#         y="ResponseMetricName",
+#         color="ResponseMetricName",
+#         text_auto=True,
+#         title="Efficiency",
+#     )
+
+#     # Update layout and traces
+#     fig1.update_traces(
+#         customdata=effectiveness_overall["Efficiency"], textposition="auto"
+#     )
+#     fig1.update_layout(showlegend=False)
+#     fig1.update_yaxes(title="", showticklabels=False)
+#     fig1.update_xaxes(title="", showticklabels=False)
+#     fig1.update_xaxes(tickfont=dict(size=20))
+#     fig1.update_yaxes(tickfont=dict(size=20))
+#     st.plotly_chart(fig1, use_container_width=True)
+
+# Function to format metric names
+def format_metric_name(metric_name):
+    return str(metric_name).lower().replace("response_metric_", "").replace("_", " ").strip().title()
+
+# Apply the formatting function to the 'ResponseMetricName' column
+effectiveness_overall["FormattedMetricName"] = effectiveness_overall["ResponseMetricName"].apply(format_metric_name)
+
+# Multiselect widget with all options as default, but using the formatted names for display
+all_metrics = effectiveness_overall["FormattedMetricName"].unique()
+selected_metrics = st.multiselect(
+    "Select Metrics to Display",
+    options=all_metrics,
+    default=all_metrics
+)
+
+# Filter the data based on the selected metrics (using formatted names)
+if selected_metrics:
+    filtered_data = effectiveness_overall[
+        effectiveness_overall["FormattedMetricName"].isin(selected_metrics)
+    ]
+    
+    # Sort values for funnel plot
+    filtered_data = filtered_data.sort_values(by=["ResponseMetricValue"])
+
+    # Generate a consistent color mapping for all selected metrics
+    color_map = {metric: px.colors.qualitative.Plotly[i % len(px.colors.qualitative.Plotly)] 
+                 for i, metric in enumerate(filtered_data["FormattedMetricName"].unique())}
+
+    # First plot: Funnel
+    with columns4[0]:
+        fig = px.funnel(
+            filtered_data,
+            x="ResponseMetricValue",
+            y="FormattedMetricName",  # Use formatted names for y-axis
+            color="FormattedMetricName",  # Use formatted names for color
+            color_discrete_map=color_map,  # Ensure consistent colors
+            title="Effectiveness",
+        )
+        fig.update_layout(
+            showlegend=False,
+            yaxis=dict(title="Response Metric", tickmode="array"),  # Set y-axis label to 'Response Metric'
+        )
+        fig.update_traces(
+            textinfo="value",
+            textposition="inside",
+            texttemplate="%{x:.2s} ",
+            hoverinfo="y+x+percent initial",
+        )
+        fig.update_traces(
+            marker=dict(line=dict(color="black", width=2)),
+            selector=dict(marker=dict(color="blue")),
+        )
+
+        st.plotly_chart(fig, use_container_width=True)
+
+    # Second plot: Bar chart
+    with columns4[1]:
+        fig1 = px.bar(
+            filtered_data.sort_values(by=["ResponseMetricValue"], ascending=False),
+            x="Efficiency",
+            y="FormattedMetricName",  # Use formatted names for y-axis
+            color="FormattedMetricName",  # Use formatted names for color
+            color_discrete_map=color_map,  # Ensure consistent colors
+            text_auto=True,
+            title="Efficiency",
+        )
+
+        # Update layout and traces
+        fig1.update_traces(
+            customdata=filtered_data["Efficiency"], textposition="auto"
+        )
+        fig1.update_layout(showlegend=False)
+        fig1.update_yaxes(title="", showticklabels=False)
+        fig1.update_xaxes(title="", showticklabels=False)
+        fig1.update_xaxes(tickfont=dict(size=20))
+        fig1.update_yaxes(tickfont=dict(size=20))
+        st.plotly_chart(fig1, use_container_width=True)
+else:
+    st.info("Please select at least one response metric to display the charts.")
+
+st.header("Return Forecast by Media Channel")
+
+with st.expander("Return Forecast by Media Channel"):
+
+
+    metric = metrics_name
+
+    metric = metric.lower().replace("_", " ") + " " + "actual"
+    summary_df_sorted.columns = [
+        col.lower().replace("_", " ") for col in summary_df_sorted.columns
+    ]
+
+    effectiveness = summary_df_sorted[metric]
+
+    summary_df_sorted["Efficiency"] = (
+        summary_df_sorted[metric] / summary_df_sorted["spends optimized"]
+    )
+
+    channel_colors = px.colors.qualitative.Plotly
+
+    fig = make_subplots(
+        rows=1,
+        cols=3,
+        subplot_titles=("Optimized Spends", "Effectiveness", "Efficiency"),
+        horizontal_spacing=0.05,
+    )
+
+    for i, channel in enumerate(summary_df_sorted["channel name"].unique()):
+        channel_df = summary_df_sorted[summary_df_sorted["channel name"] == channel]
+        channel_color = channel_colors[i % len(channel_colors)]
+
+        fig.add_trace(
+            go.Bar(
+                x=channel_df["spends optimized"],
+                y=channel_df["channel name"],
+                text=channel_df["spends optimized"].apply(format_numbers),
+                marker_color=channel_color,
+                orientation="h",
+            ),
+            row=1,
+            col=1,
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=channel_df[metric],
+                y=channel_df["channel name"],
+                text=channel_df[metric].apply(format_numbers),
+                marker_color=channel_color,
+                orientation="h",
+                showlegend=False,
+            ),
+            row=1,
+            col=2,
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=channel_df["Efficiency"],
+                y=channel_df["channel name"],
+                text=channel_df["Efficiency"].apply(lambda x: f"{x:.2f}"),
+                marker_color=channel_color,
+                orientation="h",
+                showlegend=False,
+            ),
+            row=1,
+            col=3,
+        )
+
+    fig.update_layout(
+        height=600,
+        width=900,
+        title="Media Channel Performance",
+        showlegend=False,
+    )
+
+    fig.update_yaxes(showticklabels=False, row=1, col=2)
+    fig.update_yaxes(showticklabels=False, row=1, col=3)
+
+    fig.update_xaxes(showticklabels=False, row=1, col=1)
+    fig.update_xaxes(showticklabels=False, row=1, col=2)
+    fig.update_xaxes(showticklabels=False, row=1, col=3)
+
+    st.plotly_chart(fig, use_container_width=True)

pages/12_Glossary.py

@@ -0,0 +1,150 @@
+# Importing necessary libraries
+import streamlit as st
+import base64
+
+from utilities import set_header, load_local_css
+
+st.set_page_config(
+    page_title="Glossary",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+load_local_css("styles.css")
+set_header()
+
+
+st.header("Glossary")
+
+# Glossary
+st.markdown(
+    """
+
+### 1. Glossary - Home​
+**Blend Employee ID:** Users should have access to the UI through their Blend Employee ID. User data should be available and validated through this ID.​  
+
+**API Data:** Incorporate data retrieved from APIs to be used in the tool.​  
+
+### 2. Glossary – Data Import​
+**Granularity:** Defines the level of detail to which the data should be aggregated.​  
+
+**Panel:** Represents columns corresponding to markets, DMAs (Designated Market Areas).​  
+
+**Response Metrics:** Target variables or metrics such as App Installs, Revenue, Form Submission/Conversion. These are the variables used for model building and spends optimization.​  
+
+**Spends:** Variables representing spend data for all media channels.​  
+
+**Exogenous:** External variables like bank holidays, GDP, online trends, interest rates.​  
+
+**Internal:** Internal variables such as discounts or promotions.​  
+
+### 3. Glossary – Data Assessment​
+**Trendline:** Represents the linear average movement over the entire time period.​  
+
+**Media Variables:** Variables related to media activities such as Impressions, Clicks, Spends, Views. Examples: Bing Search Clicks, Instagram Impressions, YouTube Impressions.​  
+
+**Non-Media Variables:** Variables that exclude media-related data. Examples: Discount, Holiday, Google Trend.​  
+
+**CPM (Cost per 1000 Impressions):** Calculated as (YouTube Spends / YouTube Impressions) * 1000.​  
+
+**CPC (Cost per 1000 Clicks):** Calculated as (YouTube Spends / YouTube Clicks) * 1000.​  
+
+**Correlation:** Pearson correlation measures the linear relationship between two continuous variables, indicating the strength and direction of their association.​  
+
+### 4. Glossary – Transformations​
+
+**Transformations:** Transformations involve adjusting input variables to capture nonlinear relationships, like diminishing returns and advertising carryover. They are always applied at the panel level if panel exist; if no panel exists, the transformations are applied directly to the aggregated level.
+
+**Lag:** Shifts the data backward by a specified number of periods. Formula: Lagged Series = Xt−lag
+
+**Lead:** Shifts the data forward by a specified number of periods. Formula: Lead Series = Xt+lead  
+
+**Moving Average:** Smooths the data by averaging values over a specified window size. Formula: Moving Average =  1/𝑛 ∑1_0^(𝑛−1)𝑋_(𝑡−1)  
+
+**Saturation:** Applies a saturation effect to the data based on a specified saturation percentage. Formula: 𝑌_𝑡= (1/(1 +[(𝑠𝑎𝑡𝑢𝑟𝑎𝑡𝑖𝑜𝑛 𝑝𝑜𝑖𝑛𝑡)/𝑋_𝑡 ]^𝑠𝑡𝑒𝑒𝑝𝑛𝑒𝑠𝑠 ))  × 𝑋_𝑡  
+
+**Power:** Raises the data to a specified power. Formula: Yt = Xtpower  
+
+**Adstock:** Applies a decay effect to the data, simulating diminishing returns over time. Formula: Yt = Xt + decay rate x Yt-1  
+
+### 5. Glossary - AI Model Build
+**Train Set:** The train set is a subset of the data on which the AI Model is trained. It is standard practice to select 70-75% of the data as train set.​  
+
+**Test Set:** The test set is the subset of data on which the AI Model’s performance is tested. There will be no common records between the train and test sets.​  
+
+**Residual:** Residual is defined as the difference between true value and predicted value from the AI Model. 𝑅𝑒𝑠𝑖𝑑𝑢𝑎𝑙=𝑇𝑟𝑢𝑒 𝑉𝑎𝑙𝑢𝑒 −𝑃𝑟𝑒𝑑𝑖𝑐𝑡𝑒𝑑 𝑉𝑎𝑙𝑢𝑒  
+
+**Actual VS Predicted Plot:** An actual vs. predicted plot visualizes the relationship between the actual values and the values predicted by the AI model, helping to assess model performance and identify patterns or discrepancies.​  
+
+**MAPE:** MAPE or Mean Absolute Percentage Error indicates the percentage of error in the AI Model’s predictions. We use a variant of MAPE called Weighted Average Percentage Error. 𝑀𝐴𝑃𝐸= ∑ |𝐴𝑐𝑡𝑢𝑎𝑙 𝑉𝑎𝑙𝑢𝑒 −𝑃𝑟𝑒𝑑𝑖𝑐𝑡𝑒𝑑 𝑉𝑎𝑙𝑢𝑒| ÷∑ [|𝐴𝑐𝑡𝑢𝑎𝑙 𝑉𝑎𝑙𝑢𝑒|]
+
+**R-Squared:** R-Squared is a number that tells you how well the independent variable(s) in a AI model explain the variation in the dependent variable. Indicates the goodness of fit, with values closer to 1 suggesting a better fit.​  
+
+**Adjusted R-Squared:** Adjusted R-squared modifies the R-squared value to account for the number of predictors in the model, providing a more accurate measure of goodness of fit by penalizing the addition of non-significant predictors.​  
+
+**Multicollinearity:** Multicollinearity refers to a situation where two or more independent variables (media channels or marketing inputs) are highly correlated with each other. This makes it difficult to isolate the individual effect of each channel on the dependent variable (e.g., sales). It can lead to unreliable coefficient estimates, making it hard to determine the true impact of each media type.
+
+### 6. Glossary – AI Model Tuning
+**Event:** An event refers to a specific occurrence or campaign, such as a promotion, holiday, or product launch, that can impact the performance of response metric​  
+
+**Trend:** Trend is a straight line which helps capture the underlying direction or pattern in the data over time, helping to identify and quantify the increasing or decreasing trend of the dependent variable.​  
+
+**Day of Week:** The "day of week" feature represents the specific day within a week (e.g., Monday, Tuesday) and is used to capture patterns or seasonal effects that occur on specific days.​  
+
+**Sine & Cosine Waves:** Sine and cosine waves are mathematical functions used to capture seasonality in the dependent variable​  
+
+**Contribution:** Contribution of a channel is the percentage of its contribution to the response metric. Contribution is an output of the AI Model, calculated using the media data and model’s coefficients​  
+
+### 7. Glossary – Response Curves
+
+**Response Curve:** A response curve in media mix modeling represents the relationship between media inputs (e.g., impressions, clicks) on the X-axis and the resulting business outcomes (e.g., revenue, app installs) on the Y-axis, illustrating the impact of media variables on the desired response metric.
+
+**R-squared (R-squared):** R-squared (R-squared) is a statistical measure that represents the proportion of the variance in the dependent variable that is predictable from the independent variables. It indicates how well the regression model fits the data, with a value between 0 and 1, where 1 indicates a perfect fit.  
+**Formula for R-squared (R-squared):**  
+R-squared = 1 − SSres / SStot  
+Where:  
+- **SSres** is the sum of squares of residuals (errors).
+- **SStot** is the total sum of squares (the variance of the dependent variable).
+
+**Actual R-squared:** Actual R-squared is used to evaluate how well a s-curve fits the data. Modified R-squared is used to evaluate how well a modified (manually tuned) s-curve fits the data. The difference between modified s-curve and actual s-curve shows how much the fit improves or worsens.
+
+### 8. Glossary – Scenario Planner
+
+**CPA (Cost Per Acquisition):** The cost associated with acquiring a customer or conversion. CPA = Total Spend / Number of Acquisitions  
+
+**ROI (Return on Investment):** A measure of the profitability of an investment relative to its cost. ROI = (Revenue - Total Spend) / Total Spend  
+
+**mROI (Marginal Return on Investment):** Measures the additional return generated by an additional unit of investment. It represents the slope of the ROI curve and shows the incremental effectiveness of the investment. mROI = Δspend / ΔRevenue​  
+
+**Bounds (Upper, Lower):** Constraints applied during optimization to ensure that media channel spends remain within specified limits. These bounds help in defining the feasible range for investments and prevent overspending or underspending.
+
+**Panel & Timeframe:**
+- **Panel:** The ability to optimize media strategies at a granular level based on specific categories such as geographies, product types, customer segments, or other defined groups. This allows for tailored strategies that address the unique characteristics and needs of each panel.
+- **Timeframe:** The ability to optimize media strategies within specific time periods, such as by month, quarter, or year. This enables precise adjustments, ensuring that the approach aligns with specific business cycles.
+
+**Actual Vs Optimized:**
+- **Actual:** This category includes the unoptimized values, such as the actual spends and actual response metrics, reflecting the current or historical performance without any optimization applied.
+- **Optimized:** This category encompasses the optimized values, such as optimized spends and optimized response metrics, representing the improved or adjusted figures after the optimization process.
+
+**Regions:**
+- **Yellow Region:** This represents the under-invested area where additional investments can lead to entering a region with a higher ROI than the average. Investing more in this region can yield better returns.
+- **Green Region:** This is the optimal area where the ROI of the channel is above the average ROI, and the Marginal Return on Investment (mROI) is greater than 1, indicating that each additional dollar spent is generating substantial returns.
+- **Red Region:** This represents the over-invested or saturated area where the mROI is less than 1, meaning that additional spending yields diminishing returns and is less effective.
+
+**Important Formulas:���**  
+**1. Incremental CPA (%)** = (Total Optimized CPA−Total Actual CPA) / (Total Actual CPA​)​  
+**2. Incremental Spend (%)** = (Total Optimized Spend−Total Actual Spend) / (Total Actual Spend​)  
+    **Note:** Calculated based on total actual spends (base spends is always zero)​  
+**3. Incremental Response Metric (%) [Media]** = (Total Optimized Response Metric−Total Actual Response​ Metric) / (Total Actual Response Metric−Total Base Response Metric​)  
+    **Note:** Calculated based on media portion of total actual response metric only, excluding the fixed base contribution  
+**4. Incremental Response Metric (%) [Total]** = (Total Optimized Response Metric−Total Actual Response​ Metric) / (Total Actual Response Metric​)  
+    **Note:** Calculated based on total actual response metric only, including the fixed base contribution
+
+### 10. Glossary – Model Optimized Recommendation
+
+**% Split:** The percentage distribution of total media spend across different channels, optimized to achieve the best results.
+
+**Return forecast by Media Channel:** Effectiveness and Efficiency for selected response metric.
+"""
+)

pages/14_User_Management.py

@@ -0,0 +1,503 @@
+import streamlit as st
+
+st.set_page_config(layout="wide")
+import pandas as pd
+from utilities import (
+    load_local_css,
+    set_header,
+    query_excecuter_postgres,
+    store_hashed_password,
+    verify_password,
+    is_pswrd_flag_set,
+)
+import psycopg2
+
+#
+import bcrypt
+import re
+import time
+import random
+import string
+from log_application import log_message
+
+# setting page config
+load_local_css("styles.css")
+set_header()
+
+# schema=db_cred['schema']
+
+db_cred = None
+
+
+def fetch_password_hash_key():
+    query = f"""
+            SELECT emp_id 
+            FROM mmo_users
+            WHERE emp_nam ='admin'; 
+            """
+    hashkey = query_excecuter_postgres(query, db_cred, insert=False)
+
+    return hashkey
+
+
+def fetch_users_with_access():
+    # Query to get allowed employee IDs for the project
+    unique_users_query = f"""
+                        SELECT DISTINCT emp_id, emp_nam, emp_typ
+                        FROM mmo_users;
+                                         """
+    try:
+        unique_users_result = query_excecuter_postgres(
+            unique_users_query, db_cred, insert=False
+        )
+
+        unique_users_result_df = pd.DataFrame(unique_users_result)
+        unique_users_result_df.columns = ["User ID", "User Name", "User Type"]
+
+        st.session_state["df_users"] = unique_users_result_df
+    except:
+        st.session_state["df_users"] = pd.DataFrame()
+
+
+def add_user(emp_id, emp_name, emp_type, admin_id, plain_text_password):
+    """
+    Adds a new user to the mmo_users table if the user does not already exist.
+
+    Parameters:
+    emp_id (str): Employee ID of the user.
+    emp_name (str): Name of the user.
+    emp_type (str): Type of the user.
+    db_cred (dict): Database credentials with keys 'dbname', 'user', 'password', 'host', 'port'.
+    schema (str): The schema name.
+
+    Returns:
+    str: A success or error message.
+    """
+    # Query to check if the user already exists
+    check_user_query = f"""
+                       SELECT emp_id FROM mmo_users
+                       WHERE emp_id = ?;
+                       """
+    params_check = (emp_id,)
+
+    try:
+        # Check if the user already exists
+        existing_user = query_excecuter_postgres(
+            check_user_query, db_cred, params=params_check, insert=False
+        )
+        # st.write(existing_user)
+
+        if existing_user:
+            # If user exists, return a warning message
+            return False
+
+        # Query to add a new user to the mmo_users table
+        else:
+            hashed_password = bcrypt.hashpw(
+                plain_text_password.encode("utf-8"), bcrypt.gensalt()
+            )
+
+            # Convert the byte string to a regular string for storage
+            hashed_password_str = hashed_password.decode("utf-8")
+            add_user_query = f"""
+                            INSERT INTO mmo_users 
+                            (emp_id, emp_nam, emp_typ, crte_dt_tm, crte_by_uid,pswrd_key)
+                            VALUES (?, ?, ?,datetime('now'), ?,?);
+                            """
+            params_add = (emp_id, emp_name, emp_type, admin_id, hashed_password_str)
+
+            # Execute the insert query
+            query_excecuter_postgres(
+                add_user_query, db_cred, params=params_add, insert=True
+            )
+
+            return True
+
+    except Exception as e:
+        return False
+
+
+# def delete_users_by_names(emp_id):
+
+#     # Sanitize and format the list of names for the SQL query
+#     formatted_ids =tuple(emp_id)
+
+#     # Query to delete users based on the employee names
+#     delete_user_query = f"""
+#                         DELETE FROM mmo_users
+#                         WHERE emp_id IN ?;
+#                         """
+#     try:
+#         # Execute the delete query
+#         query_excecuter_postgres(delete_user_query, db_cred,params=(formatted_ids,), insert=True)
+
+#         return f"{len(emp_id)} users deleted successfully."
+
+#     except Exception as e:
+#         st.write(e)
+#         print(e)
+#         return f"An error occurred: {e}"
+
+
+def delete_users_by_names(emp_ids):
+    """
+    Deletes users from the mmo_users table and their associated projects from the mmo_project_meta_data
+    and mmo_projects tables.
+
+    Parameters:
+    emp_ids (list): A list of employee IDs to delete.
+
+    Returns:
+    str: A success or error message.
+    """
+    # Convert the list of employee IDs into a tuple for the SQL query
+    formatted_ids = tuple(emp_ids)
+
+    # Queries to delete projects and users
+    delete_projects_meta_query = """
+    DELETE FROM mmo_project_meta_data
+    WHERE prj_id IN (
+        SELECT prj_id FROM mmo_projects
+        WHERE prj_ownr_id IN ({})
+    );
+    """.format(
+        ",".join("?" * len(formatted_ids))
+    )
+
+    delete_projects_query = """
+    DELETE FROM mmo_projects
+    WHERE prj_ownr_id IN ({});
+    """.format(
+        ",".join("?" * len(formatted_ids))
+    )
+
+    delete_user_query = """
+    DELETE FROM mmo_users
+    WHERE emp_id IN ({});
+    """.format(
+        ",".join("?" * len(formatted_ids))
+    )
+
+    try:
+        # Execute the delete queries using query_excecuter_postgres
+        query_excecuter_postgres(
+            delete_projects_meta_query, params=formatted_ids, insert=True
+        )
+        query_excecuter_postgres(
+            delete_projects_query, params=formatted_ids, insert=True
+        )
+        query_excecuter_postgres(delete_user_query, params=formatted_ids, insert=True)
+
+        return (
+            f"{len(emp_ids)} users and their associated projects deleted successfully."
+        )
+
+    except Exception as e:
+        return f"An error occurred: {e}"
+
+
+def reset_user_name():
+    st.session_state.user_id = ""
+    st.session_state.user_name = ""
+
+
+def contains_sql_keywords_check(user_input):
+
+    sql_keywords = [
+        "SELECT",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "DROP",
+        "ALTER",
+        "CREATE",
+        "GRANT",
+        "REVOKE",
+        "UNION",
+        "JOIN",
+        "WHERE",
+        "HAVING",
+        "EXEC",
+        "TRUNCATE",
+        "REPLACE",
+        "MERGE",
+        "DECLARE",
+        "SHOW",
+        "FROM",
+    ]
+
+    pattern = "|".join(re.escape(keyword) for keyword in sql_keywords)
+
+    return re.search(pattern, user_input, re.IGNORECASE)
+
+
+def update_password_and_flag(user_id, plain_text_password, flag=False):
+    """
+    Hashes a plain text password using bcrypt, converts it to a UTF-8 string, and stores it as text.
+
+    Parameters:
+    plain_text_password (str): The plain text password to be hashed.
+    db_cred (dict): The database credentials including dbname, user, password, host, and port.
+    """
+    # Hash the plain text password
+    hashed_password = bcrypt.hashpw(
+        plain_text_password.encode("utf-8"), bcrypt.gensalt()
+    )
+
+    # Convert the byte string to a regular string for storage
+    hashed_password_str = hashed_password.decode("utf-8")
+
+    # SQL query to update the pswrd_key for the specified user_id
+    if flag:
+        query = f"""
+        UPDATE mmo_users
+        SET pswrd_key = ?, pswrd_flag = 0
+        WHERE emp_id = ?;
+        """
+    else:
+        query = f"""
+        UPDATE mmo_users
+        SET pswrd_key = ?
+        WHERE emp_id = ?;
+        """
+    # Execute the query using the existing query_excecuter_postgres function
+    query_excecuter_postgres(
+        query=query, db_cred=db_cred, params=(hashed_password_str, user_id), insert=True
+    )
+
+
+def reset_pass_key():
+    st.session_state["pass_key"] = ""
+
+
+st.title("User Management")
+
+# hashed_key=fetch_password_hash_key()[0][0]
+
+if "df_users" not in st.session_state:
+    fetch_users_with_access()
+# st.write(hashed_key.encode())
+if "unique_ids_admin" not in st.session_state:
+    unique_users_query = f"""
+            SELECT DISTINCT emp_id, emp_nam, emp_typ from mmo_users
+            Where emp_typ= 'admin';
+            """
+    unique_users_result = query_excecuter_postgres(
+        unique_users_query, db_cred, insert=False
+    )  # retrieves all the users who has access to MMO TOOL
+    st.session_state["unique_ids_admin"] = {
+        emp_id: (emp_nam, emp_type) for emp_id, emp_nam, emp_type in unique_users_result
+    }
+
+
+if len(st.session_state["unique_ids_admin"]) == 0:
+
+    st.error("No admin found in the database!")
+    st.markdown(
+        """
+        - You can add an admin to the database.
+        - **Ensure** that you store the passkey securely, as losing it will prevent anyone from accessing the tool.
+        - Once done, reset the password on the "**Home**" page.
+    """
+    )
+
+emp_id = st.text_input("employee id", key="emp1111").lower()
+password = st.text_input("Enter password to access the page", type="password")
+
+
+if emp_id not in st.session_state["unique_ids_admin"] and len(emp_id) > 4:
+
+    if not is_pswrd_flag_set(emp_id):
+        st.warning("Reset password in Home page to continue")
+        st.stop()
+
+    st.warning(
+        'Incorrect username or password. If you are using the default password, please reset it on the "Home" page.'
+    )
+
+    st.stop()
+
+
+# st.write(st.session_state["unique_ids_admin"])
+
+
+# Check password or if no admin is present
+
+if verify_password(emp_id, password) or len(st.session_state["unique_ids_admin"]) == 0:
+
+    st.success("Access Granted", icon="✅")
+
+    st.header("All Users")
+
+    st.dataframe(
+        st.session_state["df_users"], use_container_width=True, hide_index=True
+    )
+
+    user_manage = st.radio(
+        "Select a Option",
+        ["Add New User", "Delete Users", "Manage User Passwords"],
+        horizontal=True,
+    )
+
+    if user_manage == "Add New User":
+        with st.expander("", expanded=True):
+            add_col = st.columns(3)
+
+            with add_col[0]:
+                user_id = st.text_input("Enter User ID", key="user_id").lower()
+
+            with add_col[1]:
+                user_name = st.text_input("Enter User Name", key="user_name").lower()
+
+            with add_col[2]:
+
+                if len(st.session_state["unique_ids_admin"]) == 0:
+                    user_types_options = ["Admin"]
+                else:
+                    user_types_options = ["Data Scientist", "Media Planner", "Admin"]
+
+                user_type = (
+                    st.selectbox("Select User Type", user_types_options)
+                    .replace(" ", "_")
+                    .lower()
+                )
+            warning_box = st.empty()
+
+            if "passkey" not in st.session_state:
+                st.session_state["passkey"] = ""
+            if len(user_id) < 3 or len(user_name) < 3:
+                st.session_state["passkey"] = ""
+
+            pass_key_col = st.columns(3)
+            with pass_key_col[0]:
+                st.markdown(
+                    '<div style="display: flex; position:relative; justify-content: flex-end; align-items: center; height: 100%;">'
+                    '<p style="font-weight: bold; margin: 0; position:absolute; top:4px;">Default Password</p>'
+                    "</div>",
+                    unsafe_allow_html=True,
+                )
+            with pass_key_col[1]:
+                passkey_box = st.empty()
+                with passkey_box:
+                    st.code(f"{st.session_state['passkey']}", language="text")
+
+            st.button(
+                "Reset Values", on_click=reset_user_name, use_container_width=True
+            )
+
+            if st.button("Add User", use_container_width=True):
+
+                if user_id in st.session_state["df_users"]["User ID"]:
+                    with warning_box:
+                        st.warning("User id already exists")
+                    st.stop()
+
+                if (
+                    len(user_id) == 0
+                    or len(user_name) == 0
+                    or not user_id.startswith("e")
+                    or len(user_id) < 6
+                ):
+                    with warning_box:
+                        st.warning("Enter a Valid User ID and User Name!")
+                    st.stop()
+
+                if contains_sql_keywords_check(user_id):
+                    with warning_box:
+                        st.warning(
+                            "Input contains SQL keywords. Please avoid using SQL commands."
+                        )
+                        st.stop()
+                else:
+                    pass
+
+                if not (2 <= len(user_name) <= 50):
+                    # Store the warning message details in session state
+
+                    with warning_box:
+                        st.warning(
+                            "Please provide a valid user name (2-50 characters, only A-Z, a-z, 0-9, and _)."
+                        )
+                        st.stop()
+
+                if contains_sql_keywords_check(user_name):
+                    with warning_box:
+                        st.warning(
+                            "Input contains SQL keywords. Please avoid using SQL commands."
+                        )
+                        st.stop()
+                else:
+                    pass
+
+                characters = string.ascii_letters + string.digits  # Letters and digits
+                plain_text_password = "".join(
+                    random.choice(characters) for _ in range(10)
+                )
+                st.session_state["passkey"] = plain_text_password
+
+                if add_user(user_id, user_name, user_type, emp_id, plain_text_password):
+                    with st.spinner("Adding New User"):
+
+                        update_password_and_flag(user_id, plain_text_password)
+                        fetch_users_with_access()
+                        st.success("User added successfully")
+                        st.rerun()
+                else:
+                    st.warning(
+                        f"User with emp_id {user_id} already exists in the database."
+                    )
+                    st.stop()
+
+    if user_manage == "Delete Users":
+        with st.expander("", expanded=True):
+            user_names_to_delete = st.multiselect(
+                "Select User IDS to Delete", st.session_state["df_users"]["User ID"]
+            )
+
+            if st.button("Delete", use_container_width=True):
+                delete_users_by_names(user_names_to_delete)
+                st.success("Users Deleted")
+                fetch_users_with_access()
+                st.rerun()
+
+    if user_manage == "Manage User Passwords":
+
+        with st.expander("**Manage User Passwords**", expanded=True):
+            st.markdown(
+                """
+            1.Click on "**Reset Password**" to generate a new default password for the selected user.
+            2. Share the pass key with the corresponding user
+            """,
+                unsafe_allow_html=True,
+            )
+
+            user_id = st.selectbox(
+                "Select A User ID",
+                st.session_state["df_users"]["User ID"],
+                on_change=reset_pass_key,
+            )
+
+            if "pass_key" not in st.session_state:
+                st.session_state["pass_key"] = ""
+
+            default_passkey = st.code(
+                f"Default Password: {st.session_state['pass_key']}", language="text"
+            )
+
+            if st.button("Reset Password", use_container_width=True, key="reset_pass_button_key"):
+                with st.spinner("Reseting Password"):
+                    characters = (
+                        string.ascii_letters + string.digits
+                    )  # Letters and digits
+                    plain_text_password = "".join(
+                        random.choice(characters) for _ in range(10)
+                    )
+                    st.session_state["pass_key"] = plain_text_password
+
+                    update_password_and_flag(user_id, plain_text_password, flag=True)
+                    time.sleep(1)
+                    del st.session_state["reset_pass_button_key"]
+                    st.rerun()
+
+elif len(password):
+    st.warning("Wrong user name or password!")

pages/1_Data_Import.py

@@ -0,0 +1,1213 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="Data Import",
+    page_icon="⚖️",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+
+import re
+import sys
+import pickle
+import numbers
+import traceback
+import pandas as pd
+from scenario import numerize
+from post_gres_cred import db_cred
+from collections import OrderedDict
+from log_application import log_message
+from utilities import set_header, load_local_css, update_db, project_selection
+from constants import (
+    upload_rows_limit,
+    upload_column_limit,
+    word_length_limit_lower,
+    word_length_limit_upper,
+    minimum_percent_overlap,
+    minimum_row_req,
+    percent_drop_col_threshold,
+)
+
+schema = db_cred["schema"]
+load_local_css("styles.css")
+set_header()
+
+
+# Initialize project name session state
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+# Fetch project dictionary
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+# Display Username and Project Name
+if "username" in st.session_state and st.session_state["username"] is not None:
+
+    cols1 = st.columns([2, 1])
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+
+# Initialize session state keys
+if "granularity_selection_key" not in st.session_state:
+    st.session_state["granularity_selection_key"] = st.session_state["project_dct"][
+        "data_import"
+    ]["granularity_selection"]
+
+
+# Function to format name
+def name_format_func(name):
+    return str(name).strip().title()
+
+
+# Function to get columns with specified prefix and remove prefix
+def get_columns_with_prefix(df, prefix):
+    return [
+        col.replace(prefix, "")
+        for col in df.columns
+        if col.startswith(prefix) and str(col) != str(prefix)
+    ]
+
+
+# Function to fetch columns info
+@st.cache_data(show_spinner=False)
+def fetch_columns(gold_layer_df, data_upload_df):
+    # Get lists of columns starting with 'spends_' and 'response_metric_' from gold_layer_df
+    spends_columns_gold_layer = get_columns_with_prefix(gold_layer_df, "spends_")
+    response_metric_columns_gold_layer = get_columns_with_prefix(
+        gold_layer_df, "response_metric_"
+    )
+
+    # Get lists of columns starting with 'spends_' and 'response_metric_' from data_upload_df
+    spends_columns_upload = get_columns_with_prefix(data_upload_df, "spends_")
+    response_metric_columns_upload = get_columns_with_prefix(
+        data_upload_df, "response_metric_"
+    )
+
+    # Combine lists from both DataFrames
+    spends_columns = spends_columns_gold_layer + spends_columns_upload
+    # Remove 'total' from the spends_columns list if it exists
+    spends_columns = list(
+        set([col for col in spends_columns if not col.endswith("_total")])
+    )
+
+    response_metric_columns = (
+        response_metric_columns_gold_layer + response_metric_columns_upload
+    )
+    # Filter columns ending with '_total' and remove the '_total' suffix
+    response_metric_columns = list(
+        set(
+            [
+                col[:-6]
+                for col in response_metric_columns
+                if col.endswith("_total") and len(col[:-6]) != 0
+            ]
+        )
+    )
+
+    # Get list of all columns from both DataFrames
+    gold_layer_columns = list(gold_layer_df.columns)
+    data_upload_columns = list(data_upload_df.columns)
+
+    # Combine all columns and get unique columns
+    all_columns = list(set(gold_layer_columns + data_upload_columns))
+
+    return (
+        spends_columns,
+        response_metric_columns,
+        all_columns,
+        gold_layer_columns,
+        data_upload_columns,
+    )
+
+
+# Function to format values for display
+@st.cache_data(show_spinner=False)
+def format_values_for_display(values_list):
+    # Format value
+    formatted_list = [value.lower().strip() for value in values_list]
+    # Join values with commas and 'and' before the last value
+    if len(formatted_list) > 1:
+        return ", ".join(formatted_list[:-1]) + ", and " + formatted_list[-1]
+    elif formatted_list:
+        return formatted_list[0]
+    return "No values available"
+
+
+# Function to validate input DataFrame
+@st.cache_data(show_spinner=False)
+def valid_input_df(
+    df,
+    spends_columns,
+    response_metric_columns,
+    total_columns,
+    gold_layer_columns,
+    data_upload_columns,
+):
+    # Check if DataFrame is empty
+    if df.empty or len(df) < 1:
+        return (True, None)
+
+    # Check for invalid column names
+    invalid_columns = [
+        col
+        for col in df.columns
+        if not re.match(r"^[A-Za-z0-9_]+$", col)
+        or not (word_length_limit_lower <= len(col) <= word_length_limit_upper)
+    ]
+    if invalid_columns:
+        return (
+            False,
+            f"Invalid column names: {format_values_for_display(invalid_columns)}. Use only letters, numbers, and underscores. Column name length should be {word_length_limit_lower} to {word_length_limit_upper} characters long.",
+        )
+
+    # Ensure 'panel' column values are strings and conform to specified pattern and length
+    if "panel" in df.columns:
+        df["panel"] = df["panel"].astype(str).str.strip()
+        invalid_panel_values = [
+            val
+            for val in df["panel"].unique()
+            if not re.match(r"^[A-Za-z0-9_]+$", val)
+            or not (word_length_limit_lower <= len(val) <= word_length_limit_upper)
+        ]
+        if invalid_panel_values:
+            return (
+                False,
+                f"Invalid panel values: {format_values_for_display(invalid_panel_values)}. Use only letters, numbers, and underscores. Panel name length should be {word_length_limit_lower} to {word_length_limit_upper} characters long.",
+            )
+
+    # Check for missing required columns
+    required_columns = ["date", "panel"]
+    missing_columns = [col for col in required_columns if col not in df.columns]
+    if missing_columns:
+        return (
+            False,
+            f"Missing compulsory columns: {format_values_for_display(missing_columns)}.",
+        )
+
+    # Check if all other columns are numeric
+    non_numeric_columns = [
+        col
+        for col in df.columns
+        if col not in required_columns and not pd.api.types.is_numeric_dtype(df[col])
+    ]
+    if non_numeric_columns:
+        return (
+            False,
+            f"Non-numeric columns: {format_values_for_display(non_numeric_columns)}. All columns except {format_values_for_display(required_columns)} should be numeric.",
+        )
+
+    # Ensure all columns in data_upload_columns are unique
+    duplicate_columns_in_upload = [
+        col for col in data_upload_columns if data_upload_columns.count(col) > 1
+    ]
+    if duplicate_columns_in_upload:
+        return (
+            False,
+            f"Duplicate columns found in the uploaded data: {format_values_for_display(set(duplicate_columns_in_upload))}.",
+        )
+
+    # Convert 'date' column to datetime format
+    try:
+        df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")
+    except:
+        return False, "The 'date' column is not in the correct format 'YYYY-MM-DD'."
+
+    # Check date frequency
+    unique_panels = df["panel"].unique()
+    for panel in unique_panels:
+        date_diff = df[df["panel"] == panel]["date"].diff().dropna()
+        if not (
+            (date_diff == pd.Timedelta(days=1)).all()
+            or (date_diff == pd.Timedelta(weeks=1)).all()
+        ):
+            return False, "The 'date' column does not have a daily or weekly frequency."
+
+    # Check for null values in 'date' or 'panel' columns
+    if df[required_columns].isnull().any().any():
+        return (
+            False,
+            f"The {format_values_for_display(required_columns)} should not contain null values.",
+        )
+
+    # Check for panels with less than 1% date overlap
+    if not gold_layer_df.empty:
+        panels_with_low_overlap = []
+        unique_panels = list(
+            set(df["panel"].unique()).union(set(gold_layer_df["panel"].unique()))
+        )
+        for panel in unique_panels:
+            gold_layer_dates = set(
+                gold_layer_df[gold_layer_df["panel"] == panel]["date"]
+            )
+            data_upload_dates = set(df[df["panel"] == panel]["date"])
+            if gold_layer_dates and data_upload_dates:
+                overlap = len(gold_layer_dates & data_upload_dates) / len(
+                    gold_layer_dates | data_upload_dates
+                )
+            else:
+                overlap = 0
+            if overlap < (minimum_percent_overlap / 100):
+                panels_with_low_overlap.append(panel)
+
+        if panels_with_low_overlap:
+            return (
+                False,
+                f"Date columns in the gold layer and uploaded data do not have at least {minimum_percent_overlap}% overlap for panels: {format_values_for_display(panels_with_low_overlap)}.",
+            )
+
+    # Check if spends_columns is less than two
+    if len(spends_columns) < 2:
+        return False, "Please add at least two spends columns."
+
+    # Check if response_metric_columns is empty
+    if len(response_metric_columns) < 1:
+        return False, "Please add response metric columns."
+
+    # Check if all numeric columns are positive except those starting with 'exogenous_' or 'internal_'
+    valid_prefixes = ["exogenous_", "internal_"]
+    negative_values_columns = [
+        col
+        for col in df.select_dtypes(include=[float, int]).columns
+        if not any(col.startswith(prefix) for prefix in valid_prefixes)
+        and (df[col] < 0).any()
+    ]
+    if negative_values_columns:
+        return (
+            False,
+            f"Negative values detected in columns: {format_values_for_display(negative_values_columns)}. Ensure all media and response metric columns are positive.",
+        )
+
+    # Check for unassociated columns
+    detected_channels = spends_columns + ["total"]
+    unassociated_columns = []
+    for col in df.columns:
+        if (col.startswith("_") or col.endswith("_")) or not (
+            col.startswith("exogenous_")  # Column starts with "exogenous_"
+            or col.startswith("internal_")  # Column starts with "internal_"
+            or any(
+                col == f"spends_{channel}" for channel in spends_columns
+            )  # Column is not in the format "spends_<channel>"
+            or any(
+                col == f"response_metric_{metric}_{channel}"
+                for metric in response_metric_columns
+                for channel in detected_channels
+            )  # Column is not in the format "response_metric_<metric>_<channel>"
+            or any(
+                col.startswith("media_")
+                and col.endswith(f"_{channel}")
+                and len(col) > len(f"media__{channel}")
+                for channel in spends_columns
+            )  # Column is not in the format "media_<media_variable_name>_<channel>"
+            or col in ["date", "panel"]
+        ):
+            unassociated_columns.append(col)
+
+    if unassociated_columns:
+        return (
+            False,
+            f"Columns with incorrect format detected: {format_values_for_display(unassociated_columns)}.",
+        )
+
+    return True, "The data is valid and meets all requirements."
+
+
+# Function to load the uploaded Excel file into a DataFrame
+@st.cache_data(show_spinner=False)
+def load_and_transform_data(uploaded_file):
+    # Load the uploaded file into a DataFrame if a file is uploaded
+    if uploaded_file is not None:
+        df = pd.read_excel(uploaded_file)
+    else:
+        df = pd.DataFrame()
+        return df
+
+    # Check if DataFrame exceeds row and column limits
+    if len(df) > upload_rows_limit or len(df.columns) > upload_column_limit:
+        st.warning(
+            f"Data exceeds the row limit of {numerize(upload_rows_limit)} or column limit of {numerize(upload_column_limit)}. Please upload a smaller file.",
+            icon="⚠️",
+        )
+
+        # Log message
+        log_message(
+            "warning",
+            f"Data exceeds the row limit of {numerize(upload_rows_limit)} or column limit of {numerize(upload_column_limit)}. Please upload a smaller file.",
+            "Data Import",
+        )
+
+        return pd.DataFrame()
+
+    # If the DataFrame contains only 'panel' and 'date' columns, return empty DataFrame
+    if set(df.columns) == {"date", "panel"}:
+        return pd.DataFrame()
+
+    # Transform column names: lower, strip start and end, replace spaces with _
+    df.columns = [str(col).strip().lower().replace(" ", "_") for col in df.columns]
+
+    # If 'panel' column exists, clean its values
+    try:
+        if "panel" in df.columns:
+            df["panel"] = (
+                df["panel"].astype(str).str.lower().str.strip().str.replace(" ", "_")
+            )
+    except:
+        return df
+
+    try:
+        df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")
+    except:
+        # The 'date' column is not in the correct format 'YYYY-MM-DD'
+        return df
+
+    # Check date frequency and convert to daily if needed
+    date_diff = df["date"].diff().dropna()
+    if (date_diff == pd.Timedelta(days=1)).all():
+        # Data is already at daily level
+        return df
+    elif (date_diff == pd.Timedelta(weeks=1)).all():
+        # Data is at weekly level, convert to daily
+        weekly_data = df.copy()
+        daily_data = []
+
+        for index, row in weekly_data.iterrows():
+            week_start = row["date"] - pd.to_timedelta(row["date"].weekday(), unit="D")
+            for i in range(7):
+                daily_date = week_start + pd.DateOffset(days=i)
+                new_row = row.copy()
+                new_row["date"] = daily_date
+                for col in df.columns:
+                    if isinstance(new_row[col], numbers.Number):
+                        new_row[col] = new_row[col] / 7
+                daily_data.append(new_row)
+
+        daily_data_df = pd.DataFrame(daily_data)
+        return daily_data_df
+    else:
+        # The 'date' column does not have a daily or weekly frequency
+        return df
+
+
+# Function to merge DataFrames if present
+@st.cache_data(show_spinner=False)
+def merge_dataframes(gold_layer_df, data_upload_df):
+    if gold_layer_df.empty and data_upload_df.empty:
+        return pd.DataFrame()
+
+    if not gold_layer_df.empty and not data_upload_df.empty:
+        # Merge gold_layer_df and data_upload_df on 'panel', and 'date'
+        merged_df = pd.merge(
+            gold_layer_df,
+            data_upload_df,
+            on=["panel", "date"],
+            how="outer",
+            suffixes=("_gold", "_upload"),
+        )
+
+        # Handle duplicate columns
+        for col in merged_df.columns:
+            if col.endswith("_gold"):
+                base_col = col[:-5]  # Remove '_gold' suffix
+                upload_col = base_col + "_upload"  # Column name in data_upload_df
+                if upload_col in merged_df.columns:
+                    # Prefer values from data_upload_df
+                    merged_df[base_col] = merged_df[upload_col].combine_first(
+                        merged_df[col]
+                    )
+                    merged_df.drop(columns=[col, upload_col], inplace=True)
+                else:
+                    # Rename column to remove the suffix
+                    merged_df.rename(columns={col: base_col}, inplace=True)
+
+    elif data_upload_df.empty:
+        merged_df = gold_layer_df.copy()
+
+    elif gold_layer_df.empty:
+        merged_df = data_upload_df.copy()
+
+    return merged_df
+
+
+# Function to check if all required columns are present in the Uploaded DataFrame
+@st.cache_data(show_spinner=False)
+def check_required_columns(df, detected_channels, detected_response_metric):
+    required_columns = []
+
+    # Add all channels with 'spends_' + detected channel name
+    for channel in detected_channels:
+        required_columns.append(f"spends_{channel}")
+
+    # Add all channels with 'response_metric_' + detected channel name
+    for response_metric in detected_response_metric:
+        for channel in detected_channels + ["total"]:
+            required_columns.append(f"response_metric_{response_metric}_{channel}")
+
+    # Check for missing columns
+    missing_columns = [col for col in required_columns if col not in df.columns]
+
+    # Channel groupings
+    no_media_data = []
+    channel_columns_dict = {}
+    for channel in detected_channels:
+        channel_columns = [
+            col
+            for col in merged_df.columns
+            if channel in col
+            and not (
+                col.startswith("response_metric_")
+                or col.startswith("exogenous_")
+                or col.startswith("internal_")
+            )
+            and col.endswith(channel)
+        ]
+        channel_columns_dict[channel] = channel_columns
+
+        if len(channel_columns) <= 1:
+            no_media_data.append(channel)
+
+    return missing_columns, no_media_data, channel_columns_dict
+
+
+# Function to prepare tool DataFrame
+def prepare_tool_df(merged_df, granularity_selection):
+    # Drop all response metric columns that do not end with '_total'
+    cols_to_drop = [
+        col
+        for col in merged_df.columns
+        if col.startswith("response_metric_") and not col.endswith("_total")
+    ]
+
+    # Create a DataFrame to be used for the tool
+    tool_df = merged_df.drop(columns=cols_to_drop)
+
+    # Convert to weekly granularity by aggregating all data for given panel and week
+    if granularity_selection.lower() == "weekly":
+        tool_df.set_index("date", inplace=True)
+        tool_df = (
+            tool_df.groupby(
+                [pd.Grouper(freq="W-MON", closed="left", label="left"), "panel"]
+            )
+            .sum()
+            .reset_index()
+        )
+
+    return tool_df
+
+
+# Function to generate imputation DataFrame
+def generate_imputation_df(tool_df):
+    # Initialize lists to store the column details
+    column_names = []
+    categories = []
+    missing_values_info = []
+    zero_values_info = []
+    imputation_methods = []
+
+    # Define the function to calculate the percentage of missing values
+    def calculate_missing_percentage(series):
+        return series.isnull().sum(), (series.isnull().mean() * 100)
+
+    # Define the function to calculate the percentage of zero values
+    def calculate_zero_percentage(series):
+        return (series == 0).sum(), ((series == 0).mean() * 100)
+
+    # Iterate over each column to categorize and calculate missing and zero values
+    for col in tool_df.columns:
+        # Determine category based on column name prefix
+        if col == "date" or col == "panel":
+            continue
+        elif col.startswith("response_metric_"):
+            categories.append("Response Metrics")
+        elif col.startswith("spends_"):
+            categories.append("Spends")
+        elif col.startswith("exogenous_"):
+            categories.append("Exogenous")
+        elif col.startswith("internal_"):
+            categories.append("Internal")
+        else:
+            categories.append("Media")
+
+        # Calculate missing values and percentage
+        missing_count, missing_percentage = calculate_missing_percentage(tool_df[col])
+        missing_values_info.append(f"{missing_count} ({missing_percentage:.1f}%)")
+
+        # Calculate zero values and percentage
+        zero_count, zero_percentage = calculate_zero_percentage(tool_df[col])
+        zero_values_info.append(f"{zero_count} ({zero_percentage:.1f}%)")
+
+        # Determine default imputation method based on conditions
+        if col.startswith("spends_"):
+            imputation_methods.append("Fill with 0")
+        elif col.startswith("response_metric_"):
+            imputation_methods.append("Fill with Mean")
+        elif zero_percentage + missing_percentage > percent_drop_col_threshold:
+            imputation_methods.append("Drop Column")
+        else:
+            imputation_methods.append("Fill with Mean")
+
+        column_names.append(col)
+
+    # Create the DataFrame
+    imputation_df = pd.DataFrame(
+        {
+            "Column Name": column_names,
+            "Category": categories,
+            "Missing Values": missing_values_info,
+            "Zero Values": zero_values_info,
+            "Imputation Method": imputation_methods,
+        }
+    )
+
+    # Define the category order for sorting
+    category_order = {
+        "Response Metrics": 1,
+        "Spends": 2,
+        "Media": 3,
+        "Exogenous": 4,
+        "Internal": 5,
+    }
+
+    # Add a temporary column for sorting based on the category order
+    imputation_df["Category Order"] = imputation_df["Category"].map(category_order)
+
+    # Sort the DataFrame based on the category order and then drop the temporary column
+    imputation_df = imputation_df.sort_values(
+        by=["Category Order", "Column Name"]
+    ).drop(columns=["Category Order"])
+
+    return imputation_df
+
+
+# Function to perform imputation as per user requests
+def perform_imputation(imputation_df, tool_df):
+    # Detect channels associated with spends
+    detected_channels = [
+        col.replace("spends_", "")
+        for col in tool_df.columns
+        if col.startswith("spends_")
+    ]
+
+    # Create a dictionary with keys as channels and values as associated columns
+    group_dict = {
+        channel: [
+            col
+            for col in tool_df.columns
+            if channel in col
+            and not (
+                col.startswith("response_metric_")
+                or col.startswith("exogenous_")
+                or col.startswith("internal_")
+            )
+        ]
+        for channel in detected_channels
+    }
+
+    # Create a reverse dictionary with keys as columns and values as channels
+    column_to_channel_dict = {
+        col: channel for channel, cols in group_dict.items() for col in cols
+    }
+
+    # Perform imputation
+    already_dropped = []
+    for index, row in imputation_df.iterrows():
+        col_name = row["Column Name"]
+        impute_method = row["Imputation Method"]
+
+        # Skip already dropped columns
+        if col_name in already_dropped:
+            continue
+
+        # Skip imputation if dropping response metric column and add warning
+        if impute_method == "Drop Column" and col_name.startswith("response_metric_"):
+            return None, {}, f"Cannot drop response metric column: {col_name}"
+
+        # Drop column if requested
+        if impute_method == "Drop Column":
+            # If spends column is dropped, drop all related columns
+            if col_name.startswith("spends_"):
+                tool_df.drop(
+                    columns=group_dict[col_name.replace("spends_", "")],
+                    inplace=True,
+                )
+                already_dropped += group_dict[col_name.replace("spends_", "")]
+                del group_dict[col_name.replace("spends_", "")]
+            else:
+                tool_df.drop(columns=[col_name], inplace=True)
+                if not (
+                    col_name.startswith("exogenous_")
+                    or col_name.startswith("internal_")
+                ):
+                    group_name = column_to_channel_dict[col_name]
+                    group_dict[group_name].remove(col_name)
+
+                    # Check for channels with one or fewer associated columns and add warning if needed
+                    if len(group_dict[group_name]) <= 1:
+                        return (
+                            None,
+                            {},
+                            f"No media variable associated with category {col_name.replace('spends_', '')}.",
+                        )
+            continue
+
+        # Check for each panel
+        for panel in tool_df["panel"].unique():
+            panel_df = tool_df[tool_df["panel"] == panel]
+
+            # Check if the column is entirely null or empty for the current panel
+            if panel_df[col_name].isnull().all():
+                if impute_method in ["Fill with Mean", "Fill with Median"]:
+                    return (
+                        None,
+                        {},
+                        f"Cannot impute for empty column(s) with mean or median. Select 'Fill with 0'. Details: Panel: {panel}, Column: {col_name}",
+                    )
+
+        # Fill missing values as requested
+        if impute_method == "Fill with Mean":
+            tool_df[col_name] = tool_df.groupby("panel")[col_name].transform(
+                lambda x: x.fillna(x.mean())
+            )
+        elif impute_method == "Fill with Median":
+            tool_df[col_name] = tool_df.groupby("panel")[col_name].transform(
+                lambda x: x.fillna(x.median())
+            )
+        elif impute_method == "Fill with 0":
+            tool_df[col_name].fillna(0, inplace=True)
+
+    # Check if final DataFrame has at least one response metric and two spends categories
+    response_metrics = [
+        col for col in tool_df.columns if col.startswith("response_metric_")
+    ]
+    spends_categories = [col for col in tool_df.columns if col.startswith("spends_")]
+
+    if len(response_metrics) < 1:
+        return (None, {}, "The final DataFrame must have at least one response metric.")
+    if len(spends_categories) < 2:
+        return (
+            None,
+            {},
+            "The final DataFrame must have at least two spends categories.",
+        )
+
+    return tool_df, group_dict, "Imputed Successfully!"
+
+
+# Function to display groups with custom styling
+def display_groups(input_dict):
+    # Define custom CSS for pastel light blue rounded rectangle
+    custom_css = """
+    <style>
+    .group-box {
+        background-color: #ffdaab;
+        border-radius: 10px;
+        padding: 10px;
+        margin: 5px 0;
+    }
+    </style>
+    """
+    st.markdown(custom_css, unsafe_allow_html=True)
+
+    for group_name, values in input_dict.items():
+        group_html = f"<div class='group-box'><strong>{group_name}:</strong> {format_values_for_display(values)}</div>"
+        st.markdown(group_html, unsafe_allow_html=True)
+
+
+# Function to categorize columns and create an ordered dictionary
+def create_ordered_category_dict(df):
+    category_dict = {
+        "Response Metrics": [],
+        "Spends": [],
+        "Media": [],
+        "Exogenous": [],
+        "Internal": [],
+    }
+
+    # Define the category order for sorting
+    category_order = {
+        "Response Metrics": 1,
+        "Spends": 2,
+        "Media": 3,
+        "Exogenous": 4,
+        "Internal": 5,
+    }
+
+    for column in df.columns:
+        if column == "date" or column == "panel":
+            continue  # Skip 'date' and 'panel' columns
+
+        if column.startswith("response_metric_"):
+            category_dict["Response Metrics"].append(column)
+        elif column.startswith("spends_"):
+            category_dict["Spends"].append(column)
+        elif column.startswith("exogenous_"):
+            category_dict["Exogenous"].append(column)
+        elif column.startswith("internal_"):
+            category_dict["Internal"].append(column)
+        else:
+            category_dict["Media"].append(column)
+
+    # Sort the dictionary based on the defined category order
+    sorted_category_dict = OrderedDict(
+        sorted(category_dict.items(), key=lambda item: category_order[item[0]])
+    )
+
+    return sorted_category_dict
+
+
+try:
+    # Page Title
+    st.title("Data Import")
+
+    # Create file uploader
+    uploaded_file = st.file_uploader(
+        "Upload Data", type=["xlsx"], accept_multiple_files=False
+    )
+
+    # Expander with markdown for upload rules
+    with st.expander("Upload Rules and Guidelines"):
+        st.markdown(
+            """
+        ### Upload Guidelines
+        
+        Please ensure your data adheres to the following rules:
+        
+        1. **File Format**: 
+        - Upload all data in a single Excel file.
+        
+        2. **Compulsory Columns**: 
+        - **Date**: Must be in the format `YYYY-MM-DD` only.
+        - **Panel**: If no panel data exists, use `aggregated` as a single panel.
+        
+        3. **Column Naming Conventions**: 
+        - All columns should start with the associated category prefix.
+        
+        **Examples**:
+
+        - **Response Metric Column**: 
+        - Format: `response_metric_<response_metric_name>_<channel_name>`
+        - Example: `response_metric_revenue_facebook`
+        
+        - **Total Response Metric**: 
+        - Format: `response_metric_<response_metric_name>_total`
+        - Example: `response_metric_revenue_total`
+
+        - **Spend Column**: 
+        - Format: `spends_<channel_name>`
+        - Example: `spends_facebook`
+
+        - **Media Column**: 
+        - Format: `media_<media_variable_name>_<channel_name>`
+        - Example: `media_clicks_facebook`
+        
+        - **Exogenous Column**: 
+        - Format: `exogenous_<variable_name>`
+        - Example: `exogenous_unemployment_rate`
+        
+        - **Internal Column**: 
+        - Format: `internal_<variable_name>`
+        - Example: `internal_discount`
+        
+        **Notes**:
+        
+        - The `total` response metric should represent the total for a particular date and panel, including all channels and organic contributions.
+        - The `date` column for weekly data should be the Monday of that week, representing the data from that Monday to the following Sunday. Example: If the week starts on Monday, August 5th, 2024, and ends on Sunday, August 11th, 2024, the date column for that week should display 2024-08-05.
+        """
+        )
+
+    # Upload warning placeholder
+    upload_warning_placeholder = st.container()
+
+    # Load the uploaded file into a DataFrame if a file is uploaded
+    data_upload_df = load_and_transform_data(uploaded_file)
+
+    # Columns for user input
+    granularity_col, validate_process_col = st.columns(2)
+
+    # Dropdown for data granularity
+    granularity_selection = granularity_col.selectbox(
+        "Select data granularity",
+        options=["daily", "weekly"],
+        format_func=name_format_func,
+        key="granularity_selection_key",
+    )
+
+    # Gold Layer DataFrame
+    gold_layer_df = st.session_state["project_dct"]["data_import"]["gold_layer_df"]
+    if not gold_layer_df.empty:
+        st.subheader("Gold Layer DataFrame")
+        with st.expander("Gold Layer DataFrame"):
+            st.dataframe(
+                gold_layer_df,
+                hide_index=True,
+                column_config={
+                    "date": st.column_config.DateColumn("date", format="YYYY-MM-DD")
+                },
+            )
+    else:
+        st.info(
+            "No gold layer data is selected for this project. Please upload data manually.",
+            icon="📊",
+        )
+
+    # Check input data
+    with validate_process_col:
+        st.write("##")  # Padding
+
+    if validate_process_col.button("Validate and Process", use_container_width=True):
+        with st.spinner("Processing ..."):
+            # Check if both DataFrames are empty
+            valid_input = True
+            if gold_layer_df.empty and data_upload_df.empty:
+                # If both gold_layer_df and data_upload_df are empty, display a warning and stop the script
+                st.warning(
+                    "Both the Gold Layer data and the uploaded data are empty. Please provide at least one data source.",
+                    icon="⚠️",
+                )
+
+                # Log message
+                log_message(
+                    "warning",
+                    "Both the Gold Layer data and the uploaded data are empty. Please provide at least one data source.",
+                    "Data Import",
+                )
+                valid_input = False
+
+            # If the uploaded DataFrame is empty and the Gold Layer is not, swap them to ensure all validation conditions are checked
+            elif not gold_layer_df.empty and data_upload_df.empty:
+                data_upload_df, gold_layer_df = (
+                    gold_layer_df.copy(),
+                    data_upload_df.copy(),
+                )
+                valid_input = True
+
+            if valid_input:
+                # Fetch all necessary columns list
+                (
+                    spends_columns,
+                    response_metric_columns,
+                    total_columns,
+                    gold_layer_columns,
+                    data_upload_columns,
+                ) = fetch_columns(gold_layer_df, data_upload_df)
+
+                with upload_warning_placeholder:
+                    valid_input, message = valid_input_df(
+                        data_upload_df,
+                        spends_columns,
+                        response_metric_columns,
+                        total_columns,
+                        gold_layer_columns,
+                        data_upload_columns,
+                    )
+                    if not valid_input:
+                        st.warning(message, icon="⚠️")
+
+                        # Log message
+                        log_message("warning", message, "Data Import")
+
+            # Merge gold_layer_df and data_upload_df on 'panel' and 'date'
+            if valid_input:
+                merged_df = merge_dataframes(gold_layer_df, data_upload_df)
+
+                missing_columns, no_media_data, channel_columns_dict = (
+                    check_required_columns(
+                        merged_df, spends_columns, response_metric_columns
+                    )
+                )
+
+                with upload_warning_placeholder:
+                    # Warning for categories with no media data
+                    if no_media_data:
+                        st.warning(
+                            f"Categories without media data: {format_values_for_display(no_media_data)}. Please upload at least one media column to proceed.",
+                            icon="⚠️",
+                        )
+                        valid_input = False
+
+                        # Log message
+                        log_message(
+                            "warning",
+                            f"Categories without media data: {format_values_for_display(no_media_data)}. Please upload at least one media column to proceed.",
+                            "Data Import",
+                        )
+
+                    # Warning for insufficient rows
+                    elif any(
+                        granularity_selection == "daily"
+                        and len(merged_df[merged_df["panel"] == panel])
+                        < minimum_row_req
+                        for panel in merged_df["panel"].unique()
+                    ):
+                        st.warning(
+                            f"Insufficient data. Please provide at least {minimum_row_req} days of data for all panel.",
+                            icon="⚠️",
+                        )
+                        valid_input = False
+
+                        # Log message
+                        log_message(
+                            "warning",
+                            f"Insufficient data. Please provide at least {minimum_row_req} days of data for all panel.",
+                            "Data Import",
+                        )
+
+                    elif any(
+                        granularity_selection == "weekly"
+                        and len(merged_df[merged_df["panel"] == panel])
+                        < minimum_row_req * 7
+                        for panel in merged_df["panel"].unique()
+                    ):
+                        st.warning(
+                            f"Insufficient data. Please provide at least {minimum_row_req} weeks of data for all panel.",
+                            icon="⚠️",
+                        )
+                        valid_input = False
+
+                        # Log message
+                        log_message(
+                            "warning",
+                            f"Insufficient data. Please provide at least {minimum_row_req} weeks of data for all panel.",
+                            "Data Import",
+                        )
+
+                    # Info for missing columns
+                    elif missing_columns:
+                        st.info(
+                            f"Missing columns: {format_values_for_display(missing_columns)}. Please upload all required columns.",
+                            icon="💡",
+                        )
+
+            if valid_input:
+                # Create a copy of the merged DataFrame for dashboard purposes
+                dashboard_df = merged_df
+
+                # Create a DataFrame for tool purposes
+                tool_df = prepare_tool_df(merged_df, granularity_selection)
+
+                # Create Imputation DataFrame
+                imputation_df = generate_imputation_df(tool_df)
+
+                # Save data to project dictionary
+                st.session_state["project_dct"]["data_import"][
+                    "granularity_selection"
+                ] = st.session_state["granularity_selection_key"]
+                st.session_state["project_dct"]["data_import"][
+                    "dashboard_df"
+                ] = dashboard_df
+                st.session_state["project_dct"]["data_import"]["tool_df"] = tool_df
+                st.session_state["project_dct"]["data_import"]["unique_panels"] = (
+                    tool_df["panel"].unique()
+                )
+                st.session_state["project_dct"]["data_import"][
+                    "imputation_df"
+                ] = imputation_df
+
+                # Success message
+                with upload_warning_placeholder:
+                    st.success("Processed Successfully!", icon="🗂️")
+                    st.toast("Processed Successfully!", icon="🗂️")
+
+                # Log message
+                log_message("info", "Processed Successfully!", "Data Import")
+
+    # Load saved data from project dictionary
+    if st.session_state["project_dct"]["data_import"]["tool_df"] is None:
+        st.stop()
+    else:
+        tool_df = st.session_state["project_dct"]["data_import"]["tool_df"]
+        imputation_df = st.session_state["project_dct"]["data_import"]["imputation_df"]
+        unique_panels = st.session_state["project_dct"]["data_import"]["unique_panels"]
+
+    # Unique Panel
+    st.subheader("Unique Panel")
+
+    # Get unique panels count
+    total_count = len(unique_panels)
+
+    # Define custom CSS for pastel light blue rounded rectangle
+    custom_css = """
+    <style>
+    .panel-box {
+        background-color: #ffdaab;
+        border-radius: 10px;
+        padding: 10px;
+        margin: 0 0;
+    }
+    </style>
+    """
+
+    # Display unique panels with total count
+    st.markdown(custom_css, unsafe_allow_html=True)
+    panel_html = f"<div class='panel-box'><strong>Unique Panels:</strong> {format_values_for_display(unique_panels)}<br><strong>Total Count:</strong> {total_count}</div>"
+    st.markdown(panel_html, unsafe_allow_html=True)
+    st.write("##")  # Padding
+
+    # Impute Missing Values
+    st.subheader("Impute Missing Values")
+    edited_imputation_df = st.data_editor(
+        imputation_df,
+        column_config={
+            "Imputation Method": st.column_config.SelectboxColumn(
+                options=[
+                    "Drop Column",
+                    "Fill with Mean",
+                    "Fill with Median",
+                    "Fill with 0",
+                ],
+                required=True,
+                default="Fill with 0",
+            ),
+        },
+        column_order=[
+            "Column Name",
+            "Category",
+            "Missing Values",
+            "Zero Values",
+            "Imputation Method",
+        ],
+        disabled=["Column Name", "Category", "Missing Values", "Zero Values"],
+        hide_index=True,
+        use_container_width=True,
+        key="imputation_df_key",
+    )
+
+    # Expander with markdown for imputation rules
+    with st.expander("Impute Missing Values Guidelines"):
+        st.markdown(
+            f"""
+        ### Imputation Guidelines
+        
+        Please adhere to the following rules when handling missing values:
+
+        1. **Default Imputation Strategies**:
+        - **Response Metrics**: Imputed using the **mean** value of the column.
+        - **Spends**: Imputed with **zero** values.
+        - **Media, Exogenous, Internal**: Imputation strategy is **dynamic** based on the data.
+
+        2. **Drop Threshold**:
+        - If the combined percentage of **zeros** and **null values** in any column exceeds `{percent_drop_col_threshold}%`, the column will be **categorized to drop** by default which user can change manually.
+        - **Example**: If `spends_facebook` has more than `{percent_drop_col_threshold}%` of zeros and nulls combined, it will be marked for dropping.
+
+        3. **Category Generation and Association**:
+        - Categories are automatically generated from the **Spends** columns. 
+        - **Example**: The column `spends_facebook` will generate the **facebook** category. This means columns like `spends_facebook`, `media_impression_facebook` and `media_clicks_facebook` will also be associated with this category.
+
+        4. **Column Association and Imputation**:
+        - Each category must have at least **one Media column** associated with it for imputation to proceed.
+        - **Example**: If the **facebook** category does not have any media columns like `media_impression_facebook`, imputation will not be allowed for that category.
+        - Solution: Either **drop the entire category** if it is empty, or **impute the columns** associated with the category instead of dropping them.
+
+        5. **Response Metrics and Category Count**:
+        - Dropping **Response Metric** columns is **not allowed** under any circumstances.
+        - At least **two categories** must exist after imputation, or the Imputation will not proceed.
+        - **Example**: If only **facebook** remains after selection, imputation will be halted.
+
+        **Notes**:
+
+        - The decision to drop a spends column will result in all associated columns being dropped. 
+        - **Example**: Dropping `spends_facebook` will also drop all related columns like `media_impression_facebook` and `media_clicks_facebook`.
+        """
+        )
+
+    # Imputation Warning Placeholder
+    imputation_warning_placeholder = st.container()
+
+    # Save the DataFrame and dictionary from the current session
+    if st.button("Impute and Save", use_container_width=True):
+        with st.spinner("Imputing ..."):
+            with imputation_warning_placeholder:
+                # Perform Imputation
+                imputed_tool_df, group_dict, message = perform_imputation(
+                    edited_imputation_df.copy(), tool_df.copy()
+                )
+
+                if imputed_tool_df is None:
+                    st.warning(message, icon="⚠️")
+
+                    # Log message
+                    log_message("warning", message, "Data Import")
+
+                else:
+                    st.session_state["project_dct"]["data_import"][
+                        "imputed_tool_df"
+                    ] = imputed_tool_df
+                    st.session_state["project_dct"]["data_import"][
+                        "imputation_df"
+                    ] = edited_imputation_df
+                    st.session_state["project_dct"]["data_import"][
+                        "group_dict"
+                    ] = group_dict
+                    st.session_state["project_dct"]["data_import"]["category_dict"] = (
+                        create_ordered_category_dict(imputed_tool_df)
+                    )
+
+            if imputed_tool_df is not None:
+                # Update DB
+                update_db(
+                    prj_id=st.session_state["project_number"],
+                    page_nam="Data Import",
+                    file_nam="project_dct",
+                    pkl_obj=pickle.dumps(st.session_state["project_dct"]),
+                    schema=schema,
+                )
+
+                # Success message
+                st.success("Saved Successfully!", icon="💾")
+                st.toast("Saved Successfully!", icon="💾")
+
+                # Log message
+                log_message("info", "Saved Successfully!", "Data Import")
+
+    # Load saved data from project dictionary
+    if st.session_state["project_dct"]["data_import"]["imputed_tool_df"] is None:
+        st.stop()
+    else:
+        imputed_tool_df = st.session_state["project_dct"]["data_import"][
+            "imputed_tool_df"
+        ]
+        group_dict = st.session_state["project_dct"]["data_import"]["group_dict"]
+        category_dict = st.session_state["project_dct"]["data_import"]["category_dict"]
+
+    # Channel Groupings
+    st.subheader("Channel Groupings")
+    display_groups(group_dict)
+    st.write("##")  # Padding
+
+    # Variable Categorization
+    st.subheader("Variable Categorization")
+    display_groups(category_dict)
+    st.write("##")  # Padding
+
+    # Final DataFrame
+    st.subheader("Final DataFrame")
+    st.dataframe(
+        imputed_tool_df,
+        hide_index=True,
+        column_config={
+            "date": st.column_config.DateColumn("date", format="YYYY-MM-DD")
+        },
+    )
+    st.write("##")  # Padding
+
+except Exception as e:
+    # Capture the error details
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+
+    # Log message
+    log_message("error", f"An error occurred: {error_message}.", "Data Import")
+
+    # Display a warning message
+    st.warning(
+        "Oops! Something went wrong. Please try refreshing the tool or creating a new project.",
+        icon="⚠️",
+    )

pages/2_Data_Assessment.py

@@ -0,0 +1,467 @@
+import streamlit as st
+import pandas as pd
+from data_analysis import *
+import numpy as np
+import pickle
+import streamlit as st
+from utilities import set_header, load_local_css, update_db, project_selection
+from post_gres_cred import db_cred
+from utilities import update_db
+import re
+
+st.set_page_config(
+    page_title="Data Assessment​",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+schema = db_cred["schema"]
+load_local_css("styles.css")
+set_header()
+
+if "username" not in st.session_state:
+    st.session_state["username"] = None
+
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+if "username" in st.session_state and st.session_state["username"] is not None:
+
+    if st.session_state["project_dct"]["data_import"]["imputed_tool_df"] is None:
+
+        st.error(f"Please import data from the Data Import Page")
+        st.stop()
+
+    st.session_state["cleaned_data"] = st.session_state["project_dct"]["data_import"][
+        "imputed_tool_df"
+    ]
+
+    st.session_state["category_dict"] = st.session_state["project_dct"]["data_import"][
+        "category_dict"
+    ]
+
+    # st.write(st.session_state['category_dict'])
+    cols1 = st.columns([2, 1])
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+    st.title("Data Assessment​")
+
+    target_variables = [
+        st.session_state["category_dict"][key]
+        for key in st.session_state["category_dict"].keys()
+        if key == "Response Metrics"
+    ]
+
+    def format_display(inp):
+        return (
+            inp.title()
+            .replace("_", " ")
+            .replace("Media", "")
+            .replace("Cnt", "")
+            .strip()
+        )
+
+    target_variables = list(*target_variables)
+    target_column = st.selectbox(
+        "Select the Target Feature/Dependent Variable (will be used in all charts as reference)",
+        target_variables,
+        index=st.session_state["project_dct"]["data_validation"]["target_column"],
+        format_func=format_display,
+    )
+
+    st.session_state["project_dct"]["data_validation"]["target_column"] = (
+        target_variables.index(target_column)
+    )
+
+    st.session_state["target_column"] = target_column
+
+
+    if "panel" not in st.session_state["cleaned_data"].columns:
+        st.write('True')
+        st.session_state["cleaned_data"]["panel"] = ["Aggregated"] * len(
+            st.session_state["cleaned_data"]
+        )
+        
+        disable = True
+
+    else:
+        panels = st.session_state["cleaned_data"]["panel"]
+
+        disable = False
+
+    selected_panels = st.multiselect(
+        "Please choose the panels you wish to analyze.If no panels are selected, insights will be derived from the overall data.",
+        st.session_state["cleaned_data"]["panel"].unique(),
+        default=st.session_state["project_dct"]["data_validation"]["selected_panels"],
+        disabled=disable,
+    )
+
+    st.session_state["project_dct"]["data_validation"][
+        "selected_panels"
+    ] = selected_panels
+
+    aggregation_dict = {
+        item: "sum" if key == "Media" else "mean"
+        for key, value in st.session_state["category_dict"].items()
+        for item in value
+        if item not in ["date", "panel"]
+    }
+
+    aggregation_dict = {
+        key: value
+        for key, value in aggregation_dict.items()
+        if key in st.session_state["cleaned_data"].columns
+    }
+
+    with st.expander("**Target Variable  Analysis**"):
+
+        if len(selected_panels) > 0:
+            st.session_state["Cleaned_data_panel"] = st.session_state["cleaned_data"][
+                st.session_state["cleaned_data"]["panel"].isin(selected_panels)
+            ]
+
+            st.session_state["Cleaned_data_panel"] = (
+                st.session_state["Cleaned_data_panel"]
+                .groupby(by="date")
+                .agg(aggregation_dict)
+            )
+            st.session_state["Cleaned_data_panel"] = st.session_state[
+                "Cleaned_data_panel"
+            ].reset_index()
+        else:
+            # st.write(st.session_state['cleaned_data'])
+            st.session_state["Cleaned_data_panel"] = (
+                st.session_state["cleaned_data"]
+                .groupby(by="date")
+                .agg(aggregation_dict)
+            )
+            st.session_state["Cleaned_data_panel"] = st.session_state[
+                "Cleaned_data_panel"
+            ].reset_index()
+
+        fig = line_plot_target(
+            st.session_state["Cleaned_data_panel"],
+            target=target_column,
+            title=f"{target_column} Over Time",
+        )
+        st.plotly_chart(fig, use_container_width=True)
+
+        media_channel = list(
+            *[
+                st.session_state["category_dict"][key]
+                for key in st.session_state["category_dict"].keys()
+                if key == "Media"
+            ]
+        )
+
+        spends_features = list(
+            *[
+                st.session_state["category_dict"][key]
+                for key in st.session_state["category_dict"].keys()
+                if key == "Spends"
+            ]
+        )
+        # st.write(media_channel)
+
+        exo_var = list(
+            *[
+                st.session_state["category_dict"][key]
+                for key in st.session_state["category_dict"].keys()
+                if key == "Exogenous"
+            ]
+        )
+        internal_var = list(
+            *[
+                st.session_state["category_dict"][key]
+                for key in st.session_state["category_dict"].keys()
+                if key == "Internal"
+            ]
+        )
+
+        Non_media_variables = exo_var + internal_var
+
+        st.markdown("### Annual Data Summary")
+
+        summary_df = summary(
+            st.session_state["Cleaned_data_panel"],
+            media_channel + [target_column] + spends_features,
+            spends=None,
+            Target=True,
+        )
+
+        st.dataframe(
+            summary_df.sort_index(axis=1),
+            use_container_width=True,
+        )
+
+        if st.checkbox("View Raw Data"):
+            st.cache_resource(show_spinner=False)
+
+            def raw_df_gen():
+                # Convert 'date' to datetime but do not convert to string yet for sorting
+                dates = pd.to_datetime(st.session_state["Cleaned_data_panel"]["date"])
+
+                # Concatenate the dates with other numeric columns formatted
+                raw_df = pd.concat(
+                    [
+                        dates,
+                        st.session_state["Cleaned_data_panel"]
+                        .select_dtypes(np.number)
+                        .applymap(format_numbers),
+                    ],
+                    axis=1,
+                )
+
+                # Now sort raw_df by the 'date' column, which is still in datetime format
+                sorted_raw_df = raw_df.sort_values(by="date", ascending=True)
+
+                # After sorting, convert 'date' to string format for display
+                sorted_raw_df["date"] = sorted_raw_df["date"].dt.strftime("%m/%d/%Y")
+
+                return sorted_raw_df
+
+            # Display the sorted DataFrame in Streamlit
+            st.dataframe(raw_df_gen())
+
+    col1 = st.columns(1)
+
+    if "selected_feature" not in st.session_state:
+        st.session_state["selected_feature"] = None
+
+    # st.warning('Work in Progress')
+    with st.expander("Media Variables Analysis"):
+        # Get the selected feature
+
+        st.session_state["selected_feature"] = st.selectbox(
+            "Select Media", media_channel + spends_features, format_func=format_display
+        )
+
+        # st.write(st.session_state["selected_feature"].split('cnt_')[1] )
+        # st.session_state["project_dct"]["data_validation"]["selected_feature"] = (
+
+        # )
+
+        # Filter spends features based on the selected feature
+        spends_col = st.columns(2)
+        spends_feature = [
+            col
+            for col in spends_features
+            if re.split(r"cost_|spends_", col.lower())[1]
+            in st.session_state["selected_feature"]
+        ]
+
+        with spends_col[0]:
+            if len(spends_feature) == 0:
+                st.warning(
+                    "The selected metric does not include a 'spends' variable in the data. Please verify that the columns are correctly named or select the appropriate columns in the provided selection box."
+                )
+            else:
+                st.write(
+                    f'Selected "{spends_feature[0]}" as the corresponding spends variable automatically. If this is incorrect, please click the checkbox to change the variable.'
+                )
+
+        with spends_col[1]:
+            if len(spends_feature) == 0 or st.checkbox(
+                'Select "Spends" variable for CPM and CPC calculation'
+            ):
+                spends_feature = [st.selectbox("Spends Variable", spends_features)]
+
+        if "validation" not in st.session_state:
+
+            st.session_state["validation"] = st.session_state["project_dct"][
+                "data_validation"
+            ]["validated_variables"]
+
+        val_variables = [col for col in media_channel if col != "date"]
+
+        if not set(
+            st.session_state["project_dct"]["data_validation"]["validated_variables"]
+        ).issubset(set(val_variables)):
+
+            st.session_state["validation"] = []
+
+        else:
+            fig_row1 = line_plot(
+                st.session_state["Cleaned_data_panel"],
+                x_col="date",
+                y1_cols=[st.session_state["selected_feature"]],
+                y2_cols=[target_column],
+                title=f'Analysis of {st.session_state["selected_feature"]} and {[target_column][0]} Over Time',
+            )
+            st.plotly_chart(fig_row1, use_container_width=True)
+            st.markdown("### Summary")
+            st.dataframe(
+                summary(
+                    st.session_state["Cleaned_data_panel"],
+                    [st.session_state["selected_feature"]],
+                    spends=spends_feature[0],
+                ),
+                use_container_width=True,
+            )
+
+            cols2 = st.columns(2)
+
+            if len(
+                set(st.session_state["validation"]).intersection(val_variables)
+            ) == len(val_variables):
+                disable = True
+                help = "All media variables are validated"
+            else:
+                disable = False
+                help = ""
+
+            with cols2[0]:
+                if st.button("Validate", disabled=disable, help=help):
+                    st.session_state["validation"].append(
+                        st.session_state["selected_feature"]
+                    )
+            with cols2[1]:
+
+                if st.checkbox("Validate All", disabled=disable, help=help):
+                    st.session_state["validation"].extend(val_variables)
+                    st.success("All media variables are validated ✅")
+
+            if len(
+                set(st.session_state["validation"]).intersection(val_variables)
+            ) != len(val_variables):
+                validation_data = pd.DataFrame(
+                    {
+                        "Validate": [
+                            (True if col in st.session_state["validation"] else False)
+                            for col in val_variables
+                        ],
+                        "Variables": val_variables,
+                    }
+                )
+
+                sorted_validation_df = validation_data.sort_values(
+                    by="Variables", ascending=True, na_position="first"
+                )
+                cols3 = st.columns([1, 30])
+                with cols3[1]:
+                    validation_df = st.data_editor(
+                        sorted_validation_df,
+                        # column_config={
+                        # 'Validate':st.column_config.CheckboxColumn(wi)
+                        # },
+                        column_config={
+                            "Validate": st.column_config.CheckboxColumn(
+                                default=False,
+                                width=100,
+                            ),
+                            "Variables": st.column_config.TextColumn(width=1000),
+                        },
+                        hide_index=True,
+                    )
+
+                    selected_rows = validation_df[validation_df["Validate"] == True][
+                        "Variables"
+                    ]
+
+                    # st.write(selected_rows)
+
+                    st.session_state["validation"].extend(selected_rows)
+
+                    st.session_state["project_dct"]["data_validation"][
+                        "validated_variables"
+                    ] = st.session_state["validation"]
+
+                    not_validated_variables = [
+                        col
+                        for col in val_variables
+                        if col not in st.session_state["validation"]
+                    ]
+
+                    if not_validated_variables:
+                        not_validated_message = f'The following variables are not validated:\n{" , ".join(not_validated_variables)}'
+                        st.warning(not_validated_message)
+
+    with st.expander("Non-Media Variables Analysis"):
+        if len(Non_media_variables) == 0:
+            st.warning("Non-Media variables not present")
+
+        else:
+            selected_columns_row4 = st.selectbox(
+                "Select Channel",
+                Non_media_variables,
+                format_func=format_display,
+                index=st.session_state["project_dct"]["data_validation"][
+                    "Non_media_variables"
+                ],
+            )
+
+            st.session_state["project_dct"]["data_validation"][
+                "Non_media_variables"
+            ] = Non_media_variables.index(selected_columns_row4)
+
+            #     # Create the dual-axis line plot
+            fig_row4 = line_plot(
+                st.session_state["Cleaned_data_panel"],
+                x_col="date",
+                y1_cols=[selected_columns_row4],
+                y2_cols=[target_column],
+                title=f"Analysis of {selected_columns_row4} and {target_column} Over Time",
+            )
+            st.plotly_chart(fig_row4, use_container_width=True)
+            selected_non_media = selected_columns_row4
+            sum_df = st.session_state["Cleaned_data_panel"][
+                ["date", selected_non_media, target_column]
+            ]
+            sum_df["Year"] = pd.to_datetime(
+                st.session_state["Cleaned_data_panel"]["date"]
+            ).dt.year
+            # st.dataframe(df)
+            # st.dataframe(sum_df.head(2))
+            
+            sum_df = sum_df.drop("date", axis=1).groupby("Year").agg("sum")
+            sum_df.loc["Grand Total"] = sum_df.sum()
+            sum_df = sum_df.applymap(format_numbers)
+            sum_df.fillna("-", inplace=True)
+            sum_df = sum_df.replace({"0.0": "-", "nan": "-"})
+            st.markdown("### Summary")
+            st.dataframe(sum_df, use_container_width=True)
+
+    with st.expander("Correlation Analysis"):
+        options = list(
+            st.session_state["Cleaned_data_panel"].select_dtypes(np.number).columns
+        )
+         
+
+        if "correlation" not in st.session_state["project_dct"]["data_import"]:
+            st.session_state["project_dct"]["data_import"]["correlation"]=[]
+            
+        selected_options = st.multiselect(
+            "Select Variables for Correlation Plot",
+            [var for var in options if var != target_column],
+            default=st.session_state["project_dct"]["data_import"]["correlation"],
+        )
+
+        st.session_state["project_dct"]["data_import"]["correlation"] = selected_options
+
+        st.pyplot(
+            correlation_plot(
+                st.session_state["Cleaned_data_panel"],
+                selected_options,
+                target_column,
+            )
+        )
+
+    if st.button("Save Changes", use_container_width=True):
+        # Update DB
+        update_db(
+            prj_id=st.session_state["project_number"],
+            page_nam="Data Validation and Insights",
+            file_nam="project_dct",
+            pkl_obj=pickle.dumps(st.session_state["project_dct"]),
+            schema=schema,
+        )
+        st.success("Changes saved")

pages/3_AI_Model_Transformations.py

@@ -0,0 +1,1326 @@
+# Importing necessary libraries
+import streamlit as st
+
+st.set_page_config(
+    page_title="AI Model Transformations",
+    page_icon="⚖️",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+import sys
+import pickle
+import traceback
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+from post_gres_cred import db_cred
+from log_application import log_message
+from utilities import (
+    set_header,
+    load_local_css,
+    update_db,
+    project_selection,
+    delete_entries,
+    retrieve_pkl_object,
+)
+from constants import (
+    predefined_defaults,
+    lead_min_value,
+    lead_max_value,
+    lead_step,
+    lag_min_value,
+    lag_max_value,
+    lag_step,
+    moving_average_min_value,
+    moving_average_max_value,
+    moving_average_step,
+    saturation_min_value,
+    saturation_max_value,
+    saturation_step,
+    power_min_value,
+    power_max_value,
+    power_step,
+    adstock_min_value,
+    adstock_max_value,
+    adstock_step,
+    display_max_col,
+)
+
+
+schema = db_cred["schema"]
+load_local_css("styles.css")
+set_header()
+
+
+# Initialize project name session state
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+# Fetch project dictionary
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+# Display Username and Project Name
+if "username" in st.session_state and st.session_state["username"] is not None:
+
+    cols1 = st.columns([2, 1])
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+# Load saved data from project dictionary
+if st.session_state["project_dct"]["data_import"]["imputed_tool_df"] is None:
+    st.warning(
+        "The data import is incomplete. Please go back to the Data Import page and complete the save.",
+        icon="🔙",
+    )
+
+    # Log message
+    log_message(
+        "warning",
+        "The data import is incomplete. Please go back to the Data Import page and complete the save.",
+        "Transformations",
+    )
+
+    st.stop()
+else:
+    final_df_loaded = st.session_state["project_dct"]["data_import"][
+        "imputed_tool_df"
+    ].copy()
+    bin_dict_loaded = st.session_state["project_dct"]["data_import"][
+        "category_dict"
+    ].copy()
+    unique_panels = st.session_state["project_dct"]["data_import"][
+        "unique_panels"
+    ].copy()
+
+# Initialize project dictionary data
+if st.session_state["project_dct"]["transformations"]["final_df"] is None:
+    st.session_state["project_dct"]["transformations"][
+        "final_df"
+    ] = final_df_loaded  # Default as original dataframe
+
+# Extract original columns for specified categories
+original_columns = {
+    category: bin_dict_loaded[category]
+    for category in ["Media", "Internal", "Exogenous"]
+    if category in bin_dict_loaded
+}
+
+# Retrive Panel columns
+panel = ["panel"] if len(unique_panels) > 1 else []
+
+
+# Function to clear model metadata
+def clear_pages():
+    # Reset Pages
+    st.session_state["project_dct"]["model_build"] = {
+        "sel_target_col": None,
+        "all_iters_check": False,
+        "iterations": 0,
+        "build_button": False,
+        "show_results_check": False,
+        "session_state_saved": {},
+    }
+    st.session_state["project_dct"]["model_tuning"] = {
+        "sel_target_col": None,
+        "sel_model": {},
+        "flag_expander": False,
+        "start_date_default": None,
+        "end_date_default": None,
+        "repeat_default": "No",
+        "flags": {},
+        "select_all_flags_check": {},
+        "selected_flags": {},
+        "trend_check": False,
+        "week_num_check": False,
+        "sine_cosine_check": False,
+        "session_state_saved": {},
+    }
+    st.session_state["project_dct"]["saved_model_results"] = {
+        "selected_options": None,
+        "model_grid_sel": [1],
+    }
+    if "model_results_df" in st.session_state:
+        del st.session_state["model_results_df"]
+    if "model_results_data" in st.session_state:
+        del st.session_state["model_results_data"]
+    if "coefficients_df" in st.session_state:
+        del st.session_state["coefficients_df"]
+
+
+# Function to update transformation change
+def transformation_change(category, transformation, key):
+    st.session_state["project_dct"]["transformations"][category][transformation] = (
+        st.session_state[key]
+    )
+
+
+# Function to update specific transformation change
+def transformation_specific_change(channel_name, transformation, key):
+    st.session_state["project_dct"]["transformations"]["Specific"][transformation][
+        channel_name
+    ] = st.session_state[key]
+
+
+# Function to update transformations to apply change
+def transformations_to_apply_change(category, key):
+    st.session_state["project_dct"]["transformations"][category][key] = (
+        st.session_state[key]
+    )
+
+
+# Function to update channel select specific change
+def channel_select_specific_change():
+    st.session_state["project_dct"]["transformations"]["Specific"][
+        "channel_select_specific"
+    ] = st.session_state["channel_select_specific"]
+
+
+# Function to update specific transformation change
+def specific_transformation_change(specific_transformation_key):
+    st.session_state["project_dct"]["transformations"]["Specific"][
+        specific_transformation_key
+    ] = st.session_state[specific_transformation_key]
+
+
+# Function to build transformation widgets
+def transformation_widgets(category, transform_params, date_granularity):
+    # Transformation Options
+    transformation_options = {
+        "Media": [
+            "Lag",
+            "Moving Average",
+            "Saturation",
+            "Power",
+            "Adstock",
+        ],
+        "Internal": ["Lead", "Lag", "Moving Average"],
+        "Exogenous": ["Lead", "Lag", "Moving Average"],
+    }
+
+    # Define a helper function to create widgets for each transformation
+    def create_transformation_widgets(column, transformations):
+        with column:
+            for transformation in transformations:
+                transformation_key = f"{transformation}_{category}"
+
+                slider_value = st.session_state["project_dct"]["transformations"][
+                    category
+                ].get(transformation, predefined_defaults[transformation])
+
+                # Conditionally create widgets for selected transformations
+                if transformation == "Lead":
+                    st.markdown(f"**{transformation} ({date_granularity})**")
+
+                    lead = st.slider(
+                        label="Lead periods",
+                        min_value=lead_min_value,
+                        max_value=lead_max_value,
+                        step=lead_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = lead[0]
+                    end = lead[1]
+                    step = lead_step
+                    transform_params[category][transformation] = np.arange(
+                        start, end + step, step
+                    )
+
+                if transformation == "Lag":
+                    st.markdown(f"**{transformation} ({date_granularity})**")
+
+                    lag = st.slider(
+                        label="Lag periods",
+                        min_value=lag_min_value,
+                        max_value=lag_max_value,
+                        step=lag_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = lag[0]
+                    end = lag[1]
+                    step = lag_step
+                    transform_params[category][transformation] = np.arange(
+                        start, end + step, step
+                    )
+
+                if transformation == "Moving Average":
+                    st.markdown(f"**{transformation} ({date_granularity})**")
+
+                    window = st.slider(
+                        label="Window size for Moving Average",
+                        min_value=moving_average_min_value,
+                        max_value=moving_average_max_value,
+                        step=moving_average_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = window[0]
+                    end = window[1]
+                    step = moving_average_step
+                    transform_params[category][transformation] = np.arange(
+                        start, end + step, step
+                    )
+
+                if transformation == "Saturation":
+                    st.markdown(f"**{transformation} (%)**")
+
+                    saturation_point = st.slider(
+                        label="Saturation Percentage",
+                        min_value=saturation_min_value,
+                        max_value=saturation_max_value,
+                        step=saturation_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = saturation_point[0]
+                    end = saturation_point[1]
+                    step = saturation_step
+                    transform_params[category][transformation] = np.arange(
+                        start, end + step, step
+                    )
+
+                if transformation == "Power":
+                    st.markdown(f"**{transformation}**")
+
+                    power = st.slider(
+                        label="Power",
+                        min_value=power_min_value,
+                        max_value=power_max_value,
+                        step=power_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = power[0]
+                    end = power[1]
+                    step = power_step
+                    transform_params[category][transformation] = np.arange(
+                        start, end + step, step
+                    )
+
+                if transformation == "Adstock":
+                    st.markdown(f"**{transformation}**")
+
+                    rate = st.slider(
+                        label="Decay Factor",
+                        min_value=adstock_min_value,
+                        max_value=adstock_max_value,
+                        step=adstock_step,
+                        value=slider_value,
+                        key=transformation_key,
+                        label_visibility="collapsed",
+                        on_change=transformation_change,
+                        args=(
+                            category,
+                            transformation,
+                            transformation_key,
+                        ),
+                    )
+
+                    start = rate[0]
+                    end = rate[1]
+                    step = adstock_step
+                    adstock_range = [
+                        round(a, 3) for a in np.arange(start, end + step, step)
+                    ]
+                    transform_params[category][transformation] = np.array(adstock_range)
+
+    with st.expander(f"All {category} Transformations", expanded=True):
+
+        transformation_key = f"transformation_{category}"
+
+        # Select which transformations to apply
+        sel_transformations = st.session_state["project_dct"]["transformations"][
+            category
+        ].get(transformation_key, [])
+
+        # Reset default selected channels list if options are changed
+        for channel in sel_transformations:
+            if channel not in transformation_options[category]:
+                (
+                    st.session_state["project_dct"]["transformations"][category][
+                        transformation_key
+                    ],
+                    sel_transformations,
+                ) = ([], [])
+
+        transformations_to_apply = st.multiselect(
+            label="Select transformations to apply",
+            options=transformation_options[category],
+            default=sel_transformations,
+            key=transformation_key,
+            on_change=transformations_to_apply_change,
+            args=(
+                category,
+                transformation_key,
+            ),
+        )
+
+        # Determine the number of transformations to put in each column
+        transformations_per_column = (
+            len(transformations_to_apply) // 2 + len(transformations_to_apply) % 2
+        )
+
+        # Create two columns
+        col1, col2 = st.columns(2)
+
+        # Assign transformations to each column
+        transformations_col1 = transformations_to_apply[:transformations_per_column]
+        transformations_col2 = transformations_to_apply[transformations_per_column:]
+
+        # Create widgets in each column
+        create_transformation_widgets(col1, transformations_col1)
+        create_transformation_widgets(col2, transformations_col2)
+
+
+# Define a helper function to create widgets for each specific transformation
+def create_specific_transformation_widgets(
+    column,
+    transformations,
+    channel_name,
+    date_granularity,
+    specific_transform_params,
+):
+    with column:
+        for transformation in transformations:
+            transformation_key = f"{transformation}_{channel_name}_specific"
+
+            if (
+                transformation
+                not in st.session_state["project_dct"]["transformations"]["Specific"]
+            ):
+                st.session_state["project_dct"]["transformations"]["Specific"][
+                    transformation
+                ] = {}
+
+            slider_value = st.session_state["project_dct"]["transformations"][
+                "Specific"
+            ][transformation].get(channel_name, predefined_defaults[transformation])
+
+            # Conditionally create widgets for selected transformations
+            if transformation == "Lead":
+                st.markdown(f"**Lead ({date_granularity})**")
+
+                lead = st.slider(
+                    label="Lead periods",
+                    min_value=lead_min_value,
+                    max_value=lead_max_value,
+                    step=lead_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = lead[0]
+                end = lead[1]
+                step = lead_step
+                specific_transform_params[channel_name]["Lead"] = np.arange(
+                    start, end + step, step
+                )
+
+            if transformation == "Lag":
+                st.markdown(f"**Lag ({date_granularity})**")
+
+                lag = st.slider(
+                    label="Lag periods",
+                    min_value=lag_min_value,
+                    max_value=lag_max_value,
+                    step=lag_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = lag[0]
+                end = lag[1]
+                step = lag_step
+                specific_transform_params[channel_name]["Lag"] = np.arange(
+                    start, end + step, step
+                )
+
+            if transformation == "Moving Average":
+                st.markdown(f"**Moving Average ({date_granularity})**")
+
+                window = st.slider(
+                    label="Window size for Moving Average",
+                    min_value=moving_average_min_value,
+                    max_value=moving_average_max_value,
+                    step=moving_average_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = window[0]
+                end = window[1]
+                step = moving_average_step
+                specific_transform_params[channel_name]["Moving Average"] = np.arange(
+                    start, end + step, step
+                )
+
+            if transformation == "Saturation":
+                st.markdown("**Saturation (%)**")
+
+                saturation_point = st.slider(
+                    label="Saturation Percentage",
+                    min_value=saturation_min_value,
+                    max_value=saturation_max_value,
+                    step=saturation_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = saturation_point[0]
+                end = saturation_point[1]
+                step = saturation_step
+                specific_transform_params[channel_name]["Saturation"] = np.arange(
+                    start, end + step, step
+                )
+
+            if transformation == "Power":
+                st.markdown("**Power**")
+
+                power = st.slider(
+                    label="Power",
+                    min_value=power_min_value,
+                    max_value=power_max_value,
+                    step=power_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = power[0]
+                end = power[1]
+                step = power_step
+                specific_transform_params[channel_name]["Power"] = np.arange(
+                    start, end + step, step
+                )
+
+            if transformation == "Adstock":
+                st.markdown("**Adstock**")
+                rate = st.slider(
+                    label="Decay Factor",
+                    min_value=adstock_min_value,
+                    max_value=adstock_max_value,
+                    step=adstock_step,
+                    value=slider_value,
+                    key=transformation_key,
+                    label_visibility="collapsed",
+                    on_change=transformation_specific_change,
+                    args=(
+                        channel_name,
+                        transformation,
+                        transformation_key,
+                    ),
+                )
+
+                start = rate[0]
+                end = rate[1]
+                step = adstock_step
+                adstock_range = [
+                    round(a, 3) for a in np.arange(start, end + step, step)
+                ]
+                specific_transform_params[channel_name]["Adstock"] = np.array(
+                    adstock_range
+                )
+
+
+# Function to apply Lag transformation
+def apply_lag(df, lag):
+    return df.shift(lag)
+
+
+# Function to apply Lead transformation
+def apply_lead(df, lead):
+    return df.shift(-lead)
+
+
+# Function to apply Moving Average transformation
+def apply_moving_average(df, window_size):
+    return df.rolling(window=window_size).mean()
+
+
+# Function to apply Saturation transformation
+def apply_saturation(df, saturation_percent_100):
+    # Convert percentage to fraction
+    saturation_percent = min(max(saturation_percent_100, 0.01), 99.99) / 100.0
+
+    # Get the maximum and minimum values
+    column_max = df.max()
+    column_min = df.min()
+    
+    # If the data is constant, scale it directly
+    if column_min == column_max:
+        return df.apply(lambda x: x * saturation_percent)
+    
+    # Compute the saturation point from the data range
+    saturation_point = (column_min + saturation_percent * column_max) / 2
+
+    # Calculate steepness for the saturation curve
+    numerator = np.log((1 / saturation_percent) - 1)
+    denominator = np.log(saturation_point / column_max)
+    steepness = numerator / denominator
+
+    # Apply the saturation transformation
+    transformed_series = df.apply(
+        lambda x: (1 / (1 + (saturation_point / (x if x != 0 else 1e-9)) ** steepness)) * x
+    )
+
+    return transformed_series
+
+
+# Function to apply Power transformation
+def apply_power(df, power):
+    return df**power
+
+
+# Function to apply Adstock transformation
+def apply_adstock(df, factor):
+    x = 0
+    # Use the walrus operator to update x iteratively with the Adstock formula
+    adstock_var = [x := x * factor + v for v in df]
+    ans = pd.Series(adstock_var, index=df.index)
+    return ans
+
+
+# Function to generate transformed columns names
+@st.cache_resource(show_spinner=False)
+def generate_transformed_columns(
+    original_columns, transform_params, specific_transform_params
+):
+    transformed_columns, summary = {}, {}
+
+    for category, columns in original_columns.items():
+        for column in columns:
+            transformed_columns[column] = []
+            summary_details = (
+                []
+            )  # List to hold transformation details for the current column
+
+            if (
+                column in specific_transform_params.keys()
+                and len(specific_transform_params[column]) > 0
+            ):
+                for transformation, values in specific_transform_params[column].items():
+                    # Generate transformed column names for each value
+                    for value in values:
+                        transformed_name = f"{column}@{transformation}_{value}"
+                        transformed_columns[column].append(transformed_name)
+
+                    # Format the values list as a string with commas and "and" before the last item
+                    if len(values) > 1:
+                        formatted_values = (
+                            ", ".join(map(str, values[:-1])) + " and " + str(values[-1])
+                        )
+                    else:
+                        formatted_values = str(values[0])
+
+                    # Add transformation details
+                    summary_details.append(f"{transformation} ({formatted_values})")
+
+            else:
+                if category in transform_params:
+                    for transformation, values in transform_params[category].items():
+                        # Generate transformed column names for each value
+                        if column not in specific_transform_params.keys():
+                            for value in values:
+                                transformed_name = f"{column}@{transformation}_{value}"
+                                transformed_columns[column].append(transformed_name)
+
+                            # Format the values list as a string with commas and "and" before the last item
+                            if len(values) > 1:
+                                formatted_values = (
+                                    ", ".join(map(str, values[:-1]))
+                                    + " and "
+                                    + str(values[-1])
+                                )
+                            else:
+                                formatted_values = str(values[0])
+
+                            # Add transformation details
+                            summary_details.append(
+                                f"{transformation} ({formatted_values})"
+                            )
+
+                        else:
+                            summary_details = ["No transformation selected"]
+
+            # Only add to summary if there are transformation details for the column
+            if summary_details:
+                formatted_summary = "⮕ ".join(summary_details)
+                # Use <strong> tags to make the column name bold
+                summary[column] = f"<strong>{column}</strong>: {formatted_summary}"
+
+    # Generate a comprehensive summary string for all columns
+    summary_items = [
+        f"{idx + 1}. {details}" for idx, details in enumerate(summary.values())
+    ]
+
+    summary_string = "\n".join(summary_items)
+
+    return transformed_columns, summary_string
+
+
+# Function to transform Dataframe slice
+def transform_slice(
+    transform_params,
+    transformation_functions,
+    panel,
+    df,
+    df_slice,
+    category,
+    category_df,
+):
+    # Iterate through each transformation and its parameters for the current category
+    for transformation, parameters in transform_params[category].items():
+        transformation_function = transformation_functions[transformation]
+
+        # Check if there is panel data to group by
+        if len(panel) > 0:
+            # Apply the transformation to each group
+            category_df = pd.concat(
+                [
+                    df_slice.groupby(panel)
+                    .transform(transformation_function, p)
+                    .add_suffix(f"@{transformation}_{p}")
+                    for p in parameters
+                ],
+                axis=1,
+            )
+
+            # Replace all NaN or null values in category_df with 0
+            category_df.fillna(0, inplace=True)
+
+            # Update df_slice
+            df_slice = pd.concat(
+                [df[panel], category_df],
+                axis=1,
+            )
+
+        else:
+            for p in parameters:
+                # Apply the transformation function to each column
+                temp_df = df_slice.apply(
+                    lambda x: transformation_function(x, p), axis=0
+                ).rename(
+                    lambda x: f"{x}@{transformation}_{p}",
+                    axis="columns",
+                )
+                # Concatenate the transformed DataFrame slice to the category DataFrame
+                category_df = pd.concat([category_df, temp_df], axis=1)
+
+            # Replace all NaN or null values in category_df with 0
+            category_df.fillna(0, inplace=True)
+
+            # Update df_slice
+            df_slice = pd.concat(
+                [df[panel], category_df],
+                axis=1,
+            )
+
+    return category_df, df, df_slice
+
+
+# Function to apply transformations to DataFrame slices based on specified categories and parameters
+@st.cache_resource(show_spinner=False)
+def apply_category_transformations(
+    df_main, bin_dict, transform_params, panel, specific_transform_params
+):
+    # Dictionary for function mapping
+    transformation_functions = {
+        "Lead": apply_lead,
+        "Lag": apply_lag,
+        "Moving Average": apply_moving_average,
+        "Saturation": apply_saturation,
+        "Power": apply_power,
+        "Adstock": apply_adstock,
+    }
+
+    # List to collect all transformed DataFrames
+    transformed_dfs = []
+
+    # Iterate through each category specified in transform_params
+    for category in ["Media", "Exogenous", "Internal"]:
+        if (
+            category not in transform_params
+            or category not in bin_dict
+            or not transform_params[category]
+        ):
+            continue  # Skip categories without transformations
+
+        # Initialize category_df as an empty DataFrame
+        category_df = pd.DataFrame()
+
+        # Slice the DataFrame based on the columns specified in bin_dict for the current category
+        df_slice = df_main[bin_dict[category] + panel].copy()
+
+        # Drop the column from df_slice to skip specific transformations
+        df_slice = df_slice.drop(
+            columns=list(specific_transform_params.keys()), errors="ignore"
+        ).copy()
+
+        category_df, df, df_slice_updated = transform_slice(
+            transform_params.copy(),
+            transformation_functions.copy(),
+            panel,
+            df_main.copy(),
+            df_slice.copy(),
+            category,
+            category_df.copy(),
+        )
+
+        # Append the transformed category DataFrame to the list if it's not empty
+        if not category_df.empty:
+            transformed_dfs.append(category_df)
+
+    # Apply channel specific transforms
+    for channel_specific in specific_transform_params:
+        # Initialize category_df as an empty DataFrame
+        category_df = pd.DataFrame()
+
+        df_slice_specific = df_main[[channel_specific] + panel].copy()
+        transform_params_specific = {
+            "Media": specific_transform_params[channel_specific]
+        }
+
+        category_df, df, df_slice_specific_updated = transform_slice(
+            transform_params_specific.copy(),
+            transformation_functions.copy(),
+            panel,
+            df_main.copy(),
+            df_slice_specific.copy(),
+            "Media",
+            category_df.copy(),
+        )
+
+        # Append the transformed category DataFrame to the list if it's not empty
+        if not category_df.empty:
+            transformed_dfs.append(category_df)
+
+    # If category_df has been modified, concatenate it with the panel and response metrics from the original DataFrame
+    if len(transformed_dfs) > 0:
+        final_df = pd.concat([df_main] + transformed_dfs, axis=1)
+    else:
+        # If no transformations were applied, use the original DataFrame
+        final_df = df_main
+
+    # Find columns with '@' in their names
+    columns_with_at = [col for col in final_df.columns if "@" in col]
+
+    # Create a set of columns to drop
+    columns_to_drop = set()
+
+    # Iterate through columns with '@' to find shorter names to drop
+    for col in columns_with_at:
+        base_name = col.split("@")[0]
+        for other_col in columns_with_at:
+            if other_col.startswith(base_name) and len(other_col.split("@")) > len(
+                col.split("@")
+            ):
+                columns_to_drop.add(col)
+                break
+
+    # Drop the identified columns from the DataFrame
+    final_df.drop(columns=list(columns_to_drop), inplace=True)
+
+    return final_df
+
+
+# Function to infers the granularity of the date column in a DataFrame
+@st.cache_resource(show_spinner=False)
+def infer_date_granularity(df):
+    # Find the most common difference
+    common_freq = pd.Series(df["date"].unique()).diff().dt.days.dropna().mode()[0]
+
+    # Map the most common difference to a granularity
+    if common_freq == 1:
+        return "daily"
+    elif common_freq == 7:
+        return "weekly"
+    elif 28 <= common_freq <= 31:
+        return "monthly"
+    else:
+        return "irregular"
+
+
+# Function to clean display DataFrame
+@st.cache_data(show_spinner=False)
+def clean_display_df(df, display_max_col=500):
+    # Sort by 'panel' and 'date'
+    sort_columns = ["panel", "date"]
+    sorted_df = df.sort_values(by=sort_columns, ascending=True, na_position="first")
+
+    # Drop duplicate columns
+    sorted_df = sorted_df.loc[:, ~sorted_df.columns.duplicated()]
+
+    # Check if the DataFrame has more than display_max_col columns
+    exceeds_max_col = sorted_df.shape[1] > display_max_col
+
+    if exceeds_max_col:
+        # Create a new DataFrame with 'date' and 'panel' at the start
+        display_df = sorted_df[["date", "panel"]]
+
+        # Add the next display_max_col - 2 columns (as 'date' and 'panel' already occupy 2 columns)
+        additional_columns = sorted_df.columns.difference(["date", "panel"]).tolist()[
+            : display_max_col - 2
+        ]
+        display_df = pd.concat([display_df, sorted_df[additional_columns]], axis=1)
+    else:
+        # Ensure 'date' and 'panel' are the first two columns in the final display DataFrame
+        column_order = ["date", "panel"] + sorted_df.columns.difference(
+            ["date", "panel"]
+        ).tolist()
+        display_df = sorted_df[column_order]
+
+    # Return the display DataFrame and whether it exceeds 500 columns
+    return display_df, exceeds_max_col
+
+
+#########################################################################################################################################################
+# User input for transformations
+#########################################################################################################################################################
+
+try:
+    # Page Title
+    st.title("AI Model Transformations")
+
+    # Infer date granularity
+    date_granularity = infer_date_granularity(final_df_loaded)
+
+    # Initialize the main dictionary to store the transformation parameters for each category
+    transform_params = {"Media": {}, "Internal": {}, "Exogenous": {}}
+
+    st.markdown("### Select Transformations to Apply")
+
+    with st.expander("Specific Media Transformations"):
+        # Select which transformations to apply
+        sel_channel_specific = st.session_state["project_dct"]["transformations"][
+            "Specific"
+        ].get("channel_select_specific", [])
+
+        # Reset default selected channels list if options are changed
+        for channel in sel_channel_specific:
+            if channel not in bin_dict_loaded["Media"]:
+                (
+                    st.session_state["project_dct"]["transformations"]["Specific"][
+                        "channel_select_specific"
+                    ],
+                    sel_channel_specific,
+                ) = ([], [])
+
+        select_specific_channels = st.multiselect(
+            label="Select channel variable",
+            default=sel_channel_specific,
+            options=bin_dict_loaded["Media"],
+            key="channel_select_specific",
+            on_change=channel_select_specific_change,
+            max_selections=30,
+        )
+
+        specific_transform_params = {}
+        for select_specific_channel in select_specific_channels:
+            specific_transform_params[select_specific_channel] = {}
+
+            st.divider()
+            channel_name = str(select_specific_channel).replace("_", " ").title()
+            st.markdown(f"###### {channel_name}")
+
+            specific_transformation_key = (
+                f"specific_transformation_{select_specific_channel}_Media"
+            )
+
+            transformations_options = [
+                "Lag",
+                "Moving Average",
+                "Saturation",
+                "Power",
+                "Adstock",
+            ]
+
+            # Select which transformations to apply
+            sel_transformations = st.session_state["project_dct"]["transformations"][
+                "Specific"
+            ].get(specific_transformation_key, [])
+
+            # Reset default selected channels list if options are changed
+            for channel in sel_transformations:
+                if channel not in transformations_options:
+                    (
+                        st.session_state["project_dct"]["transformations"]["Specific"][
+                            specific_transformation_key
+                        ],
+                        sel_channel_specific,
+                    ) = ([], [])
+
+            transformations_to_apply = st.multiselect(
+                label="Select transformations to apply",
+                options=transformations_options,
+                default=sel_transformations,
+                key=specific_transformation_key,
+                on_change=specific_transformation_change,
+                args=(specific_transformation_key,),
+            )
+
+            # Determine the number of transformations to put in each column
+            transformations_per_column = (
+                len(transformations_to_apply) // 2 + len(transformations_to_apply) % 2
+            )
+
+            # Create two columns
+            col1, col2 = st.columns(2)
+
+            # Assign transformations to each column
+            transformations_col1 = transformations_to_apply[:transformations_per_column]
+            transformations_col2 = transformations_to_apply[transformations_per_column:]
+
+            # Create widgets in each column
+            create_specific_transformation_widgets(
+                col1,
+                transformations_col1,
+                select_specific_channel,
+                date_granularity,
+                specific_transform_params,
+            )
+            create_specific_transformation_widgets(
+                col2,
+                transformations_col2,
+                select_specific_channel,
+                date_granularity,
+                specific_transform_params,
+            )
+
+    # Create Widgets
+    for category in ["Media", "Internal", "Exogenous"]:
+        # Skip Internal
+        if category == "Internal":
+            continue
+
+        # Skip category if no column available
+        elif (
+            category not in bin_dict_loaded.keys()
+            or len(bin_dict_loaded[category]) == 0
+        ):
+            st.info(
+                f"{str(category).title()} category has no column associated with it. Skipping transformation step for this category.",
+                icon="💬",
+            )
+            continue
+
+        transformation_widgets(category, transform_params, date_granularity)
+
+    #########################################################################################################################################################
+    # Apply transformations
+    #########################################################################################################################################################
+
+    # Reset transformation selection to default
+    button_col = st.columns(2)
+    with button_col[1]:
+        if st.button("Reset to Default", use_container_width=True):
+            st.session_state["project_dct"]["transformations"]["Media"] = {}
+            st.session_state["project_dct"]["transformations"]["Exogenous"] = {}
+            st.session_state["project_dct"]["transformations"]["Internal"] = {}
+            st.session_state["project_dct"]["transformations"]["Specific"] = {}
+
+            # Log message
+            log_message(
+                "info",
+                "All persistent selections have been reset to their default settings and cleared.",
+                "Transformations",
+            )
+
+            st.rerun()
+
+    # Apply category-based transformations to the DataFrame
+    with button_col[0]:
+        if st.button("Accept and Proceed", use_container_width=True):
+            with st.spinner("Applying transformations ..."):
+                final_df = apply_category_transformations(
+                    final_df_loaded.copy(),
+                    bin_dict_loaded.copy(),
+                    transform_params.copy(),
+                    panel.copy(),
+                    specific_transform_params.copy(),
+                )
+
+                # Generate a dictionary mapping original column names to lists of transformed column names
+                transformed_columns_dict, summary_string = generate_transformed_columns(
+                    original_columns, transform_params, specific_transform_params
+                )
+
+                # Store into transformed dataframe and summary session state
+                st.session_state["project_dct"]["transformations"][
+                    "final_df"
+                ] = final_df
+                st.session_state["project_dct"]["transformations"][
+                    "summary_string"
+                ] = summary_string
+
+                # Display success message
+                st.success("Transformation of the DataFrame is successful!", icon="✅")
+
+                # Log message
+                log_message(
+                    "info",
+                    "Transformation of the DataFrame is successful!",
+                    "Transformations",
+                )
+
+    #########################################################################################################################################################
+    # Display the transformed DataFrame and summary
+    #########################################################################################################################################################
+
+    # Display the transformed DataFrame in the Streamlit app
+    st.markdown("### Transformed DataFrame")
+    with st.spinner("Please wait while the transformed DataFrame is loading ..."):
+        final_df = st.session_state["project_dct"]["transformations"]["final_df"].copy()
+
+        # Clean display DataFrame
+        display_df, exceeds_max_col = clean_display_df(final_df, display_max_col)
+
+        # Check the number of columns and show only the first display_max_col if there are more
+        if exceeds_max_col:
+            # Display a info if the DataFrame has more than display_max_col columns
+            st.info(
+                f"The transformed DataFrame has more than {display_max_col} columns. Displaying only the first {display_max_col} columns.",
+                icon="💬",
+            )
+
+        # Display Final DataFrame
+        st.dataframe(
+            display_df,
+            hide_index=True,
+            column_config={
+                "date": st.column_config.DateColumn("date", format="YYYY-MM-DD")
+            },
+        )
+
+        # Total rows and columns
+        total_rows, total_columns = st.session_state["project_dct"]["transformations"][
+            "final_df"
+        ].shape
+        st.markdown(
+            f"<p style='text-align: justify;'>The transformed DataFrame contains <strong>{total_rows}</strong> rows and <strong>{total_columns}</strong> columns.</p>",
+            unsafe_allow_html=True,
+        )
+
+        # Display the summary of transformations as markdown
+        if (
+            "summary_string" in st.session_state["project_dct"]["transformations"]
+            and st.session_state["project_dct"]["transformations"]["summary_string"]
+        ):
+            with st.expander("Summary of Transformations"):
+                st.markdown("### Summary of Transformations")
+                st.markdown(
+                    st.session_state["project_dct"]["transformations"][
+                        "summary_string"
+                    ],
+                    unsafe_allow_html=True,
+                )
+
+    #########################################################################################################################################################
+    # Correlation Plot
+    #########################################################################################################################################################
+
+    # Filter out the 'date' column
+    variables = [
+        col for col in final_df.columns if col.lower() not in ["date", "panel"]
+    ]
+
+    with st.expander("Transformed Variable Correlation Plot"):
+        selected_vars = st.multiselect(
+            label="Choose variables for correlation plot:",
+            options=variables,
+            max_selections=30,
+            default=st.session_state["project_dct"]["transformations"][
+                "correlation_plot_selection"
+            ],
+            key="correlation_plot_key",
+        )
+
+        # Calculate correlation
+        if selected_vars:
+            corr_df = final_df[selected_vars].corr()
+
+            # Prepare text annotations with 2 decimal places
+            annotations = []
+            for i in range(len(corr_df)):
+                for j in range(len(corr_df.columns)):
+                    annotations.append(
+                        go.layout.Annotation(
+                            text=f"{corr_df.iloc[i, j]:.2f}",
+                            x=corr_df.columns[j],
+                            y=corr_df.index[i],
+                            showarrow=False,
+                            font=dict(color="black"),
+                        )
+                    )
+
+            # Plotly correlation plot using go
+            heatmap = go.Heatmap(
+                z=corr_df.values,
+                x=corr_df.columns,
+                y=corr_df.index,
+                colorscale="RdBu",
+                zmin=-1,
+                zmax=1,
+            )
+
+            layout = go.Layout(
+                title="Transformed Variable Correlation Plot",
+                xaxis=dict(title="Variables"),
+                yaxis=dict(title="Variables"),
+                width=1000,
+                height=1000,
+                annotations=annotations,
+            )
+
+            fig = go.Figure(data=[heatmap], layout=layout)
+
+            st.plotly_chart(fig)
+        else:
+            st.write("Please select at least one variable to plot.")
+
+    #########################################################################################################################################################
+    # Accept and Save
+    #########################################################################################################################################################
+
+    # Check for saved model
+    if (
+        retrieve_pkl_object(
+            st.session_state["project_number"], "Model_Build", "best_models", schema
+        )
+        is not None
+    ):  # db
+        st.warning(
+            "Saving transformations will overwrite existing ones and delete all saved models. To keep previous models, please start a new project.",
+            icon="⚠️",
+        )
+
+    if st.button("Accept and Save", use_container_width=True):
+
+        with st.spinner("Saving Changes"):
+            # Update correlation plot selection
+            st.session_state["project_dct"]["transformations"][
+                "correlation_plot_selection"
+            ] = st.session_state["correlation_plot_key"]
+
+            # Clear model metadata
+            clear_pages()
+
+            # Update DB
+            update_db(
+                prj_id=st.session_state["project_number"],
+                page_nam="Transformations",
+                file_nam="project_dct",
+                pkl_obj=pickle.dumps(st.session_state["project_dct"]),
+                schema=schema,
+            )
+
+            # Clear data from DB
+            delete_entries(
+                st.session_state["project_number"],
+                ["Model_Build", "Model_Tuning"],
+                db_cred,
+                schema,
+            )
+
+            # Success message
+            st.success("Saved Successfully!", icon="💾")
+            st.toast("Saved Successfully!", icon="💾")
+
+            # Log message
+            log_message("info", "Saved Successfully!", "Transformations")
+
+except Exception as e:
+    # Capture the error details
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+
+    # Log message
+    log_message("error", f"An error occurred: {error_message}.", "Transformations")
+
+    # Display a warning message
+    st.warning(
+        "Oops! Something went wrong. Please try refreshing the tool or creating a new project.",
+        icon="⚠️",
+    )

pages/5_AI Model_Tuning.py

@@ -0,0 +1,1215 @@
+"""
+MMO Build Sprint 3
+date :
+changes : capability to tune MixedLM as well as simple LR in the same page
+"""
+
+import os
+import streamlit as st
+import pandas as pd
+from data_analysis import format_numbers
+import pickle
+from utilities import set_header, load_local_css
+import statsmodels.api as sm
+import re
+from sklearn.preprocessing import MaxAbsScaler
+import matplotlib.pyplot as plt
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+import statsmodels.formula.api as smf
+from data_prep import *
+import sqlite3
+from utilities import (
+    set_header,
+    load_local_css,
+    update_db,
+    project_selection,
+    retrieve_pkl_object,
+)
+import numpy as np
+from post_gres_cred import db_cred
+import re
+from constants import (
+    NUM_FLAG_COLS_TO_DISPLAY,
+    HALF_YEAR_THRESHOLD,
+    FULL_YEAR_THRESHOLD,
+    TREND_MIN,
+    ANNUAL_FREQUENCY,
+    QTR_FREQUENCY_FACTOR,
+    HALF_YEARLY_FREQUENCY_FACTOR,
+)
+from log_application import log_message
+import sys, traceback
+
+schema = db_cred["schema"]
+
+st.set_option("deprecation.showPyplotGlobalUse", False)
+
+st.set_page_config(
+    page_title="AI Model Tuning",
+    page_icon=":shark:",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+load_local_css("styles.css")
+set_header()
+
+
+# Define functions
+# Get random effect from MixedLM Model
+def get_random_effects(media_data, panel_col, _mdf):
+    # create an empty dataframe
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+
+    # Iterate over all panel values and add to dataframe
+    for i, market in enumerate(media_data[panel_col].unique()):
+        intercept = _mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, "random_effect"] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+# Predict on df using MixedLM model
+def mdf_predict(X_df, mdf, random_eff_df):
+    # Create a copy of input df and predict using MixedLM model i.e fixed effect
+    X = X_df.copy()
+    X["fixed_effect"] = mdf.predict(X)
+
+    # Merge random effects
+    X = pd.merge(X, random_eff_df, on=panel_col, how="left")
+
+    # Get final predictions by adding random effect to fixed effect
+    X["pred"] = X["fixed_effect"] + X["random_effect"]
+
+    # Drop intermediate columns
+    X.drop(columns=["fixed_effect", "random_effect"], inplace=True)
+
+    return X["pred"]
+
+
+def format_display(inp):
+    # Format display titles
+    return inp.title().replace("_", " ").strip()
+
+
+if "username" not in st.session_state:
+    st.session_state["username"] = None
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+if "Flags" not in st.session_state:
+    st.session_state["Flags"] = {}
+
+try:
+    # Check Authentications
+    if "username" in st.session_state and st.session_state["username"] is not None:
+
+        if (
+            retrieve_pkl_object(
+                st.session_state["project_number"], "Model_Build", "best_models", schema
+            )
+            is None
+        ):  # db
+
+            st.error("Please save a model before tuning")
+            log_message(
+                "warning",
+                "No models saved",
+                "Model Tuning",
+            )
+            st.stop()
+
+        # Read previous progress (persistence)
+        if (
+            "session_state_saved"
+            in st.session_state["project_dct"]["model_build"].keys()
+        ):
+            for key in [
+                "Model",
+                "date",
+                "saved_model_names",
+                "media_data",
+                "X_test_spends",
+                "spends_data",
+            ]:
+                if key not in st.session_state:
+                    st.session_state[key] = st.session_state["project_dct"][
+                        "model_build"
+                    ]["session_state_saved"][key]
+                st.session_state["bin_dict"] = st.session_state["project_dct"][
+                    "model_build"
+                ]["session_state_saved"]["bin_dict"]
+                if (
+                    "used_response_metrics" not in st.session_state
+                    or st.session_state["used_response_metrics"] == []
+                ):
+                    st.session_state["used_response_metrics"] = st.session_state[
+                        "project_dct"
+                    ]["model_build"]["session_state_saved"]["used_response_metrics"]
+        else:
+            st.error("Please load a session with a built model")
+            log_message(
+                "error",
+                "Session state saved not found in Project Dictionary",
+                "Model Tuning",
+            )
+            st.stop()
+
+        for key in ["select_all_flags_check", "selected_flags", "sel_model"]:
+            if key not in st.session_state["project_dct"]["model_tuning"].keys():
+                st.session_state["project_dct"]["model_tuning"][key] = {}
+
+        # is_panel = st.session_state['is_panel']
+        # panel_col = 'markets'  # set the panel column
+        date_col = "date"
+
+        # set the panel column
+        panel_col = "panel"
+        is_panel = (
+            True if st.session_state["media_data"][panel_col].nunique() > 1 else False
+        )
+
+        if "Model_Tuned" not in st.session_state:
+            st.session_state["Model_Tuned"] = {}
+        cols1 = st.columns([2, 1])
+        with cols1[0]:
+            st.markdown(f"**Welcome {st.session_state['username']}**")
+        with cols1[1]:
+            st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+        st.title("AI Model Tuning")
+
+        # flag indicating there is not tuned model till now
+        if "is_tuned_model" not in st.session_state:
+            st.session_state["is_tuned_model"] = {}
+
+        # # Read all saved models
+        model_dict = retrieve_pkl_object(
+            st.session_state["project_number"], "Model_Build", "best_models", schema
+        )
+        saved_models = model_dict.keys()
+
+        # Get list of response metrics
+        st.session_state["used_response_metrics"] = list(
+            set([model.split("__")[1] for model in saved_models])
+        )
+
+        # Select previously selected response_metric (persistence)
+        default_target_idx = (
+            st.session_state["project_dct"]["model_tuning"].get("sel_target_col", None)
+            if st.session_state["project_dct"]["model_tuning"].get(
+                "sel_target_col", None
+            )
+            is not None
+            else st.session_state["used_response_metrics"][0]
+        )
+
+        # Dropdown to select response metric
+        sel_target_col = st.selectbox(
+            "Select the response metric",
+            st.session_state["used_response_metrics"],
+            index=st.session_state["used_response_metrics"].index(default_target_idx),
+            format_func=format_display,
+        )
+        # Format selected response metrics (target col)
+        target_col = (
+            sel_target_col.lower()
+            .replace(" ", "_")
+            .replace("-", "")
+            .replace(":", "")
+            .replace("__", "_")
+        )
+        st.session_state["project_dct"]["model_tuning"][
+            "sel_target_col"
+        ] = sel_target_col
+
+        # Look through all saved models, only show saved models of the selected resp metric (target_col)
+        # Get a list of models saved for selected response metric
+        required_saved_models = [
+            m.split("__")[0] for m in saved_models if m.split("__")[1] == target_col
+        ]
+
+        # Get previously seelcted model if available (persistence)
+        default_model_idx = st.session_state["project_dct"]["model_tuning"][
+            "sel_model"
+        ].get(sel_target_col, required_saved_models[0])
+        sel_model = st.selectbox(
+            "Select the model to tune",
+            required_saved_models,
+            index=required_saved_models.index(default_model_idx),
+        )
+
+        st.session_state["project_dct"]["model_tuning"]["sel_model"][
+            sel_target_col
+        ] = default_model_idx
+
+        sel_model_dict = model_dict[
+            sel_model + "__" + target_col
+        ]  # get the model obj of the selected model
+
+        X_train = sel_model_dict["X_train"]
+        X_test = sel_model_dict["X_test"]
+        y_train = sel_model_dict["y_train"]
+        y_test = sel_model_dict["y_test"]
+        df = st.session_state["media_data"]
+
+        st.markdown("### Event Flags")
+        st.markdown("Helps in quantifying the impact of specific occurrences of events")
+
+        try:
+            # Dropdown to add event flags
+            with st.expander("Apply Event Flags"):
+
+                model = sel_model_dict["Model_object"]
+                date = st.session_state["date"]
+                date = pd.to_datetime(date)
+                X_train = sel_model_dict["X_train"]
+
+                features_set = sel_model_dict["feature_set"]
+
+                col = st.columns(3)
+
+                # Get date range
+                min_date = min(date).date()
+                max_date = max(date).date()
+
+                # Get previously selected start and end date of flag (persistence)
+                start_date_default = (
+                    st.session_state["project_dct"]["model_tuning"].get(
+                        "start_date_default"
+                    )
+                    if st.session_state["project_dct"]["model_tuning"].get(
+                        "start_date_default"
+                    )
+                    is not None
+                    else min_date
+                )
+                start_date_default = (
+                    start_date_default if start_date_default > min_date else min_date
+                )
+                start_date_default = (
+                    start_date_default if start_date_default < max_date else min_date
+                )
+
+                end_date_default = (
+                    st.session_state["project_dct"]["model_tuning"].get(
+                        "end_date_default"
+                    )
+                    if st.session_state["project_dct"]["model_tuning"].get(
+                        "end_date_default"
+                    )
+                    is not None
+                    else max_date
+                )
+                end_date_default = (
+                    end_date_default if end_date_default > min_date else max_date
+                )
+                end_date_default = (
+                    end_date_default if end_date_default < max_date else max_date
+                )
+
+                # Flag start and end date input boxes
+                with col[0]:
+                    start_date = st.date_input(
+                        "Select Start Date",
+                        start_date_default,
+                        min_value=min_date,
+                        max_value=max_date,
+                    )
+
+                if (start_date < min_date) or (start_date > max_date):
+                    st.error(
+                        "Please select dates in the range of the dates in the data"
+                    )
+                    st.stop()
+                with col[1]:
+                    # Check if end date default > selected start date
+                    end_date_default = (
+                        end_date_default
+                        if pd.Timestamp(end_date_default) >= pd.Timestamp(start_date)
+                        else start_date
+                    )
+                    end_date = st.date_input(
+                        "Select End Date",
+                        end_date_default,
+                        min_value=max(
+                            pd.to_datetime(min_date), pd.to_datetime(start_date)
+                        ),
+                        max_value=pd.to_datetime(max_date),
+                    )
+
+                if (
+                    (start_date < min_date)
+                    or (end_date < min_date)
+                    or (start_date > max_date)
+                    or (end_date > max_date)
+                ):
+                    st.error(
+                        "Please select dates in the range of the dates in the data"
+                    )
+                    st.stop()
+                if end_date < start_date:
+                    st.error("Please select end date after start date")
+                    st.stop()
+                with col[2]:
+                    # Get default value of repeat check box (persistence)
+                    repeat_default = (
+                        st.session_state["project_dct"]["model_tuning"].get(
+                            "repeat_default"
+                        )
+                        if st.session_state["project_dct"]["model_tuning"].get(
+                            "repeat_default"
+                        )
+                        is not None
+                        else "No"
+                    )
+                    repeat_default_idx = 0 if repeat_default.lower() == "yes" else 1
+                    repeat = st.selectbox(
+                        "Repeat Annually", ["Yes", "No"], index=repeat_default_idx
+                    )
+
+                # Update selected values to session dictionary (persistence)
+                st.session_state["project_dct"]["model_tuning"][
+                    "start_date_default"
+                ] = start_date
+                st.session_state["project_dct"]["model_tuning"][
+                    "end_date_default"
+                ] = end_date
+                st.session_state["project_dct"]["model_tuning"][
+                    "repeat_default"
+                ] = repeat
+
+                if repeat == "Yes":
+                    repeat = True
+                else:
+                    repeat = False
+
+                if "flags" in st.session_state["project_dct"]["model_tuning"].keys():
+                    st.session_state["Flags"] = st.session_state["project_dct"][
+                        "model_tuning"
+                    ]["flags"]
+
+                if is_panel:
+                    # Create flag on Train
+                    met, line_values, fig_flag = plot_actual_vs_predicted(
+                        X_train[date_col],
+                        y_train,
+                        model.fittedvalues,
+                        model,
+                        target_column=sel_target_col,
+                        flag=(start_date, end_date),
+                        repeat_all_years=repeat,
+                        is_panel=True,
+                    )
+                    st.plotly_chart(fig_flag, use_container_width=True)
+
+                    # create flag on test
+                    met, test_line_values, fig_flag = plot_actual_vs_predicted(
+                        X_test[date_col],
+                        y_test,
+                        sel_model_dict["pred_test"],
+                        model,
+                        target_column=sel_target_col,
+                        flag=(start_date, end_date),
+                        repeat_all_years=repeat,
+                        is_panel=True,
+                    )
+
+                else:
+                    pred_train = model.predict(X_train[features_set])
+                    # Create flag on Train
+                    met, line_values, fig_flag = plot_actual_vs_predicted(
+                        X_train[date_col],
+                        y_train,
+                        pred_train,
+                        model,
+                        flag=(start_date, end_date),
+                        repeat_all_years=repeat,
+                        is_panel=False,
+                    )
+                    st.plotly_chart(fig_flag, use_container_width=True)
+
+                    # create flag on test
+                    pred_test = model.predict(X_test[features_set])
+                    met, test_line_values, fig_flag = plot_actual_vs_predicted(
+                        X_test[date_col],
+                        y_test,
+                        pred_test,
+                        model,
+                        flag=(start_date, end_date),
+                        repeat_all_years=repeat,
+                        is_panel=False,
+                    )
+
+                flag_name = "f1_flag"
+                flag_name = st.text_input("Enter Flag Name")
+
+                # add selected target col to flag name
+                # Save the flag name, flag train values, flag test values to session state
+                if st.button("Save flag"):
+                    st.session_state["Flags"][flag_name + "_flag__" + target_col] = {}
+                    st.session_state["Flags"][flag_name + "_flag__" + target_col][
+                        "train"
+                    ] = line_values
+                    st.session_state["Flags"][flag_name + "_flag__" + target_col][
+                        "test"
+                    ] = test_line_values
+                    st.success(f'{flag_name + "_flag__" + target_col} stored')
+
+                    st.session_state["project_dct"]["model_tuning"]["flags"] = (
+                        st.session_state["Flags"]
+                    )
+
+            # Only show flags created for the particular target col
+            target_model_flags = [
+                f.split("__")[0]
+                for f in st.session_state["Flags"].keys()
+                if f.split("__")[1] == target_col
+            ]
+            options = list(target_model_flags)
+            num_rows = -(-len(options) // NUM_FLAG_COLS_TO_DISPLAY)
+
+            tick = False
+            # Select all flags checkbox
+            if st.checkbox(
+                "Select all",
+                value=st.session_state["project_dct"]["model_tuning"][
+                    "select_all_flags_check"
+                ].get(sel_target_col, False),
+            ):
+                tick = True
+                st.session_state["project_dct"]["model_tuning"][
+                    "select_all_flags_check"
+                ][sel_target_col] = True
+            else:
+                st.session_state["project_dct"]["model_tuning"][
+                    "select_all_flags_check"
+                ][sel_target_col] = False
+
+            # Get previous flag selection (persistence)
+            selection_defualts = st.session_state["project_dct"]["model_tuning"][
+                "selected_flags"
+            ].get(sel_target_col, [])
+            selected_options = selection_defualts
+
+            # create a checkbox for each available flag for selected response metric
+            for row in range(num_rows):
+                cols = st.columns(NUM_FLAG_COLS_TO_DISPLAY)
+                for col in cols:
+                    if options:
+                        option = options.pop(0)
+                        option_default = True if option in selection_defualts else False
+                        selected = col.checkbox(option, value=(tick or option_default))
+                        if selected:
+                            selected_options.append(option)
+                        else:
+                            if option in selected_options:
+                                selected_options.remove(option)
+            selected_options = list(set(selected_options))
+
+            # Check if flag values match Data
+            # This is necessary because different models can have different train/test dates
+            remove_flags = []
+            for opt in selected_options:
+                train_match = len(
+                    st.session_state["Flags"][opt + "__" + target_col]["train"]
+                ) == len(X_train[date_col])
+                test_match = len(
+                    st.session_state["Flags"][opt + "__" + target_col]["test"]
+                ) == len(X_test[date_col])
+                if not train_match:
+                    st.warning(f"Flag {opt} can not be used due to train date mismatch")
+                    # selected_options.remove(opt)
+                    remove_flags.append(opt)
+                if not test_match:
+                    st.warning(f"Flag {opt} can not be used due to test date mismatch")
+                    # selected_options.remove(opt)
+                    remove_flags.append(opt)
+
+            if (
+                len(remove_flags) > 0
+                and len(list(set(selected_options).intersection(set(remove_flags)))) > 0
+            ):
+                selected_options = list(set(selected_options) - set(remove_flags))
+
+            st.session_state["project_dct"]["model_tuning"]["selected_flags"][
+                sel_target_col
+            ] = selected_options
+        except:
+            # Capture the error details
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            error_message = "".join(
+                traceback.format_exception(exc_type, exc_value, exc_traceback)
+            )
+            log_message(
+                "error", f"Error while creating flags: {error_message}", "Model Tuning"
+            )
+            st.warning("An error occured, please try again", icon="⚠️")
+
+        try:
+            st.markdown("### Trend and Seasonality Calibration")
+            parameters = st.columns(3)
+
+            # Trend checkbox
+            with parameters[0]:
+                Trend = st.checkbox(
+                    "**Trend**",
+                    value=st.session_state["project_dct"]["model_tuning"].get(
+                        "trend_check", False
+                    ),
+                )
+                st.markdown(
+                    "Helps account for long-term trends or seasonality that could influence advertising effectiveness"
+                )
+
+            # Day of Week (week number) checkbox
+            with parameters[1]:
+                day_of_week = st.checkbox(
+                    "**Day of Week**",
+                    value=st.session_state["project_dct"]["model_tuning"].get(
+                        "week_num_check", False
+                    ),
+                )
+                st.markdown(
+                    "Assists in detecting and incorporating weekly patterns or seasonality"
+                )
+
+            # Sine and cosine Waves checkbox
+            with parameters[2]:
+                sine_cosine = st.checkbox(
+                    "**Sine and Cosine Waves**",
+                    value=st.session_state["project_dct"]["model_tuning"].get(
+                        "sine_cosine_check", False
+                    ),
+                )
+                st.markdown(
+                    "Helps in capturing long term cyclical patterns or seasonality in the data"
+                )
+
+                if sine_cosine:
+                    # Drop down to select Frequency of waves
+                    xtrain_time_period_months = (
+                        X_train[date_col].max() - X_train[date_col].min()
+                    ).days / 30
+
+                    # If we have 6 months of data, only quarter frequency is possible
+                    if xtrain_time_period_months <= HALF_YEAR_THRESHOLD:
+                        available_frequencies = ["Quarter"]
+
+                    # If we have less than 12 months of data, we have quarter and semi-annual frequencies
+                    elif xtrain_time_period_months < FULL_YEAR_THRESHOLD:
+                        available_frequencies = ["Quarter", "Semi-Annual"]
+
+                    # If we have 12 months of data or more, we have quarter, semi-annual and annual frequencies
+                    elif xtrain_time_period_months >= FULL_YEAR_THRESHOLD:
+                        available_frequencies = ["Quarter", "Semi-Annual", "Annual"]
+
+                    wave_freq = st.selectbox("Select Frequency", available_frequencies)
+        except:
+            # Capture the error details
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            error_message = "".join(
+                traceback.format_exception(exc_type, exc_value, exc_traceback)
+            )
+            log_message(
+                "error",
+                f"Error while selecting tuning parameters: {error_message}",
+                "Model Tuning",
+            )
+            st.warning("An error occured, please try again", icon="⚠️")
+
+        try:
+            # Build tuned model
+            if st.button(
+                "Build model with Selected Parameters and Flags",
+                key="build_tuned_model",
+                use_container_width=True,
+            ):
+                new_features = features_set
+                st.header("2.1 Results Summary")
+                ss = MaxAbsScaler()
+                if is_panel == True:
+                    X_train_tuned = X_train[features_set]
+                    X_train_tuned[target_col] = X_train[target_col]
+                    X_train_tuned[date_col] = X_train[date_col]
+                    X_train_tuned[panel_col] = X_train[panel_col]
+
+                    X_test_tuned = X_test[features_set]
+                    X_test_tuned[target_col] = X_test[target_col]
+                    X_test_tuned[date_col] = X_test[date_col]
+                    X_test_tuned[panel_col] = X_test[panel_col]
+
+                else:
+                    X_train_tuned = X_train[features_set]
+
+                    X_test_tuned = X_test[features_set]
+
+                for flag in selected_options:
+                    # Get the flag values of train and test and add to the data
+                    X_train_tuned[flag] = st.session_state["Flags"][
+                        flag + "__" + target_col
+                    ]["train"]
+                    X_test_tuned[flag] = st.session_state["Flags"][
+                        flag + "__" + target_col
+                    ]["test"]
+
+                if Trend:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "trend_check"
+                    ] = True
+
+                    # group by panel, calculate trend of each panel spearately. Add trend to new feature set
+                    if is_panel:
+                        newdata = pd.DataFrame()
+                        panel_wise_end_point_train = {}
+                        for panel, groupdf in X_train_tuned.groupby(panel_col):
+                            groupdf.sort_values(date_col, inplace=True)
+                            groupdf["Trend"] = np.arange(
+                                TREND_MIN, len(groupdf) + TREND_MIN, 1
+                            )  # Trend is a straight line with starting point as TREND_MIN
+                            newdata = pd.concat([newdata, groupdf])
+                            panel_wise_end_point_train[panel] = len(groupdf) + TREND_MIN
+                        X_train_tuned = newdata.copy()
+
+                        test_newdata = pd.DataFrame()
+                        for panel, test_groupdf in X_test_tuned.groupby(panel_col):
+                            test_groupdf.sort_values(date_col, inplace=True)
+                            start = panel_wise_end_point_train[panel]
+                            end = start + len(test_groupdf)
+                            test_groupdf["Trend"] = np.arange(start, end, 1)
+                            test_newdata = pd.concat([test_newdata, test_groupdf])
+                        X_test_tuned = test_newdata.copy()
+
+                        new_features = new_features + ["Trend"]
+
+                    else:
+                        X_train_tuned["Trend"] = np.arange(
+                            TREND_MIN, len(X_train_tuned) + TREND_MIN, 1
+                        )  # Trend is a straight line with starting point as TREND_MIN
+                        X_test_tuned["Trend"] = np.arange(
+                            len(X_train_tuned) + TREND_MIN,
+                            len(X_train_tuned) + len(X_test_tuned) + TREND_MIN,
+                            1,
+                        )
+                        new_features = new_features + ["Trend"]
+
+                else:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "trend_check"
+                    ] = False  # persistence
+
+                # Add day of week (Week_num) to test & train
+                if day_of_week:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "week_num_check"
+                    ] = True
+
+                    if is_panel:
+                        X_train_tuned[date_col] = pd.to_datetime(
+                            X_train_tuned[date_col]
+                        )
+                        X_train_tuned["day_of_week"] = X_train_tuned[
+                            date_col
+                        ].dt.day_of_week  # Day of week
+                        # if all the dates in the data have the same day of week number this feature cant be used
+                        if X_train_tuned["day_of_week"].nunique() == 1:
+                            st.error(
+                                "All dates in the data are of the same week day. Hence Week number can't be used."
+                            )
+                        else:
+                            X_test_tuned[date_col] = pd.to_datetime(
+                                X_test_tuned[date_col]
+                            )
+                            X_test_tuned["day_of_week"] = X_test_tuned[
+                                date_col
+                            ].dt.day_of_week  # Day of week
+                            new_features = new_features + ["day_of_week"]
+
+                    else:
+                        date = pd.to_datetime(date.values)
+                        X_train_tuned["day_of_week"] = pd.to_datetime(
+                            X_train[date_col]
+                        ).dt.day_of_week  # Day of week
+                        X_test_tuned["day_of_week"] = pd.to_datetime(
+                            X_test[date_col]
+                        ).dt.day_of_week  # Day of week
+
+                        # if all the dates in the data have the same day of week number this feature cant be used
+                        if X_train_tuned["day_of_week"].nunique() == 1:
+                            st.error(
+                                "All dates in the data are of the same week day. Hence Week number can't be used."
+                            )
+                        else:
+                            new_features = new_features + ["day_of_week"]
+                else:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "week_num_check"
+                    ] = False
+
+                # create sine and cosine wave and add to data
+                if sine_cosine:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "sine_cosine_check"
+                    ] = True
+                    frequency = ANNUAL_FREQUENCY  # Annual Frequency
+                    if wave_freq == "Quarter":
+                        frequency = frequency * QTR_FREQUENCY_FACTOR
+                    elif wave_freq == "Semi-Annual":
+                        frequency = frequency * HALF_YEARLY_FREQUENCY_FACTOR
+                    # create panel wise sine cosine waves in xtrain tuned. add to new feature set
+                    if is_panel:
+                        new_features = new_features + ["sine_wave", "cosine_wave"]
+                        newdata = pd.DataFrame()
+                        newdata_test = pd.DataFrame()
+                        groups = X_train_tuned.groupby(panel_col)
+
+                        train_panel_wise_end_point = {}
+                        for panel, groupdf in groups:
+                            num_samples = len(groupdf)
+                            train_panel_wise_end_point[panel] = num_samples
+                            days_since_start = np.arange(num_samples)
+                            sine_wave = np.sin(frequency * days_since_start)
+                            cosine_wave = np.cos(frequency * days_since_start)
+                            sine_cosine_df = pd.DataFrame(
+                                {"sine_wave": sine_wave, "cosine_wave": cosine_wave}
+                            )
+                            assert len(sine_cosine_df) == len(groupdf)
+
+                            groupdf["sine_wave"] = sine_wave
+                            groupdf["cosine_wave"] = cosine_wave
+                            newdata = pd.concat([newdata, groupdf])
+
+                        X_train_tuned = newdata.copy()
+
+                        test_groups = X_test_tuned.groupby(panel_col)
+                        for panel, test_groupdf in test_groups:
+                            num_samples = len(test_groupdf)
+                            start = train_panel_wise_end_point[panel]
+                            days_since_start = np.arange(start, start + num_samples, 1)
+                            # print("##", panel, num_samples, start, len(np.arange(start, start+num_samples, 1)))
+                            sine_wave = np.sin(frequency * days_since_start)
+                            cosine_wave = np.cos(frequency * days_since_start)
+                            sine_cosine_df = pd.DataFrame(
+                                {"sine_wave": sine_wave, "cosine_wave": cosine_wave}
+                            )
+                            assert len(sine_cosine_df) == len(test_groupdf)
+                            # groupdf = pd.concat([groupdf, sine_cosine_df], axis=1)
+                            test_groupdf["sine_wave"] = sine_wave
+                            test_groupdf["cosine_wave"] = cosine_wave
+                            newdata_test = pd.concat([newdata_test, test_groupdf])
+
+                        X_test_tuned = newdata_test.copy()
+
+                    else:
+                        new_features = new_features + ["sine_wave", "cosine_wave"]
+
+                        num_samples = len(X_train_tuned)
+
+                        days_since_start = np.arange(num_samples)
+                        sine_wave = np.sin(frequency * days_since_start)
+                        cosine_wave = np.cos(frequency * days_since_start)
+                        sine_cosine_df = pd.DataFrame(
+                            {"sine_wave": sine_wave, "cosine_wave": cosine_wave}
+                        )
+                        # Concatenate the sine and cosine waves with the scaled X DataFrame
+                        X_train_tuned = pd.concat(
+                            [X_train_tuned, sine_cosine_df], axis=1
+                        )
+
+                        test_num_samples = len(X_test_tuned)
+                        start = num_samples
+                        days_since_start = np.arange(start, start + test_num_samples, 1)
+                        sine_wave = np.sin(frequency * days_since_start)
+                        cosine_wave = np.cos(frequency * days_since_start)
+                        sine_cosine_df = pd.DataFrame(
+                            {"sine_wave": sine_wave, "cosine_wave": cosine_wave}
+                        )
+                        # Concatenate the sine and cosine waves with the scaled X DataFrame
+                        X_test_tuned = pd.concat([X_test_tuned, sine_cosine_df], axis=1)
+
+                else:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "sine_cosine_check"
+                    ] = False
+
+                # Build model
+
+                # Get list of parameters added and scale
+                # previous features are scaled already during model build
+                added_params = list(set(new_features) - set(features_set))
+                if len(added_params) > 0:
+                    concat_df = pd.concat([X_train_tuned, X_test_tuned]).reset_index(
+                        drop=True
+                    )
+
+                    if is_panel:
+                        train_max_date = X_train_tuned[date_col].max()
+
+                    # concat_df = concat_df.reset_index(drop=True)
+                    # concat_df=concat_df[added_params]
+                    train_idx = X_train_tuned.index[-1]
+
+                    concat_df[added_params] = ss.fit_transform(concat_df[added_params])
+                    # added_params_df = pd.DataFrame(added_params_df)
+                    # added_params_df.columns = added_params
+
+                    if is_panel:
+                        X_train_tuned[added_params] = concat_df[
+                            concat_df[date_col] <= train_max_date
+                        ][added_params].reset_index(drop=True)
+                        X_test_tuned[added_params] = concat_df[
+                            concat_df[date_col] > train_max_date
+                        ][added_params].reset_index(drop=True)
+                    else:
+                        added_params_df = concat_df[added_params]
+                        X_train_tuned[added_params] = added_params_df[: train_idx + 1]
+                        X_test_tuned[added_params] = added_params_df.loc[
+                            train_idx + 1 :
+                        ].reset_index(drop=True)
+
+                # Add flags (flags are 0, 1 only so need to scale)
+                if selected_options:
+                    new_features = new_features + selected_options
+
+                # Build Mixed LM model for panel level data
+                if is_panel:
+                    X_train_tuned.sort_values([date_col, panel_col]).reset_index(
+                        drop=True, inplace=True
+                    )
+
+                    new_features = list(set(new_features))
+                    inp_vars_str = " + ".join(new_features)
+
+                    md_str = target_col + " ~ " + inp_vars_str
+                    md_tuned = smf.mixedlm(
+                        md_str,
+                        data=X_train_tuned[[target_col] + new_features],
+                        groups=X_train_tuned[panel_col],
+                    )
+                    model_tuned = md_tuned.fit()
+
+                    # plot actual vs predicted for original model and tuned model
+                    metrics_table, line, actual_vs_predicted_plot = (
+                        plot_actual_vs_predicted(
+                            X_train[date_col],
+                            y_train,
+                            model.fittedvalues,
+                            model,
+                            target_column=sel_target_col,
+                            is_panel=True,
+                        )
+                    )
+                    metrics_table_tuned, line, actual_vs_predicted_plot_tuned = (
+                        plot_actual_vs_predicted(
+                            X_train_tuned[date_col],
+                            X_train_tuned[target_col],
+                            model_tuned.fittedvalues,
+                            model_tuned,
+                            target_column=sel_target_col,
+                            is_panel=True,
+                        )
+                    )
+
+                # Build OLS model for panel level data
+                else:
+                    new_features = list(set(new_features))
+                    model_tuned = sm.OLS(y_train, X_train_tuned[new_features]).fit()
+                    metrics_table, line, actual_vs_predicted_plot = (
+                        plot_actual_vs_predicted(
+                            X_train[date_col],
+                            y_train,
+                            model.predict(X_train[features_set]),
+                            model,
+                            target_column=sel_target_col,
+                        )
+                    )
+
+                    metrics_table_tuned, line, actual_vs_predicted_plot_tuned = (
+                        plot_actual_vs_predicted(
+                            X_train[date_col],
+                            y_train,
+                            model_tuned.predict(X_train_tuned[new_features]),
+                            model_tuned,
+                            target_column=sel_target_col,
+                        )
+                    )
+
+                # # ----------------------------------- TESTING -----------------------------------
+                #
+                # Plot Sine & cosine wave to test
+                # sine_cosine_plot = plot_actual_vs_predicted(
+                #     X_train[date_col],
+                #     y_train,
+                #     X_train_tuned['sine_wave'],
+                #     model_tuned,
+                #     target_column=sel_target_col,
+                #     is_panel=True,
+                # )
+                # st.plotly_chart(sine_cosine_plot, use_container_width=True)
+
+                # # Plot Trend line to test
+                # trend_plot = plot_tuned_params(
+                #     X_train[date_col],
+                #     y_train,
+                #     X_train_tuned['Trend'],
+                #     model_tuned,
+                #     target_column=sel_target_col,
+                #     is_panel=True,
+                # )
+                # st.plotly_chart(trend_plot, use_container_width=True)
+                #
+                # # Plot week number to test
+                # week_num_plot = plot_tuned_params(
+                #     X_train[date_col],
+                #     y_train,
+                #     X_train_tuned['day_of_week'],
+                #     model_tuned,
+                #     target_column=sel_target_col,
+                #     is_panel=True,
+                # )
+                # st.plotly_chart(week_num_plot, use_container_width=True)
+
+                # Get model metrics from metric table & display them
+                mape = np.round(metrics_table.iloc[0, 1], 2)
+                r2 = np.round(metrics_table.iloc[1, 1], 2)
+                adjr2 = np.round(metrics_table.iloc[2, 1], 2)
+
+                mape_tuned = np.round(metrics_table_tuned.iloc[0, 1], 2)
+                r2_tuned = np.round(metrics_table_tuned.iloc[1, 1], 2)
+                adjr2_tuned = np.round(metrics_table_tuned.iloc[2, 1], 2)
+
+                parameters_ = st.columns(3)
+                with parameters_[0]:
+                    st.metric("R-squared", r2_tuned, np.round(r2_tuned - r2, 2))
+                with parameters_[1]:
+                    st.metric(
+                        "Adj. R-squared", adjr2_tuned, np.round(adjr2_tuned - adjr2, 2)
+                    )
+                with parameters_[2]:
+                    st.metric(
+                        "MAPE", mape_tuned, np.round(mape_tuned - mape, 2), "inverse"
+                    )
+
+                st.write(model_tuned.summary())
+
+                X_train_tuned[date_col] = X_train[date_col]
+                X_train_tuned[target_col] = y_train
+                X_test_tuned[date_col] = X_test[date_col]
+                X_test_tuned[target_col] = y_test
+
+                st.header("2.2 Actual vs. Predicted Plot (Train)")
+                if is_panel:
+                    metrics_table, line, actual_vs_predicted_plot = (
+                        plot_actual_vs_predicted(
+                            X_train_tuned[date_col],
+                            X_train_tuned[target_col],
+                            model_tuned.fittedvalues,
+                            model_tuned,
+                            target_column=sel_target_col,
+                            is_panel=True,
+                        )
+                    )
+                else:
+
+                    metrics_table, line, actual_vs_predicted_plot = (
+                        plot_actual_vs_predicted(
+                            X_train_tuned[date_col],
+                            X_train_tuned[target_col],
+                            model_tuned.predict(X_train_tuned[new_features]),
+                            model_tuned,
+                            target_column=sel_target_col,
+                            is_panel=False,
+                        )
+                    )
+
+                st.plotly_chart(actual_vs_predicted_plot, use_container_width=True)
+
+                st.markdown("## 2.3 Residual Analysis (Train)")
+                if is_panel:
+                    columns = st.columns(2)
+                    with columns[0]:
+                        fig = plot_residual_predicted(
+                            y_train, model_tuned.fittedvalues, X_train_tuned
+                        )
+                        st.plotly_chart(fig)
+
+                    with columns[1]:
+                        st.empty()
+                        fig = qqplot(y_train, model_tuned.fittedvalues)
+                        st.plotly_chart(fig)
+
+                    with columns[0]:
+                        fig = residual_distribution(y_train, model_tuned.fittedvalues)
+                        st.pyplot(fig)
+                else:
+                    columns = st.columns(2)
+                    with columns[0]:
+                        fig = plot_residual_predicted(
+                            y_train,
+                            model_tuned.predict(X_train_tuned[new_features]),
+                            X_train,
+                        )
+                        st.plotly_chart(fig)
+
+                    with columns[1]:
+                        st.empty()
+                        fig = qqplot(
+                            y_train, model_tuned.predict(X_train_tuned[new_features])
+                        )
+                        st.plotly_chart(fig)
+
+                    with columns[0]:
+                        fig = residual_distribution(
+                            y_train, model_tuned.predict(X_train_tuned[new_features])
+                        )
+                        st.pyplot(fig)
+
+                # st.session_state['is_tuned_model'][target_col] = True
+                # Save tuned model in a dict
+                st.session_state["Model_Tuned"][sel_model + "__" + target_col] = {
+                    "Model_object": model_tuned,
+                    "feature_set": new_features,
+                    "X_train_tuned": X_train_tuned,
+                    "X_test_tuned": X_test_tuned,
+                }
+
+                with st.expander("Results Summary Test data"):
+                    if is_panel:
+                        random_eff_df = get_random_effects(
+                            st.session_state.media_data.copy(), panel_col, model_tuned
+                        )
+                        test_pred = mdf_predict(
+                            X_test_tuned, model_tuned, random_eff_df
+                        )
+                    else:
+                        test_pred = model_tuned.predict(X_test_tuned[new_features])
+                    st.header("2.2 Actual vs. Predicted Plot (Test)")
+
+                    metrics_table, line, actual_vs_predicted_plot = (
+                        plot_actual_vs_predicted(
+                            X_test_tuned[date_col],
+                            y_test,
+                            test_pred,
+                            model,
+                            target_column=sel_target_col,
+                            is_panel=is_panel,
+                        )
+                    )
+                    st.plotly_chart(actual_vs_predicted_plot, use_container_width=True)
+                    st.markdown("## 2.3 Residual Analysis (Test)")
+
+                    columns = st.columns(2)
+                    with columns[0]:
+                        fig = plot_residual_predicted(y_test, test_pred, X_test_tuned)
+                        st.plotly_chart(fig)
+
+                    with columns[1]:
+                        st.empty()
+                        fig = qqplot(y_test, test_pred)
+                        st.plotly_chart(fig)
+
+                    with columns[0]:
+                        fig = residual_distribution(y_test, test_pred)
+                        st.pyplot(fig)
+        except:
+            # Capture the error details
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            error_message = "".join(
+                traceback.format_exception(exc_type, exc_value, exc_traceback)
+            )
+            log_message(
+                "error",
+                f"Error while building tuned model: {error_message}",
+                "Model Tuning",
+            )
+            st.warning("An error occured, please try again", icon="⚠️")
+
+        if (
+            st.session_state["Model_Tuned"] is not None
+            and len(list(st.session_state["Model_Tuned"].keys())) > 0
+        ):
+            if st.button("Use This model for Media Planning", use_container_width=True):
+
+                # remove previous tuned models saved for this target col
+                _remove = [
+                    m
+                    for m in st.session_state["Model_Tuned"].keys()
+                    if m.split("__")[1] == target_col and m.split("__")[0] != sel_model
+                ]
+                if len(_remove) > 0:
+                    for m in _remove:
+                        del st.session_state["Model_Tuned"][m]
+
+                # Flag depicting tuned model for selected response metric
+                st.session_state["is_tuned_model"][target_col] = True
+
+                tuned_model_pkl = pickle.dumps(st.session_state["Model_Tuned"])
+
+                update_db(
+                    st.session_state["project_number"],
+                    "Model_Tuning",
+                    "tuned_model",
+                    tuned_model_pkl,
+                    schema,
+                    # resp_mtrc=None,
+                )  # db
+
+                log_message(
+                    "info",
+                    f"Tuned model {' '.join(_remove)} removed due to overwrite",
+                    "Model Tuning",
+                )
+
+                # Save session state variables (persistence)
+                st.session_state["project_dct"]["model_tuning"][
+                    "session_state_saved"
+                ] = {}
+                for key in [
+                    "bin_dict",
+                    "used_response_metrics",
+                    "is_tuned_model",
+                    "media_data",
+                    "X_test_spends",
+                    "spends_data",
+                ]:
+                    st.session_state["project_dct"]["model_tuning"][
+                        "session_state_saved"
+                    ][key] = st.session_state[key]
+
+                project_dct_pkl = pickle.dumps(st.session_state["project_dct"])
+
+                update_db(
+                    st.session_state["project_number"],
+                    "Model_Tuning",
+                    "project_dct",
+                    project_dct_pkl,
+                    schema,
+                    # resp_mtrc=None,
+                )  # db
+
+                log_message(
+                    "info",
+                    f'Tuned Model {sel_model + "__" + target_col} Saved',
+                    "Model Tuning",
+                )
+
+                # Clear page metadata
+                st.session_state["project_dct"]["scenario_planner"][
+                    "modified_metadata_file"
+                ] = None
+                st.session_state["project_dct"]["response_curves"][
+                    "modified_metadata_file"
+                ] = None
+
+                st.success(sel_model + " for " + target_col + " Tuned saved!")
+except:
+    # Capture the error details
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+    log_message("error", f"An error has occured : {error_message}", "Model Tuning")
+    st.warning("An error occured, please try again", icon="⚠️")
+    # st.write(error_message)

pages/6_AI_Model_Validation.py

@@ -0,0 +1,960 @@
+import plotly.express as px
+import numpy as np
+import plotly.graph_objects as go
+import streamlit as st
+import pandas as pd
+import statsmodels.api as sm
+
+# from sklearn.metrics import mean_absolute_percentage_error
+import sys
+import os
+from utilities import set_header, load_local_css
+import seaborn as sns
+import matplotlib.pyplot as plt
+import tempfile
+from sklearn.preprocessing import MinMaxScaler
+
+# from st_aggrid import AgGrid
+# from st_aggrid import GridOptionsBuilder, GridUpdateMode
+# from st_aggrid import GridOptionsBuilder
+import sys
+import re
+import pickle
+from sklearn.metrics import r2_score
+from data_prep import plot_actual_vs_predicted
+import sqlite3
+from utilities import (
+    set_header,
+    load_local_css,
+    update_db,
+    project_selection,
+    retrieve_pkl_object,
+)
+from post_gres_cred import db_cred
+from log_application import log_message
+import sys, traceback
+
+schema = db_cred["schema"]
+
+sys.setrecursionlimit(10**6)
+
+original_stdout = sys.stdout
+sys.stdout = open("temp_stdout.txt", "w")
+sys.stdout.close()
+sys.stdout = original_stdout
+
+st.set_page_config(layout="wide")
+load_local_css("styles.css")
+set_header()
+
+
+## DEFINE ALL FUCNTIONS
+def plot_residual_predicted(actual, predicted, df_):
+    df_["Residuals"] = actual - pd.Series(predicted)
+    df_["StdResidual"] = (df_["Residuals"] - df_["Residuals"].mean()) / df_[
+        "Residuals"
+    ].std()
+
+    # Create a Plotly scatter plot
+    fig = px.scatter(
+        df_,
+        x=predicted,
+        y="StdResidual",
+        opacity=0.5,
+        color_discrete_sequence=["#11B6BD"],
+    )
+
+    # Add horizontal lines
+    fig.add_hline(y=0, line_dash="dash", line_color="darkorange")
+    fig.add_hline(y=2, line_color="red")
+    fig.add_hline(y=-2, line_color="red")
+
+    fig.update_xaxes(title="Predicted")
+    fig.update_yaxes(title="Standardized Residuals (Actual - Predicted)")
+
+    # Set the same width and height for both figures
+    fig.update_layout(
+        title="Residuals over Predicted Values",
+        autosize=False,
+        width=600,
+        height=400,
+    )
+
+    return fig
+
+
+def residual_distribution(actual, predicted):
+    Residuals = actual - pd.Series(predicted)
+
+    # Create a Seaborn distribution plot
+    sns.set(style="whitegrid")
+    plt.figure(figsize=(6, 4))
+    sns.histplot(Residuals, kde=True, color="#11B6BD")
+
+    plt.title(" Distribution of Residuals")
+    plt.xlabel("Residuals")
+    plt.ylabel("Probability Density")
+
+    return plt
+
+
+def qqplot(actual, predicted):
+    Residuals = actual - pd.Series(predicted)
+    Residuals = pd.Series(Residuals)
+    Resud_std = (Residuals - Residuals.mean()) / Residuals.std()
+
+    # Create a QQ plot using Plotly with custom colors
+    fig = go.Figure()
+    fig.add_trace(
+        go.Scatter(
+            x=sm.ProbPlot(Resud_std).theoretical_quantiles,
+            y=sm.ProbPlot(Resud_std).sample_quantiles,
+            mode="markers",
+            marker=dict(size=5, color="#11B6BD"),
+            name="QQ Plot",
+        )
+    )
+
+    # Add the 45-degree reference line
+    diagonal_line = go.Scatter(
+        x=[
+            -2,
+            2,
+        ],  # Adjust the x values as needed to fit the range of your data
+        y=[-2, 2],  # Adjust the y values accordingly
+        mode="lines",
+        line=dict(color="red"),  # Customize the line color and style
+        name=" ",
+    )
+    fig.add_trace(diagonal_line)
+
+    # Customize the layout
+    fig.update_layout(
+        title="QQ Plot of Residuals",
+        title_x=0.5,
+        autosize=False,
+        width=600,
+        height=400,
+        xaxis_title="Theoretical Quantiles",
+        yaxis_title="Sample Quantiles",
+    )
+
+    return fig
+
+
+def get_random_effects(media_data, panel_col, mdf):
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+    for i, market in enumerate(media_data[panel_col].unique()):
+        print(i, end="\r")
+        intercept = mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, "random_effect"] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+def mdf_predict(X_df, mdf, random_eff_df):
+    X = X_df.copy()
+    X = pd.merge(
+        X,
+        random_eff_df[[panel_col, "random_effect"]],
+        on=panel_col,
+        how="left",
+    )
+    X["pred_fixed_effect"] = mdf.predict(X)
+
+    X["pred"] = X["pred_fixed_effect"] + X["random_effect"]
+    X.drop(columns=["pred_fixed_effect", "random_effect"], inplace=True)
+    return X
+
+
+def metrics_df_panel(model_dict, is_panel):
+    def wmape(actual, forecast):
+        # Weighted MAPE (WMAPE) eliminates the following shortcomings of MAPE & SMAPE
+        ## 1. MAPE becomes insanely high when actual is close to 0
+        ## 2. MAPE is more favourable to underforecast than overforecast
+        return np.sum(np.abs(actual - forecast)) / np.sum(np.abs(actual))
+
+    metrics_df = pd.DataFrame(
+        columns=[
+            "Model",
+            "R2",
+            "ADJR2",
+            "Train Mape",
+            "Test Mape",
+            "Summary",
+            "Model_object",
+        ]
+    )
+    i = 0
+    for key in model_dict.keys():
+        target = key.split("__")[1]
+        metrics_df.at[i, "Model"] = target
+        y = model_dict[key]["X_train_tuned"][target]
+
+        feature_set = model_dict[key]["feature_set"]
+
+        if is_panel:
+            random_df = get_random_effects(
+                media_data, panel_col, model_dict[key]["Model_object"]
+            )
+            pred = mdf_predict(
+                model_dict[key]["X_train_tuned"],
+                model_dict[key]["Model_object"],
+                random_df,
+            )["pred"]
+        else:
+            pred = model_dict[key]["Model_object"].predict(
+                model_dict[key]["X_train_tuned"][feature_set]
+            )
+
+        ytest = model_dict[key]["X_test_tuned"][target]
+        if is_panel:
+
+            predtest = mdf_predict(
+                model_dict[key]["X_test_tuned"],
+                model_dict[key]["Model_object"],
+                random_df,
+            )["pred"]
+
+        else:
+            predtest = model_dict[key]["Model_object"].predict(
+                model_dict[key]["X_test_tuned"][feature_set]
+            )
+
+        metrics_df.at[i, "R2"] = r2_score(y, pred)
+        metrics_df.at[i, "ADJR2"] = 1 - (1 - metrics_df.loc[i, "R2"]) * (len(y) - 1) / (
+            len(y) - len(model_dict[key]["feature_set"]) - 1
+        )
+        # metrics_df.at[i, "Train Mape"] = mean_absolute_percentage_error(y, pred)
+        # metrics_df.at[i, "Test Mape"] = mean_absolute_percentage_error(
+        #     ytest, predtest
+        # )
+        metrics_df.at[i, "Train Mape"] = wmape(y, pred)
+        metrics_df.at[i, "Test Mape"] = wmape(ytest, predtest)
+        metrics_df.at[i, "Summary"] = model_dict[key]["Model_object"].summary()
+        metrics_df.at[i, "Model_object"] = model_dict[key]["Model_object"]
+        i += 1
+    metrics_df = np.round(metrics_df, 2)
+
+    metrics_df.rename(
+        columns={"R2": "R-squared", "ADJR2": "Adj. R-squared"}, inplace=True
+    )
+    return metrics_df
+
+
+def map_channel(transformed_var, channel_dict):
+    for key, value_list in channel_dict.items():
+        if any(raw_var in transformed_var for raw_var in value_list):
+            return key
+    return transformed_var  # Return the original value if no match is found
+
+
+def contributions_nonpanel(model_dict):
+    # with open(os.path.join(st.session_state["project_path"], "channel_groups.pkl"), "rb") as f:
+    #     channels = pickle.load(f)
+
+    channels = st.session_state["project_dct"]["data_import"]["group_dict"]  # db
+    media_data = st.session_state["media_data"]
+    contribution_df = pd.DataFrame(columns=["Channel"])
+
+    for key in model_dict.keys():
+
+        best_feature_set = model_dict[key]["feature_set"]
+        model = model_dict[key]["Model_object"]
+        target = key.split("__")[1]
+        X_train = model_dict[key]["X_train_tuned"]
+        contri_df = pd.DataFrame()
+        y = []
+        y_pred = []
+
+        coef_df = pd.DataFrame(model.params)
+        coef_df.reset_index(inplace=True)
+        coef_df.columns = ["feature", "coef"]
+        x_train_contribution = X_train.copy()
+        x_train_contribution["pred"] = model.predict(X_train[best_feature_set])
+
+        for i in range(len(coef_df)):
+
+            coef = coef_df.loc[i, "coef"]
+            col = coef_df.loc[i, "feature"]
+            if col != "const":
+                x_train_contribution[str(col) + "_contr"] = (
+                    coef * x_train_contribution[col]
+                )
+            else:
+                x_train_contribution["const"] = coef
+
+        tuning_cols = [
+            c
+            for c in x_train_contribution.filter(regex="contr").columns
+            if c
+            in [
+                "day_of_week_contr",
+                "Trend_contr",
+                "sine_wave_contr",
+                "cosine_wave_contr",
+            ]
+        ]
+        flag_cols = [
+            c
+            for c in x_train_contribution.filter(regex="contr").columns
+            if "_flag" in c
+        ]
+
+        # add exogenous contribution to base
+        all_exog_vars = st.session_state["bin_dict"]["Exogenous"]
+        all_exog_vars = [
+            var.lower()
+            .replace(".", "_")
+            .replace("@", "_")
+            .replace(" ", "_")
+            .replace("-", "")
+            .replace(":", "")
+            .replace("__", "_")
+            for var in all_exog_vars
+        ]
+        exog_cols = []
+        if len(all_exog_vars) > 0:
+            for col in x_train_contribution.filter(regex="contr").columns:
+                if len([exog_var for exog_var in all_exog_vars if exog_var in col]) > 0:
+                    exog_cols.append(col)
+
+        base_cols = ["const"] + flag_cols + tuning_cols + exog_cols
+
+        x_train_contribution["base_contr"] = x_train_contribution[base_cols].sum(axis=1)
+        x_train_contribution.drop(columns=base_cols, inplace=True)
+
+        contri_df = pd.DataFrame(x_train_contribution.filter(regex="contr").sum(axis=0))
+
+        contri_df.reset_index(inplace=True)
+        contri_df.columns = ["Channel", target]
+        contri_df["Channel"] = contri_df["Channel"].apply(
+            lambda x: map_channel(x, channels)
+        )
+        contri_df[target] = 100 * contri_df[target] / contri_df[target].sum()
+        contri_df["Channel"].replace("base_contr", "base", inplace=True)
+        contribution_df = pd.merge(
+            contribution_df, contri_df, on="Channel", how="outer"
+        )
+
+    return contribution_df
+
+
+def contributions_panel(model_dict):
+    channels = st.session_state["project_dct"]["data_import"]["group_dict"]  # db
+    media_data = st.session_state["media_data"]
+    contribution_df = pd.DataFrame(columns=["Channel"])
+    for key in model_dict.keys():
+        best_feature_set = model_dict[key]["feature_set"]
+        model = model_dict[key]["Model_object"]
+        target = key.split("__")[1]
+        X_train = model_dict[key]["X_train_tuned"]
+        contri_df = pd.DataFrame()
+
+        y = []
+        y_pred = []
+
+        random_eff_df = get_random_effects(media_data, panel_col, model)
+        random_eff_df["fixed_effect"] = model.fe_params["Intercept"]
+        random_eff_df["panel_effect"] = (
+            random_eff_df["random_effect"] + random_eff_df["fixed_effect"]
+        )
+
+        coef_df = pd.DataFrame(model.fe_params)
+        coef_df.reset_index(inplace=True)
+        coef_df.columns = ["feature", "coef"]
+
+        x_train_contribution = X_train.copy()
+        x_train_contribution = mdf_predict(x_train_contribution, model, random_eff_df)
+
+        x_train_contribution = pd.merge(
+            x_train_contribution,
+            random_eff_df[[panel_col, "panel_effect"]],
+            on=panel_col,
+            how="left",
+        )
+        for i in range(len(coef_df)):
+            coef = coef_df.loc[i, "coef"]
+            col = coef_df.loc[i, "feature"]
+            if col.lower() != "intercept":
+                x_train_contribution[str(col) + "_contr"] = (
+                    coef * x_train_contribution[col]
+                )
+
+        # x_train_contribution['sum_contributions'] = x_train_contribution.filter(regex="contr").sum(axis=1)
+        # x_train_contribution['sum_contributions'] = x_train_contribution['sum_contributions'] + x_train_contribution[
+        #     'panel_effect']
+
+        # base_cols = ["panel_effect"] + [
+        #     c
+        #     for c in x_train_contribution.filter(regex="contr").columns
+        #     if c
+        #     in [
+        #         "day_of_week_contr",
+        #         "Trend_contr",
+        #         "sine_wave_contr",
+        #         "cosine_wave_contr",
+        #     ]
+        # ]
+        tuning_cols = [
+            c
+            for c in x_train_contribution.filter(regex="contr").columns
+            if c
+            in [
+                "day_of_week_contr",
+                "Trend_contr",
+                "sine_wave_contr",
+                "cosine_wave_contr",
+            ]
+        ]
+        flag_cols = [
+            c
+            for c in x_train_contribution.filter(regex="contr").columns
+            if "_flag" in c
+        ]
+
+        # add exogenous contribution to base
+        all_exog_vars = st.session_state["bin_dict"]["Exogenous"]
+        all_exog_vars = [
+            var.lower()
+            .replace(".", "_")
+            .replace("@", "_")
+            .replace(" ", "_")
+            .replace("-", "")
+            .replace(":", "")
+            .replace("__", "_")
+            for var in all_exog_vars
+        ]
+        exog_cols = []
+        if len(all_exog_vars) > 0:
+            for col in x_train_contribution.filter(regex="contr").columns:
+                if len([exog_var for exog_var in all_exog_vars if exog_var in col]) > 0:
+                    exog_cols.append(col)
+
+        base_cols = ["panel_effect"] + flag_cols + tuning_cols + exog_cols
+
+        x_train_contribution["base_contr"] = x_train_contribution[base_cols].sum(axis=1)
+        x_train_contribution.drop(columns=base_cols, inplace=True)
+
+        contri_df = pd.DataFrame(x_train_contribution.filter(regex="contr").sum(axis=0))
+        contri_df.reset_index(inplace=True)
+        contri_df.columns = ["Channel", target]
+
+        contri_df[target] = 100 * contri_df[target] / contri_df[target].sum()
+        contri_df["Channel"] = contri_df["Channel"].apply(
+            lambda x: map_channel(x, channels)
+        )
+
+        contri_df["Channel"].replace("base_contr", "base", inplace=True)
+        contribution_df = pd.merge(
+            contribution_df, contri_df, on="Channel", how="outer"
+        )
+    # st.session_state["contribution_df"] = contributions_panel(tuned_model_dict)
+    return contribution_df
+
+
+def create_grouped_bar_plot(contribution_df, contribution_selections):
+    # Extract the 'Channel' names
+    channel_names = contribution_df["Channel"].tolist()
+
+    # Dictionary to store all contributions except 'const' and 'base'
+    all_contributions = {
+        name: [] for name in channel_names if name not in ["const", "base"]
+    }
+
+    # Dictionary to store base sales for each selection
+    base_sales_dict = {}
+
+    # Accumulate contributions for each channel from each selection
+    for selection in contribution_selections:
+        contributions = contribution_df[selection].values.astype(float)
+        base_sales = 0  # Initialize base sales for the current selection
+
+        for channel_name, contribution in zip(channel_names, contributions):
+            if channel_name in all_contributions:
+                all_contributions[channel_name].append(contribution)
+            elif channel_name == "base":
+                base_sales = (
+                    contribution  # Capture base sales for the current selection
+                )
+
+        # Store base sales for each selection
+        base_sales_dict[selection] = base_sales
+
+    # Calculate the average of contributions and sort by this average
+    sorted_channels = sorted(all_contributions.items(), key=lambda x: -np.mean(x[1]))
+    sorted_channel_names = [name for name, _ in sorted_channels]
+    sorted_channel_names = [
+        "Base Sales"
+    ] + sorted_channel_names  # Adding 'Base Sales' at the start
+
+    trace_data = []
+    max_value = 0  # Initialize max_value to find the highest bar for y-axis adjustment
+
+    # Create traces for the grouped bar chart
+    for i, selection in enumerate(contribution_selections):
+        display_name = sorted_channel_names
+        display_contribution = [base_sales_dict[selection]] + [
+            all_contributions[name][i] for name in sorted_channel_names[1:]
+        ]  # Start with base sales for the current selection
+
+        # Generating text labels for each bar
+        text_values = [
+            f"{val}%" for val in np.round(display_contribution, 0).astype(int)
+        ]
+
+        # Find the max value for y-axis calculation
+        max_contribution = max(display_contribution)
+        if max_contribution > max_value:
+            max_value = max_contribution
+
+        # Create a bar trace for each selection
+        trace = go.Bar(
+            x=display_name,
+            y=display_contribution,
+            name=selection,
+            text=text_values,
+            textposition="outside",
+        )
+        trace_data.append(trace)
+
+    # Define layout for the bar chart
+    layout = go.Layout(
+        title="Metrics Contribution by Channel (Train)",
+        xaxis=dict(title="Channel Name"),
+        yaxis=dict(
+            title="Metrics Contribution", range=[0, max_value * 1.2]
+        ),  # Set y-axis 20% higher than the max bar
+        barmode="group",
+        plot_bgcolor="white",
+    )
+
+    # Create the figure with trace data and layout
+    fig = go.Figure(data=trace_data, layout=layout)
+
+    return fig
+
+
+def preprocess_and_plot(contribution_df, contribution_selections):
+    # Extract the 'Channel' names
+    channel_names = contribution_df["Channel"].tolist()
+
+    # Dictionary to store all contributions except 'const' and 'base'
+    all_contributions = {
+        name: [] for name in channel_names if name not in ["const", "base"]
+    }
+
+    # Dictionary to store base sales for each selection
+    base_sales_dict = {}
+
+    # Accumulate contributions for each channel from each selection
+    for selection in contribution_selections:
+        contributions = contribution_df[selection].values.astype(float)
+        base_sales = 0  # Initialize base sales for the current selection
+
+        for channel_name, contribution in zip(channel_names, contributions):
+            if channel_name in all_contributions:
+                all_contributions[channel_name].append(contribution)
+            elif channel_name == "base":
+                base_sales = (
+                    contribution  # Capture base sales for the current selection
+                )
+
+        # Store base sales for each selection
+        base_sales_dict[selection] = base_sales
+
+    # Calculate the average of contributions and sort by this average
+    sorted_channels = sorted(all_contributions.items(), key=lambda x: -np.mean(x[1]))
+    sorted_channel_names = [name for name, _ in sorted_channels]
+    sorted_channel_names = [
+        "Base Sales"
+    ] + sorted_channel_names  # Adding 'Base Sales' at the start
+
+    # Initialize a Plotly figure
+    fig = go.Figure()
+
+    for i, selection in enumerate(contribution_selections):
+        display_name = ["Base Sales"] + sorted_channel_names[
+            1:
+        ]  # Channel names for the plot
+        display_contribution = [
+            base_sales_dict[selection]
+        ]  # Start with base sales for the current selection
+
+        # Append average contributions for other channels
+        for name in sorted_channel_names[1:]:
+            display_contribution.append(all_contributions[name][i])
+
+        # Generating text labels for each bar
+        text_values = [
+            f"{val}%" for val in np.round(display_contribution, 0).astype(int)
+        ]
+
+        # Add a waterfall trace for each selection
+        fig.add_trace(
+            go.Waterfall(
+                orientation="v",
+                measure=["relative"] * len(display_contribution),
+                x=display_name,
+                text=text_values,
+                textposition="outside",
+                y=display_contribution,
+                increasing={"marker": {"color": "green"}},
+                decreasing={"marker": {"color": "red"}},
+                totals={"marker": {"color": "blue"}},
+                name=selection,
+            )
+        )
+
+    # Update layout of the figure
+    fig.update_layout(
+        title="Metrics Contribution by Channel (Train)",
+        xaxis={"title": "Channel Name"},
+        yaxis=dict(title="Metrics Contribution", range=[0, 100 * 1.2]),
+    )
+
+    return fig
+
+
+def selection_change():
+    edited_rows: dict = st.session_state.project_selection["edited_rows"]
+    st.session_state["selected_row_index_gd_table"] = next(iter(edited_rows))
+    st.session_state["gd_table"] = st.session_state["gd_table"].assign(selected=False)
+
+    update_dict = {idx: values for idx, values in edited_rows.items()}
+
+    st.session_state["gd_table"].update(
+        pd.DataFrame.from_dict(update_dict, orient="index")
+    )
+
+
+if "username" not in st.session_state:
+    st.session_state["username"] = None
+
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+try:
+    st.session_state["bin_dict"] = st.session_state["project_dct"]["data_import"][
+        "category_dict"
+    ]  # db
+
+except Exception as e:
+    st.warning("Save atleast one tuned model to proceed")
+    log_message("warning", "No tuned models available", "AI Model Results")
+    st.stop()
+
+
+if "gd_table" not in st.session_state:
+    st.session_state["gd_table"] = pd.DataFrame()
+
+try:
+    if "username" in st.session_state and st.session_state["username"] is not None:
+
+        if (
+            retrieve_pkl_object(
+                st.session_state["project_number"],
+                "Model_Tuning",
+                "tuned_model",
+                schema,
+            )
+            is None
+        ):
+
+            st.error("Please save a tuned model")
+            st.stop()
+
+        if (
+            "session_state_saved"
+            in st.session_state["project_dct"]["model_tuning"].keys()
+            and st.session_state["project_dct"]["model_tuning"]["session_state_saved"]
+            != []
+        ):
+            for key in ["used_response_metrics", "media_data", "bin_dict"]:
+                if key not in st.session_state:
+                    st.session_state[key] = st.session_state["project_dct"][
+                        "model_tuning"
+                    ]["session_state_saved"][key]
+                # st.session_state["bin_dict"] = st.session_state["project_dct"][
+                #     "model_build"
+                # ]["session_state_saved"]["bin_dict"]
+
+        media_data = st.session_state["media_data"]
+
+        # st.write(media_data.columns)
+
+        # set the panel column
+        panel_col = "panel"
+        is_panel = (
+            True if st.session_state["media_data"][panel_col].nunique() > 1 else False
+        )
+        # st.write(is_panel)
+
+        date_col = "date"
+
+        transformed_data = st.session_state["project_dct"]["transformations"][
+            "final_df"
+        ]  # db
+        tuned_model_dict = retrieve_pkl_object(
+            st.session_state["project_number"], "Model_Tuning", "tuned_model", schema
+        )  # db
+
+        feature_set_dct = {
+            key.split("__")[1]: key_dict["feature_set"]
+            for key, key_dict in tuned_model_dict.items()
+        }
+
+        # """ the above part should be modified so that we are fetching features set from the saved model"""
+
+        if "contribution_df" not in st.session_state:
+            st.session_state["contribution_df"] = None
+
+        metrics_table = metrics_df_panel(tuned_model_dict, is_panel)
+
+        cols1 = st.columns([2, 1])
+        with cols1[0]:
+            st.markdown(f"**Welcome {st.session_state['username']}**")
+        with cols1[1]:
+            st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+        st.title("AI Model Validation")
+
+        st.header("Contribution Overview")
+
+        # Get list of response metrics
+        st.session_state["used_response_metrics"] = list(
+            set([model.split("__")[1] for model in tuned_model_dict.keys()])
+        )
+        options = st.session_state["used_response_metrics"]
+
+        if len(options) == 0:
+            st.error("Please save and tune a model")
+            st.stop()
+        options = [
+            opt.lower()
+            .replace(" ", "_")
+            .replace("-", "")
+            .replace(":", "")
+            .replace("__", "_")
+            for opt in options
+        ]
+
+        default_options = (
+            st.session_state["project_dct"]["saved_model_results"].get(
+                "selected_options"
+            )
+            if st.session_state["project_dct"]["saved_model_results"].get(
+                "selected_options"
+            )
+            is not None
+            else [options[-1]]
+        )
+        for i in default_options:
+            if i not in options:
+                # st.write(i)
+                default_options.remove(i)
+
+        def remove_response_metric(name):
+            # Convert the name to a lowercase string and remove any leading or trailing spaces
+            name_str = str(name).lower().strip()
+
+            # Check if the name starts with "response metric" or "response_metric"
+            if name_str.startswith("response metric"):
+                return name[len("response metric") :].replace("_", " ").strip().title()
+            elif name_str.startswith("response_metric"):
+                return name[len("response_metric") :].replace("_", " ").strip().title()
+            else:
+                return name
+
+        contribution_selections = st.multiselect(
+            "Select the Response Metrics to compare contributions",
+            options,
+            default=default_options,
+            format_func=remove_response_metric,
+        )
+        trace_data = []
+
+        if is_panel:
+            st.session_state["contribution_df"] = contributions_panel(tuned_model_dict)
+
+        else:
+            st.session_state["contribution_df"] = contributions_nonpanel(
+                tuned_model_dict
+            )
+
+        # st.write(st.session_state["contribution_df"].columns)
+        # for selection in contribution_selections:
+
+        #     trace = go.Bar(
+        #         x=st.session_state["contribution_df"]["Channel"],
+        #         y=st.session_state["contribution_df"][selection],
+        #         name=selection,
+        #         text=np.round(st.session_state["contribution_df"][selection], 0)
+        #         .astype(int)
+        #         .astype(str)
+        #         + "%",
+        #         textposition="outside",
+        #     )
+        #     trace_data.append(trace)
+
+        # layout = go.Layout(
+        #     title="Metrics Contribution by Channel",
+        #     xaxis=dict(title="Channel Name"),
+        #     yaxis=dict(title="Metrics Contribution"),
+        #     barmode="group",
+        # )
+        # fig = go.Figure(data=trace_data, layout=layout)
+        # st.plotly_chart(fig, use_container_width=True)
+
+        # Display the chart in Streamlit
+        st.plotly_chart(
+            create_grouped_bar_plot(
+                st.session_state["contribution_df"], contribution_selections
+            ),
+            use_container_width=True,
+        )
+
+        ############################################ Waterfall Chart ############################################
+
+        import plotly.graph_objects as go
+
+        st.plotly_chart(
+            preprocess_and_plot(
+                st.session_state["contribution_df"], contribution_selections
+            ),
+            use_container_width=True,
+        )
+
+        ############################################ Waterfall Chart ############################################
+        st.header("Analysis of Models Result")
+        gd_table = metrics_table.iloc[:, :-2]
+        target_column = gd_table.at[0, "Model"]  # sprint8
+        st.session_state["gd_table"] = gd_table
+
+        with st.container():
+            table = st.data_editor(
+                st.session_state["gd_table"],
+                hide_index=True,
+                # on_change=selection_change,
+                key="project_selection",
+                use_container_width=True,
+            )
+
+        target_column = st.selectbox(
+            "Select a Model to analyse its results",
+            options=st.session_state.used_response_metrics,
+            placeholder=options[0],
+        )
+        feature_set = feature_set_dct[target_column]
+
+        model = metrics_table[metrics_table["Model"] == target_column][
+            "Model_object"
+        ].iloc[0]
+        target = metrics_table[metrics_table["Model"] == target_column]["Model"].iloc[0]
+        st.header("Model Summary")
+        st.write(model.summary())
+
+        sel_dict = tuned_model_dict[
+            [k for k in tuned_model_dict.keys() if k.split("__")[1] == target][0]
+        ]
+
+        feature_set = sel_dict["feature_set"]
+        X_train = sel_dict["X_train_tuned"]
+        y_train = X_train[target]
+
+        if is_panel:
+            random_effects = get_random_effects(media_data, panel_col, model)
+            pred = mdf_predict(X_train, model, random_effects)["pred"]
+        else:
+            pred = model.predict(X_train[feature_set])
+
+        X_test = sel_dict["X_test_tuned"]
+        y_test = X_test[target]
+        if is_panel:
+            predtest = mdf_predict(X_test, model, random_effects)["pred"]
+        else:
+            predtest = model.predict(X_test[feature_set])
+
+        metrics_table_train, _, fig_train = plot_actual_vs_predicted(
+            X_train[date_col],
+            y_train,
+            pred,
+            model,
+            target_column=target,
+            flag=None,
+            repeat_all_years=False,
+            is_panel=is_panel,
+        )
+
+        metrics_table_test, _, fig_test = plot_actual_vs_predicted(
+            X_test[date_col],
+            y_test,
+            predtest,
+            model,
+            target_column=target,
+            flag=None,
+            repeat_all_years=False,
+            is_panel=is_panel,
+        )
+
+        metrics_table_train = metrics_table_train.set_index("Metric").transpose()
+        metrics_table_train.index = ["Train"]
+        metrics_table_test = metrics_table_test.set_index("Metric").transpose()
+        metrics_table_test.index = ["Test"]
+        metrics_table = np.round(
+            pd.concat([metrics_table_train, metrics_table_test]), 2
+        )
+
+        st.markdown("Result Overview")
+        st.dataframe(np.round(metrics_table, 2), use_container_width=True)
+
+        st.header("Model Accuracy")
+        st.subheader("Actual vs Predicted Plot (Train)")
+
+        st.plotly_chart(fig_train, use_container_width=True)
+        st.subheader("Actual vs Predicted Plot (Test)")
+        st.plotly_chart(fig_test, use_container_width=True)
+
+        st.markdown("## Residual Analysis (Train)")
+        columns = st.columns(2)
+
+        Xtrain1 = X_train.copy()
+        with columns[0]:
+            fig = plot_residual_predicted(y_train, pred, Xtrain1)
+            st.plotly_chart(fig)
+
+        with columns[1]:
+            st.empty()
+            fig = qqplot(y_train, pred)
+            st.plotly_chart(fig)
+
+        with columns[0]:
+            fig = residual_distribution(y_train, pred)
+            st.pyplot(fig)
+
+        if st.button("Save this session", use_container_width=True):
+            project_dct_pkl = pickle.dumps(st.session_state["project_dct"])
+
+            update_db(
+                st.session_state["project_number"],
+                "AI_Model_Results",
+                "project_dct",
+                project_dct_pkl,
+                schema,
+                # resp_mtrc=None,
+            )  # db
+
+            log_message("info", "Session saved!", "AI Model Results")
+            st.success("Session Saved!")
+except:
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+    log_message("error", f"Error: {error_message}", "AI Model Results")
+    st.warning("An error occured, please try again", icon="⚠️")

pages/7_AI_Model_Media_Performance.py

@@ -0,0 +1,733 @@
+import streamlit as st
+import pandas as pd
+from sklearn.preprocessing import MinMaxScaler
+import pickle
+import os
+from utilities_with_panel import load_local_css, set_header
+import yaml
+from yaml import SafeLoader
+import sqlite3
+from datetime import timedelta
+from utilities import (
+    set_header,
+    load_local_css,
+    update_db,
+    project_selection,
+    retrieve_pkl_object,
+)
+from utilities_with_panel import (
+    overview_test_data_prep_panel,
+    overview_test_data_prep_nonpanel,
+    initialize_data_cmp,
+    create_channel_summary,
+    create_contribution_pie,
+    create_contribuion_stacked_plot,
+    create_channel_spends_sales_plot,
+    format_numbers,
+    channel_name_formating,
+)
+from log_application import log_message
+import sys, traceback
+from post_gres_cred import db_cred
+
+st.set_page_config(layout="wide")
+load_local_css("styles.css")
+set_header()
+
+
+schema = db_cred["schema"]
+
+if "username" not in st.session_state:
+    st.session_state["username"] = None
+
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+tuned_model = retrieve_pkl_object(
+    st.session_state["project_number"], "Model_Tuning", "tuned_model", schema
+)
+
+if tuned_model is None:
+    st.error("Please save a tuned model")
+    st.stop()
+
+if (
+    "session_state_saved" in st.session_state["project_dct"]["model_tuning"].keys()
+    and st.session_state["project_dct"]["model_tuning"]["session_state_saved"] != []
+):
+    for key in ["used_response_metrics", "media_data", "bin_dict"]:
+        if key not in st.session_state:
+            st.session_state[key] = st.session_state["project_dct"]["model_tuning"][
+                "session_state_saved"
+            ][key]
+
+
+## DEFINE ALL FUNCTIONS
+def get_random_effects(media_data, panel_col, mdf):
+    random_eff_df = pd.DataFrame(columns=[panel_col, "random_effect"])
+    for i, market in enumerate(media_data[panel_col].unique()):
+        print(i, end="\r")
+        intercept = mdf.random_effects[market].values[0]
+        random_eff_df.loc[i, "random_effect"] = intercept
+        random_eff_df.loc[i, panel_col] = market
+
+    return random_eff_df
+
+
+def process_train_and_test(train, test, features, panel_col, target_col):
+    X1 = train[features]
+
+    ss = MinMaxScaler()
+    X1 = pd.DataFrame(ss.fit_transform(X1), columns=X1.columns)
+
+    X1[panel_col] = train[panel_col]
+    X1[target_col] = train[target_col]
+
+    if test is not None:
+        X2 = test[features]
+        X2 = pd.DataFrame(ss.transform(X2), columns=X2.columns)
+        X2[panel_col] = test[panel_col]
+        X2[target_col] = test[target_col]
+        return X1, X2
+    return X1
+
+
+def mdf_predict(X_df, mdf, random_eff_df):
+    X = X_df.copy()
+    X = pd.merge(
+        X,
+        random_eff_df[[panel_col, "random_effect"]],
+        on=panel_col,
+        how="left",
+    )
+    X["pred_fixed_effect"] = mdf.predict(X)
+
+    X["pred"] = X["pred_fixed_effect"] + X["random_effect"]
+    X.drop(columns=["pred_fixed_effect", "random_effect"], inplace=True)
+
+    return X
+
+
+try:
+    if "username" in st.session_state and st.session_state["username"] is not None:
+
+        # conn = sqlite3.connect(
+        #     r"DB/User.db", check_same_thread=False
+        # )  # connection with sql db
+        # c = conn.cursor()
+
+        tuned_model = retrieve_pkl_object(
+            st.session_state["project_number"], "Model_Tuning", "tuned_model", schema
+        )
+
+        if tuned_model is None:
+            st.error("Please save a tuned model")
+            st.stop()
+
+        if (
+            "session_state_saved"
+            in st.session_state["project_dct"]["model_tuning"].keys()
+            and st.session_state["project_dct"]["model_tuning"]["session_state_saved"]
+            != []
+        ):
+            for key in [
+                "used_response_metrics",
+                "is_tuned_model",
+                "media_data",
+                "X_test_spends",
+                "spends_data",
+            ]:
+                st.session_state[key] = st.session_state["project_dct"]["model_tuning"][
+                    "session_state_saved"
+                ][key]
+        elif (
+            "session_state_saved"
+            in st.session_state["project_dct"]["model_build"].keys()
+            and st.session_state["project_dct"]["model_build"]["session_state_saved"]
+            != []
+        ):
+            for key in [
+                "used_response_metrics",
+                "date",
+                "saved_model_names",
+                "media_data",
+                "X_test_spends",
+            ]:
+                st.session_state[key] = st.session_state["project_dct"]["model_build"][
+                    "session_state_saved"
+                ][key]
+        else:
+            st.error("Please tune a model first")
+        st.session_state["bin_dict"] = st.session_state["project_dct"]["model_build"][
+            "session_state_saved"
+        ]["bin_dict"]
+        st.session_state["media_data"].columns = [
+            c.lower() for c in st.session_state["media_data"].columns
+        ]
+
+        # with open(
+        #     os.path.join(st.session_state["project_path"], "data_import.pkl"),
+        #     "rb",
+        # ) as f:
+        #     data = pickle.load(f)
+
+        # # Accessing the loaded objects
+
+        # st.session_state["orig_media_data"] = data["final_df"]
+
+        st.session_state["orig_media_data"] = st.session_state["project_dct"][
+            "data_import"
+        ][
+            "imputed_tool_df"
+        ].copy()  # db
+        st.session_state["channels"] = st.session_state["project_dct"]["data_import"][
+            "group_dict"
+        ].copy()
+        # target='Revenue'
+
+        # set the panel column
+        panel_col = "panel"
+        is_panel = (
+            True if st.session_state["media_data"][panel_col].nunique() > 1 else False
+        )
+
+        date_col = "date"
+
+        cols1 = st.columns([2, 1])
+        with cols1[0]:
+            st.markdown(f"**Welcome {st.session_state['username']}**")
+        with cols1[1]:
+            st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+        st.title("AI Model Media Performance")
+
+        def remove_response_metric(name):
+            # Convert the name to a lowercase string and remove any leading or trailing spaces
+            name_str = str(name).lower().strip()
+
+            # Check if the name starts with "response metric" or "response_metric"
+            if name_str.startswith("response metric"):
+                return name[len("response metric") :].replace("_", " ").strip().title()
+            elif name_str.startswith("response_metric"):
+                return name[len("response_metric") :].replace("_", " ").strip().title()
+            else:
+                return name
+
+        sel_target_col = st.selectbox(
+            "Select the response metric",
+            st.session_state["used_response_metrics"],
+            format_func=remove_response_metric,
+        )
+        sel_target_col_frmttd = sel_target_col.replace("_", " ").replace("-", " ")
+        sel_target_col_frmttd = sel_target_col_frmttd.title()
+        target_col = (
+            sel_target_col.lower()
+            .replace(" ", "_")
+            .replace("-", "")
+            .replace(":", "")
+            .replace("__", "_")
+        )
+        target = sel_target_col
+
+        # Contribution
+        if is_panel:
+            # read tuned mixedLM model
+            if st.session_state["is_tuned_model"][target_col] == True:
+
+                model_dict = retrieve_pkl_object(
+                    st.session_state["project_number"],
+                    "Model_Tuning",
+                    "tuned_model",
+                    schema,
+                )  # db
+
+                saved_models = list(model_dict.keys())
+                required_saved_models = [
+                    m.split("__")[0]
+                    for m in saved_models
+                    if m.split("__")[1] == target_col
+                ]
+
+                sel_model = required_saved_models[
+                    0
+                ]  # only 1 tuned model available per resp metric
+
+                sel_model_dict = model_dict[sel_model + "__" + target_col]
+
+                model = sel_model_dict["Model_object"]
+                X_train = sel_model_dict["X_train_tuned"]
+                X_test = sel_model_dict["X_test_tuned"]
+                best_feature_set = sel_model_dict["feature_set"]
+
+            # Calculate contributions
+
+            st.session_state["orig_media_data"].columns = [
+                col.lower()
+                .replace(".", "_")
+                .replace("@", "_")
+                .replace(" ", "_")
+                .replace("-", "")
+                .replace(":", "")
+                .replace("__", "_")
+                for col in st.session_state["orig_media_data"].columns
+            ]
+
+            media_data = st.session_state["media_data"]
+
+            contri_df = pd.DataFrame()
+
+            y = []
+            y_pred = []
+
+            random_eff_df = get_random_effects(media_data, panel_col, model)
+            random_eff_df["fixed_effect"] = model.fe_params["Intercept"]
+            random_eff_df["panel_effect"] = (
+                random_eff_df["random_effect"] + random_eff_df["fixed_effect"]
+            )
+
+            coef_df = pd.DataFrame(model.fe_params)
+            coef_df.reset_index(inplace=True)
+            coef_df.columns = ["feature", "coef"]
+
+            x_train_contribution = X_train.copy()
+            x_test_contribution = X_test.copy()
+
+            # preprocessing not needed since X_train is already preprocessed
+            # X1, X2 = process_train_and_test(x_train_contribution, x_test_contribution, best_feature_set, panel_col, target_col)
+            # x_train_contribution[best_feature_set] = X1[best_feature_set]
+            # x_test_contribution[best_feature_set] = X2[best_feature_set]
+
+            x_train_contribution = mdf_predict(
+                x_train_contribution, model, random_eff_df
+            )
+            x_test_contribution = mdf_predict(x_test_contribution, model, random_eff_df)
+
+            x_train_contribution = pd.merge(
+                x_train_contribution,
+                random_eff_df[[panel_col, "panel_effect"]],
+                on=panel_col,
+                how="left",
+            )
+            x_test_contribution = pd.merge(
+                x_test_contribution,
+                random_eff_df[[panel_col, "panel_effect"]],
+                on=panel_col,
+                how="left",
+            )
+
+            for i in range(len(coef_df))[1:]:
+                coef = coef_df.loc[i, "coef"]
+                col = coef_df.loc[i, "feature"]
+                if col.lower() != "intercept":
+                    x_train_contribution[str(col) + "_contr"] = (
+                        coef * x_train_contribution[col]
+                    )
+                    x_test_contribution[str(col) + "_contr"] = (
+                        coef * x_train_contribution[col]
+                    )
+
+            tuning_cols = [
+                c
+                for c in x_train_contribution.filter(regex="contr").columns
+                if c
+                in [
+                    "day_of_week_contr",
+                    "Trend_contr",
+                    "sine_wave_contr",
+                    "cosine_wave_contr",
+                ]
+            ]
+            flag_cols = [
+                c
+                for c in x_train_contribution.filter(regex="contr").columns
+                if "_flag" in c
+            ]
+
+            # add exogenous contribution to base
+            all_exog_vars = st.session_state["bin_dict"]["Exogenous"]
+            all_exog_vars = [
+                var.lower()
+                .replace(".", "_")
+                .replace("@", "_")
+                .replace(" ", "_")
+                .replace("-", "")
+                .replace(":", "")
+                .replace("__", "_")
+                for var in all_exog_vars
+            ]
+            exog_cols = []
+            if len(all_exog_vars) > 0:
+                for col in x_train_contribution.filter(regex="contr").columns:
+                    if (
+                        len([exog_var for exog_var in all_exog_vars if exog_var in col])
+                        > 0
+                    ):
+                        exog_cols.append(col)
+
+            base_cols = ["panel_effect"] + flag_cols + tuning_cols + exog_cols
+
+            x_train_contribution["base_contr"] = x_train_contribution[base_cols].sum(
+                axis=1
+            )
+            x_train_contribution.drop(columns=base_cols, inplace=True)
+            x_test_contribution["base_contr"] = x_test_contribution[base_cols].sum(
+                axis=1
+            )
+            x_test_contribution.drop(columns=base_cols, inplace=True)
+
+            overall_contributions = pd.concat(
+                [x_train_contribution, x_test_contribution]
+            ).reset_index(drop=True)
+
+            overview_test_data_prep_panel(
+                overall_contributions,
+                st.session_state["orig_media_data"],
+                st.session_state["spends_data"],
+                date_col,
+                panel_col,
+                target_col,
+            )
+
+        else:  # NON PANEL
+            if st.session_state["is_tuned_model"][target_col] == True:  # Sprint4
+                # with open(
+                #     os.path.join(st.session_state["project_path"], "tuned_model.pkl"),
+                #     "rb",
+                # ) as file:
+                #     model_dict = pickle.load(file)
+
+                model_dict = retrieve_pkl_object(
+                    st.session_state["project_number"],
+                    "Model_Tuning",
+                    "tuned_model",
+                    schema,
+                )  # db
+
+                saved_models = list(model_dict.keys())
+                required_saved_models = [
+                    m.split("__")[0]
+                    for m in saved_models
+                    if m.split("__")[1] == target_col
+                ]
+
+                sel_model = required_saved_models[
+                    0
+                ]  # only 1 tuned model available per resp metric
+                sel_model_dict = model_dict[sel_model + "__" + target_col]
+
+                model = sel_model_dict["Model_object"]
+                X_train = sel_model_dict["X_train_tuned"]
+                X_test = sel_model_dict["X_test_tuned"]
+                best_feature_set = sel_model_dict["feature_set"]
+
+            x_train_contribution = X_train.copy()
+            x_test_contribution = X_test.copy()
+
+            x_train_contribution["pred"] = model.predict(
+                x_train_contribution[best_feature_set]
+            )
+            x_test_contribution["pred"] = model.predict(
+                x_test_contribution[best_feature_set]
+            )
+
+            coef_df = pd.DataFrame(model.params)
+            coef_df.reset_index(inplace=True)
+            coef_df.columns = ["feature", "coef"]
+
+            # st.write(coef_df)
+            for i in range(len(coef_df)):
+                coef = coef_df.loc[i, "coef"]
+                col = coef_df.loc[i, "feature"]
+                if col != "const":
+                    x_train_contribution[str(col) + "_contr"] = (
+                        coef * x_train_contribution[col]
+                    )
+                    x_test_contribution[str(col) + "_contr"] = (
+                        coef * x_test_contribution[col]
+                    )
+                else:
+                    x_train_contribution["const"] = coef
+                    x_test_contribution["const"] = coef
+
+            tuning_cols = [
+                c
+                for c in x_train_contribution.filter(regex="contr").columns
+                if c
+                in [
+                    "day_of_week_contr",
+                    "Trend_contr",
+                    "sine_wave_contr",
+                    "cosine_wave_contr",
+                ]
+            ]
+            flag_cols = [
+                c
+                for c in x_train_contribution.filter(regex="contr").columns
+                if "_flag" in c
+            ]
+
+            # add exogenous contribution to base
+            all_exog_vars = st.session_state["bin_dict"]["Exogenous"]
+            all_exog_vars = [
+                var.lower()
+                .replace(".", "_")
+                .replace("@", "_")
+                .replace(" ", "_")
+                .replace("-", "")
+                .replace(":", "")
+                .replace("__", "_")
+                for var in all_exog_vars
+            ]
+            exog_cols = []
+            if len(all_exog_vars) > 0:
+                for col in x_train_contribution.filter(regex="contr").columns:
+                    if (
+                        len([exog_var for exog_var in all_exog_vars if exog_var in col])
+                        > 0
+                    ):
+                        exog_cols.append(col)
+
+            base_cols = ["const"] + flag_cols + tuning_cols + exog_cols
+            # st.write(base_cols)
+            x_train_contribution["base_contr"] = x_train_contribution[base_cols].sum(
+                axis=1
+            )
+            x_train_contribution.drop(columns=base_cols, inplace=True)
+
+            x_test_contribution["base_contr"] = x_test_contribution[base_cols].sum(
+                axis=1
+            )
+            x_test_contribution.drop(columns=base_cols, inplace=True)
+            # x_test_contribution.to_csv("Test/test_contr.csv", index=False)
+
+            overall_contributions = pd.concat(
+                [x_train_contribution, x_test_contribution]
+            ).reset_index(drop=True)
+            # overall_contributions.to_csv("Test/overall_contributions.csv", index=False)
+
+            overview_test_data_prep_nonpanel(
+                overall_contributions,
+                st.session_state["orig_media_data"].copy(),
+                st.session_state["spends_data"].copy(),
+                date_col,
+                target_col,
+            )
+        # for k, v in st.session_sta
+        # te.items():
+
+        #     if k not in ['logout', 'login','config'] and not k.startswith('FormSubmitter'):
+        #         st.session_state[k] = v
+
+        # authenticator = st.session_state.get('authenticator')
+
+        # if authenticator is None:
+        #     authenticator = load_authenticator()
+
+        # name, authentication_status, username = authenticator.login('Login', 'main')
+        # auth_status = st.session_state['authentication_status']
+
+        # if auth_status:
+        #     authenticator.logout('Logout', 'main')
+
+        #     is_state_initiaized = st.session_state.get('initialized',False)
+        #     if not is_state_initiaized:
+
+        min_date = X_train[date_col].min().date()
+        max_date = X_test[date_col].max().date()
+        if "media_performance" not in st.session_state["project_dct"]:
+            st.session_state["project_dct"]["media_performance"] = {
+                "start_date": None,
+                "end_date": None,
+            }
+
+        start_default = st.session_state["project_dct"]["media_performance"].get(
+            "start_date", None
+        )
+        start_default = start_default if start_default is not None else min_date
+        start_default = start_default if start_default > min_date else min_date
+        start_default = start_default if start_default < max_date else min_date
+
+        end_default = st.session_state["project_dct"]["media_performance"].get(
+            "end_date", None
+        )
+        end_default = end_default if end_default is not None else max_date
+        end_default = end_default if end_default > min_date else max_date
+        end_default = end_default if end_default < max_date else max_date
+
+        st.write("Select a timeline for analysis")
+        date_columns = st.columns(2)
+
+        with date_columns[0]:
+            start_date = st.date_input(
+                "Select Start Date",
+                start_default,
+                min_value=min_date,
+                max_value=max_date,
+            )
+            if (start_date < min_date) or (start_date > max_date):
+                st.error("Please select dates in the range of the dates in the data")
+                st.stop()
+        end_default = (
+            end_default if end_default > start_date + timedelta(days=1) else max_date
+        )
+        with date_columns[1]:
+            end_default = (
+                end_default
+                if pd.Timestamp(end_default) >= pd.Timestamp(start_date)
+                else start_date
+            )
+
+            end_date = st.date_input(
+                "Select End Date",
+                end_default,
+                min_value=start_date + timedelta(days=1),
+                max_value=max_date,
+            )
+        if (
+            (start_date < min_date)
+            or (end_date < min_date)
+            or (start_date > max_date)
+            or (end_date > max_date)
+        ):
+            st.error("Please select dates in the range of the dates in the data")
+            st.stop()
+        if end_date < start_date + timedelta(days=1):
+            st.error("Please select end date after start date")
+            st.stop()
+
+        st.session_state["project_dct"]["media_performance"]["start_date"] = start_date
+        st.session_state["project_dct"]["media_performance"]["end_date"] = end_date
+
+        st.header("Overview of Previous Media Spend")
+
+        initialize_data_cmp(target_col, is_panel, panel_col, start_date, end_date)
+        scenario = st.session_state["scenario"]
+        raw_df = st.session_state["raw_df"]
+
+        columns = st.columns(2)
+
+        with columns[0]:
+            st.metric(
+                label="Media Spend",
+                value=format_numbers(float(scenario.actual_total_spends)),
+            )
+        ###print(f"##################### {scenario.actual_total_sales} ##################")
+        with columns[1]:
+            st.metric(
+                label=sel_target_col_frmttd,
+                value=format_numbers(
+                    float(scenario.actual_total_sales), include_indicator=False
+                ),
+            )
+
+        actual_summary_df = create_channel_summary(scenario, sel_target_col_frmttd)
+        actual_summary_df["Channel"] = actual_summary_df["Channel"].apply(
+            channel_name_formating
+        )
+
+        columns = st.columns((3, 1))
+        with columns[0]:
+            with st.expander("Channel wise overview"):
+                st.markdown(
+                    actual_summary_df.style.set_table_styles(
+                        [
+                            {
+                                "selector": "th",
+                                "props": [("background-color", "#f6dcc7")],
+                            },
+                            {
+                                "selector": "tr:nth-child(even)",
+                                "props": [("background-color", "#f6dcc7")],
+                            },
+                        ]
+                    ).to_html(),
+                    unsafe_allow_html=True,
+                )
+
+        st.markdown("<hr>", unsafe_allow_html=True)
+        ##############################
+
+        st.plotly_chart(
+            create_contribution_pie(scenario, sel_target_col_frmttd),
+            use_container_width=True,
+        )
+        st.markdown("<hr>", unsafe_allow_html=True)
+
+        ################################3
+        st.plotly_chart(
+            create_contribuion_stacked_plot(scenario, sel_target_col_frmttd),
+            use_container_width=True,
+        )
+        st.markdown("<hr>", unsafe_allow_html=True)
+        #######################################
+
+        selected_channel_name = st.selectbox(
+            "Channel",
+            st.session_state["channels_list"] + ["non media"],
+            format_func=channel_name_formating,
+        )
+        selected_channel = scenario.channels.get(selected_channel_name, None)
+
+        st.plotly_chart(
+            create_channel_spends_sales_plot(selected_channel, sel_target_col_frmttd),
+            use_container_width=True,
+        )
+
+        st.markdown("<hr>", unsafe_allow_html=True)
+
+        if st.button("Save this session", use_container_width=True):
+
+            project_dct_pkl = pickle.dumps(st.session_state["project_dct"])
+
+            update_db(
+                st.session_state["project_number"],
+                "Current_Media_Performance",
+                "project_dct",
+                project_dct_pkl,
+                schema,
+                # resp_mtrc=None,
+            )  # db
+
+            st.success("Session Saved!")
+
+        # Remove "response_metric_" from the start and "_total" from the end
+        if str(target_col).startswith("response_metric_"):
+            target_col = target_col.replace("response_metric_", "", 1)
+
+        # Remove the last 6 characters (length of "_total")
+        if str(target_col).endswith("_total"):
+            target_col = target_col[:-6]
+
+        if (
+            st.session_state["project_dct"]["current_media_performance"][
+                "model_outputs"
+            ][target_col]
+            is not None
+        ):
+            if (
+                len(
+                    st.session_state["project_dct"]["current_media_performance"][
+                        "model_outputs"
+                    ][target_col]["contribution_data"]
+                )
+                > 0
+            ):
+                st.download_button(
+                    label="Download Contribution File",
+                    data=st.session_state["project_dct"]["current_media_performance"][
+                        "model_outputs"
+                    ][target_col]["contribution_data"].to_csv(),
+                    file_name="contributions.csv",
+                    key="dwnld_contr",
+                )
+except:
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+    log_message("error", f"Error: {error_message}", "Current Media Performance")
+    st.warning("An error occured, please try again", icon="⚠️")

pages/8_Response_Curves.py

@@ -0,0 +1,530 @@
+import streamlit as st
+
+st.set_page_config(
+    page_title="Response Curves",
+    page_icon="⚖️",
+    layout="wide",
+    initial_sidebar_state="collapsed",
+)
+
+# Disable +/- for number input
+st.markdown(
+    """
+<style>
+    button.step-up {display: none;}
+    button.step-down {display: none;}
+    div[data-baseweb] {border-radius: 4px;}
+</style>""",
+    unsafe_allow_html=True,
+)
+
+import sys
+import json
+import pickle
+import traceback
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from post_gres_cred import db_cred
+from sklearn.metrics import r2_score
+from log_application import log_message
+from utilities import project_selection, update_db, set_header, load_local_css
+from utilities import (
+    get_panels_names,
+    get_metrics_names,
+    name_formating,
+    generate_rcs_data,
+    load_rcs_metadata_files,
+)
+
+schema = db_cred["schema"]
+load_local_css("styles.css")
+set_header()
+
+
+# Initialize project name session state
+if "project_name" not in st.session_state:
+    st.session_state["project_name"] = None
+
+# Fetch project dictionary
+if "project_dct" not in st.session_state:
+    project_selection()
+    st.stop()
+
+# Display Username and Project Name
+if "username" in st.session_state and st.session_state["username"] is not None:
+
+    cols1 = st.columns([2, 1])
+
+    with cols1[0]:
+        st.markdown(f"**Welcome {st.session_state['username']}**")
+    with cols1[1]:
+        st.markdown(f"**Current Project: {st.session_state['project_name']}**")
+
+
+# Function to build s curve
+def s_curve(x, K, b, a, x0):
+    return K / (1 + b * np.exp(-a * (x - x0)))
+
+
+# Function to update the RCS parameters in the modified RCS metadata data
+def modify_rcs_parameters(metrics_selected, panel_selected, channel_selected):
+    # Define unique keys for each parameter based on the selection
+    K_key = f"K_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    b_key = f"b_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    a_key = f"a_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    x0_key = f"x0_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+
+    # Retrieve the updated parameters from session state
+    K_updated, b_updated, a_updated, x0_updated = (
+        st.session_state[K_key],
+        st.session_state[b_key],
+        st.session_state[a_key],
+        st.session_state[x0_key],
+    )
+
+    # Load the existing modified RCS data
+    rcs_data_modified = st.session_state["project_dct"]["response_curves"][
+        "modified_metadata_file"
+    ]
+
+    # Update the RCS parameters for the selected metric and panel
+    rcs_data_modified[metrics_selected][panel_selected][channel_selected] = {
+        "K": K_updated,
+        "b": b_updated,
+        "a": a_updated,
+        "x0": x0_updated,
+    }
+
+
+# Function to reset the parameters to their default values
+def reset_parameters(
+    metrics_selected, panel_selected, channel_selected, original_channel_data
+):
+    # Define unique keys for each parameter based on the selection
+    K_key = f"K_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    b_key = f"b_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    a_key = f"a_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    x0_key = f"x0_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+
+    # Reset session state values to original data
+    del st.session_state[K_key]
+    del st.session_state[b_key]
+    del st.session_state[a_key]
+    del st.session_state[x0_key]
+
+    # Reset the modified metadata file with original parameters
+    rcs_data_modified = st.session_state["project_dct"]["response_curves"][
+        "modified_metadata_file"
+    ]
+
+    # Update the parameters in the modified data to the original values
+    rcs_data_modified[metrics_selected][panel_selected][channel_selected] = {
+        "K": original_channel_data["K"],
+        "b": original_channel_data["b"],
+        "a": original_channel_data["a"],
+        "x0": original_channel_data["x0"],
+    }
+
+    # Update the modified metadata
+    st.session_state["project_dct"]["response_curves"][
+        "modified_metadata_file"
+    ] = rcs_data_modified
+
+
+# Function to generate updated RCS parameter DataFrame
+@st.cache_data(show_spinner=False)
+def updated_parm_gen(original_data, modified_data, metrics_selected, panel_selected):
+    # Retrieve the data for the selected metric and panel
+    original_data_selection = original_data[metrics_selected][panel_selected]
+    modified_data_selection = modified_data[metrics_selected][panel_selected]
+
+    # Initialize an empty list to hold the data for the DataFrame
+    data = []
+
+    # Iterate through each channel in the selected metric and panel
+    for channel in original_data_selection:
+        # Extract original parameters
+        K_o, b_o, a_o, x0_o = (
+            original_data_selection[channel]["K"],
+            original_data_selection[channel]["b"],
+            original_data_selection[channel]["a"],
+            original_data_selection[channel]["x0"],
+        )
+        # Extract modified parameters
+        K_m, b_m, a_m, x0_m = (
+            modified_data_selection[channel]["K"],
+            modified_data_selection[channel]["b"],
+            modified_data_selection[channel]["a"],
+            modified_data_selection[channel]["x0"],
+        )
+
+        # Check if any parameters differ
+        if (K_o != K_m) or (b_o != b_m) or (a_o != a_m) or (x0_o != x0_m):
+            # Append the data to the list only if there is a difference
+            data.append(
+                {
+                    "Metric": name_formating(metrics_selected),
+                    "Panel": name_formating(panel_selected),
+                    "Channel": name_formating(channel),
+                    "K (Original)": K_o,
+                    "b (Original)": b_o,
+                    "a (Original)": a_o,
+                    "x0 (Original)": x0_o,
+                    "K (Modified)": K_m,
+                    "b (Modified)": b_m,
+                    "a (Modified)": a_m,
+                    "x0 (Modified)": x0_m,
+                }
+            )
+
+    # Create a DataFrame from the collected data
+    df = pd.DataFrame(data)
+
+    return df
+
+
+# Function to create JSON file for RCS data
+@st.cache_data(show_spinner=False)
+def create_json_file():
+    return json.dumps(
+        st.session_state["project_dct"]["response_curves"]["modified_metadata_file"],
+        indent=4,
+    )
+
+
+try:
+    # Page Title
+    st.title("Response Curves")
+
+    # Retrieve the list of all metric names from the specified directory
+    metrics_list = get_metrics_names()
+
+    # Check if there are any metrics available in the metrics list
+    if not metrics_list:
+        # Display a warning message to the user if no metrics are found
+        st.warning(
+            "Please tune at least one model to generate response curves data.",
+            icon="⚠️",
+        )
+
+        # Log message
+        log_message(
+            "warning",
+            "Please tune at least one model to generate response curves data.",
+            "Response Curves",
+        )
+
+        # Stop further execution as there is no data to process
+        st.stop()
+
+    # Widget columns
+    metric_col, channel_col, panel_col, save_progress_col = st.columns(4)
+
+    # Metrics Selection
+    metrics_selected = metric_col.selectbox(
+        "Response Metrics",
+        sorted(metrics_list),
+        format_func=name_formating,
+        key="response_metrics_selectbox",
+        index=0,
+    )
+
+    # Retrieve the list of all panel names for specified Metrics
+    panel_list = get_panels_names(metrics_selected)
+
+    # Panel Selection
+    panel_selected = panel_col.selectbox(
+        "Panel",
+        sorted(panel_list),
+        format_func=name_formating,
+        key="panel_selected_selectbox",
+        index=0,
+    )
+
+    # Save Progress
+    with save_progress_col:
+        st.write("####")  # Padding
+        save_progress_placeholder = st.empty()
+
+    # Placeholder to display message and spinner
+    message_spinner_placeholder = st.container()
+
+    # Save page progress
+    with message_spinner_placeholder, st.spinner("Saving Progress ..."):
+        if save_progress_placeholder.button("Save Progress", use_container_width=True):
+            # Update DB
+            update_db(
+                prj_id=st.session_state["project_number"],
+                page_nam="Response Curves",
+                file_nam="project_dct",
+                pkl_obj=pickle.dumps(st.session_state["project_dct"]),
+                schema=schema,
+            )
+
+            # Store the message details in session state
+            message_spinner_placeholder.success(
+                "Progress saved successfully!", icon="💾"
+            )
+            st.toast("Progress saved successfully!", icon="💾")
+
+            # Log message
+            log_message("info", "Progress saved successfully!", "Response Curves")
+
+    # Check if the RCS metadata file does not exist
+    if (
+        st.session_state["project_dct"]["response_curves"]["original_metadata_file"]
+        is None
+        or st.session_state["project_dct"]["response_curves"]["modified_metadata_file"]
+        is None
+    ):
+        # RCS metadata file does not exist. Generating new RCS data
+        generate_rcs_data()
+
+        # Log message
+        log_message(
+            "info",
+            "RCS metadata file does not exist. Generating new RCS data.",
+            "Response Curves",
+        )
+
+    # Load metadata files if they exist
+    original_data, modified_data = load_rcs_metadata_files()
+
+    # Retrieve the list of all channels names for specified Metrics and Panel
+    chanel_list_final = list(original_data[metrics_selected][panel_selected].keys())
+
+    # Channel Selection
+    channel_selected = channel_col.selectbox(
+        "Channel",
+        sorted(chanel_list_final),
+        format_func=name_formating,
+        key="selected_channel_name_selectbox",
+    )
+
+    # Extract original channel data for the selected metric, panel, and channel
+    original_channel_data = original_data[metrics_selected][panel_selected][
+        channel_selected
+    ]
+
+    # Extract modified channel data for the same metric, panel, and channel
+    modified_channel_data = modified_data[metrics_selected][panel_selected][
+        channel_selected
+    ]
+
+    # X and Y values for plotting
+    x = original_channel_data["x"]
+    y = original_channel_data["y"]
+
+    # Scaling factor for X values and range for S-curve plotting
+    power = original_channel_data["power"]
+    x_plot = original_channel_data["x_plot"]
+
+    # Original S-curve parameters
+    K_orig = original_channel_data["K"]
+    b_orig = original_channel_data["b"]
+    a_orig = original_channel_data["a"]
+    x0_orig = original_channel_data["x0"]
+
+    # Modified S-curve parameters (user-adjusted)
+    K_mod = modified_channel_data["K"]
+    b_mod = modified_channel_data["b"]
+    a_mod = modified_channel_data["a"]
+    x0_mod = modified_channel_data["x0"]
+
+    # Create a scatter plot for the original data points
+    fig = px.scatter(
+        x=x,
+        y=y,
+        title="Original and Modified S-Curve Plot",
+        labels={"x": "Spends", "y": name_formating(metrics_selected)},
+    )
+
+    # Add the modified S-curve trace
+    fig.add_trace(
+        go.Scatter(
+            x=x_plot,
+            y=s_curve(
+                np.array(x_plot) / 10**power,
+                K_mod,
+                b_mod,
+                a_mod,
+                x0_mod,
+            ),
+            line=dict(color="red"),
+            name="Modified",
+        ),
+    )
+
+    # Add the original S-curve trace
+    fig.add_trace(
+        go.Scatter(
+            x=x_plot,
+            y=s_curve(
+                np.array(x_plot) / 10**power,
+                K_orig,
+                b_orig,
+                a_orig,
+                x0_orig,
+            ),
+            line=dict(color="rgba(0, 255, 0, 0.6)"),  # Semi-transparent green
+            name="Original",
+        ),
+    )
+
+    # Customize the layout of the plot
+    fig.update_layout(
+        title="Comparison of Original and Modified Response-Curves",
+        xaxis_title="Input (Clicks, Impressions, etc..)",
+        yaxis_title=name_formating(metrics_selected),
+        legend_title="Curve Type",
+    )
+
+    # Display s-curve
+    st.plotly_chart(fig, use_container_width=True)
+
+    # Calculate R-squared for the original curve
+    y_orig_pred = s_curve(np.array(x) / 10**power, K_orig, b_orig, a_orig, x0_orig)
+    r2_orig = r2_score(y, y_orig_pred)
+
+    # Calculate R-squared for the modified curve
+    y_mod_pred = s_curve(np.array(x) / 10**power, K_mod, b_mod, a_mod, x0_mod)
+    r2_mod = r2_score(y, y_mod_pred)
+
+    # Calculate the difference in R-squared
+    r2_diff = r2_mod - r2_orig
+
+    # Display R-squared metrics
+    st.write("## R-squared Comparison")
+    r2_col = st.columns(3)
+
+    r2_col[0].metric("R-squared (Original)", f"{r2_orig:.2f}")
+    r2_col[1].metric("R-squared (Modified)", f"{r2_mod:.2f}")
+    r2_col[2].metric("Difference in R-squared", f"{r2_diff:.2f}")
+
+    # Define unique keys for each parameter based on the selection
+    K_key = f"K_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    b_key = f"b_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    a_key = f"a_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+    x0_key = f"x0_updated_key_{metrics_selected}_{panel_selected}_{channel_selected}"
+
+    # Initialize session state keys if they do not exist
+    if K_key not in st.session_state:
+        st.session_state[K_key] = K_mod
+    if b_key not in st.session_state:
+        st.session_state[b_key] = b_mod
+    if a_key not in st.session_state:
+        st.session_state[a_key] = a_mod
+    if x0_key not in st.session_state:
+        st.session_state[x0_key] = x0_mod
+
+    # RCS parameters input
+    rsc_ip_col = st.columns(4)
+    with rsc_ip_col[0]:
+        K_updated = st.number_input(
+            "K",
+            step=0.001,
+            min_value=0.000000,
+            format="%.6f",
+            on_change=modify_rcs_parameters,
+            args=(metrics_selected, panel_selected, channel_selected),
+            key=K_key,
+        )
+    with rsc_ip_col[1]:
+        b_updated = st.number_input(
+            "b",
+            step=0.001,
+            min_value=0.000000,
+            format="%.6f",
+            on_change=modify_rcs_parameters,
+            args=(metrics_selected, panel_selected, channel_selected),
+            key=b_key,
+        )
+    with rsc_ip_col[2]:
+        a_updated = st.number_input(
+            "a",
+            step=0.001,
+            min_value=0.000000,
+            format="%.6f",
+            on_change=modify_rcs_parameters,
+            args=(metrics_selected, panel_selected, channel_selected),
+            key=a_key,
+        )
+    with rsc_ip_col[3]:
+        x0_updated = st.number_input(
+            "x0",
+            step=0.001,
+            min_value=0.000000,
+            format="%.6f",
+            on_change=modify_rcs_parameters,
+            args=(metrics_selected, panel_selected, channel_selected),
+            key=x0_key,
+        )
+
+    # Create columns for Reset and Download buttons
+    reset_download_col = st.columns(2)
+    with reset_download_col[0]:
+        if st.button(
+            "Reset",
+            use_container_width=True,
+        ):
+            reset_parameters(
+                metrics_selected,
+                panel_selected,
+                channel_selected,
+                original_channel_data,
+            )
+
+            # Log message
+            log_message(
+                "info",
+                f"METRIC: {name_formating(metrics_selected)} ; PANEL: {name_formating(panel_selected)}, CHANNEL: {name_formating(channel_selected)} has been reset to its original value.",
+                "Response Curves",
+            )
+
+            st.rerun()
+
+    with reset_download_col[1]:
+        # Provide a download button for the modified RCS data
+        try:
+            # Create JSON file for RCS data
+            json_data = create_json_file()
+            st.download_button(
+                label="Download",
+                data=json_data,
+                file_name=f"{name_formating(metrics_selected)}_{name_formating(panel_selected)}_rcs_data.json",
+                mime="application/json",
+                use_container_width=True,
+            )
+        except:
+            # Download failed
+            pass
+
+    # Generate the DataFrame showing only non-matching parameters
+    updated_parm_df = updated_parm_gen(
+        original_data, modified_data, metrics_selected, panel_selected
+    )
+
+    # Display the DataFrame or show an informational message if no updates
+    if not updated_parm_df.empty:
+        st.write("## Parameter Comparison for Selected Metric and Panel")
+        st.dataframe(updated_parm_df, hide_index=True)
+    else:
+        st.info("No parameters are updated for the selected Metric and Panel")
+
+except Exception as e:
+    # Capture the error details
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    error_message = "".join(
+        traceback.format_exception(exc_type, exc_value, exc_traceback)
+    )
+
+    # Log message
+    log_message("error", f"An error occurred: {error_message}.", "Response Curves")
+
+    # Display a warning message
+    st.warning(
+        "Oops! Something went wrong. Please try refreshing the tool or creating a new project.",
+        icon="⚠️",
+    )

post_gres_cred.py

@@ -0,0 +1,2 @@
+db_cred = {'schema': "mmo_service_owner"}
+schema = ""

ppt_utils.py

@@ -0,0 +1,1419 @@
+import pandas as pd
+import numpy as np
+import pptx
+from pptx import Presentation
+from pptx.chart.data import CategoryChartData, ChartData
+from pptx.enum.chart import XL_CHART_TYPE, XL_LEGEND_POSITION, XL_LABEL_POSITION
+from pptx.enum.chart import XL_TICK_LABEL_POSITION
+from pptx.util import Inches, Pt
+import os
+import pickle
+from pathlib import Path
+from sklearn.metrics import (
+    mean_absolute_error,
+    r2_score,
+    mean_absolute_percentage_error,
+)
+import streamlit as st
+from collections import OrderedDict
+from utilities import get_metrics_names, initialize_data, retrieve_pkl_object_without_warning
+from io import BytesIO
+from pptx.dml.color import RGBColor
+from post_gres_cred import db_cred
+schema=db_cred['schema']
+
+from constants import (
+    TITLE_FONT_SIZE,
+    AXIS_LABEL_FONT_SIZE,
+    CHART_TITLE_FONT_SIZE,
+    AXIS_TITLE_FONT_SIZE,
+    DATA_LABEL_FONT_SIZE,
+    LEGEND_FONT_SIZE,
+    PIE_LEGEND_FONT_SIZE
+)
+
+
+def format_response_metric(target):
+    if target.startswith('response_metric_'):
+        target = target.replace('response_metric_', '')
+    target = target.replace("_", " ").title()
+    return target
+
+
+def smape(actual, forecast):
+    # Symmetric Mape (SMAPE) eliminates shortcomings of MAPE :
+    ## 1. MAPE becomes insanely high when actual is close to 0
+    ## 2. MAPE is more favourable to underforecast than overforecast
+    return (1 / len(actual)) * np.sum(1 * np.abs(forecast - actual) / (np.abs(actual) + np.abs(forecast)))
+
+
+def safe_num_to_per(num):
+    try:
+        return "{:.0%}".format(num)
+    except:
+        return num
+
+
+# Function to convert numbers to abbreviated format
+def convert_number_to_abbreviation(number):
+    try:
+        number = float(number)
+        if number >= 1000000:
+            return f'{number / 1000000:.1f} M'
+        elif number >= 1000:
+            return f'{number / 1000:.1f} K'
+        else:
+            return str(number)
+    except:
+        return number
+
+
+def round_off(x, round_off_decimal=0):
+    # round off
+    try:
+        x = float(x)
+        if x < 1 and x > 0:
+            round_off_decimal = int(np.floor(np.abs(np.log10(x)))) + max(round_off_decimal, 1)
+            x = np.round(x, round_off_decimal)
+        elif x < 0 and x > -1:
+            round_off_decimal = int(np.floor(np.abs(np.log10(np.abs(x))))) + max(round_off_decimal, 1)
+            x = -np.round(x, round_off_decimal)
+        else:
+            x = np.round(x, round_off_decimal)
+        return x
+    except:
+        return x
+
+
+def fill_table_placeholder(table_placeholder, slide, df, column_width=None, table_height=None):
+    cols = len(df.columns)
+    rows = len(df)
+
+    if table_height is None:
+        table_height = table_placeholder.height
+
+    x, y, cx, cy = table_placeholder.left, table_placeholder.top, table_placeholder.width, table_height
+    table = slide.shapes.add_table(rows + 1, cols, x, y, cx, cy).table
+
+    # Populate the table with data from the DataFrame
+    for row_idx, row in enumerate(df.values):
+        for col_idx, value in enumerate(row):
+            cell = table.cell(row_idx + 1, col_idx)
+            cell.text = str(value)
+    for col_idx, value in enumerate(df.columns):
+        cell = table.cell(0, col_idx)
+        cell.text = str(value)
+
+    if column_width is not None:
+        for col_idx, column_width in column_width.items():
+            table.columns[col_idx].width = Inches(column_width)
+
+    table_placeholder._element.getparent().remove(table_placeholder._element)
+
+
+def bar_chart(chart_placeholder, slide, chart_data, titles={}, min_y=None, max_y=None, type='V', legend=True,
+              label_type=None, xaxis_pos=None):
+    x, y, cx, cy = chart_placeholder.left, chart_placeholder.top, chart_placeholder.width, chart_placeholder.height
+    if type == 'V':
+        graphic_frame = slide.shapes.add_chart(
+            XL_CHART_TYPE.COLUMN_CLUSTERED, x, y, cx, cy, chart_data
+        )
+    if type == 'H':
+        graphic_frame = slide.shapes.add_chart(
+            XL_CHART_TYPE.BAR_CLUSTERED, x, y, cx, cy, chart_data
+        )
+    chart = graphic_frame.chart
+
+    category_axis = chart.category_axis
+    value_axis = chart.value_axis
+
+    # Add chart title
+    if 'chart_title' in titles.keys():
+        chart.has_title = True
+        chart.chart_title.text_frame.text = titles['chart_title']
+        chart_title = chart.chart_title.text_frame.paragraphs[0].runs[0]
+        chart_title.font.size = Pt(CHART_TITLE_FONT_SIZE)
+
+    # Add axis titles
+    if 'x_axis' in titles.keys():
+        category_axis.has_title = True
+        category_axis.axis_title.text_frame.text = titles['x_axis']
+        category_title = category_axis.axis_title.text_frame.paragraphs[0].runs[0]
+        category_title.font.size = Pt(AXIS_TITLE_FONT_SIZE)
+
+    if 'y_axis' in titles.keys():
+        value_axis.has_title = True
+        value_axis.axis_title.text_frame.text = titles['y_axis']
+        value_title = value_axis.axis_title.text_frame.paragraphs[0].runs[0]
+        value_title.font.size = Pt(AXIS_TITLE_FONT_SIZE)
+
+    if xaxis_pos == 'low':
+        category_axis.tick_label_position = XL_TICK_LABEL_POSITION.LOW
+
+    # Customize the chart
+    if legend:
+        chart.has_legend = True
+        chart.legend.position = XL_LEGEND_POSITION.BOTTOM
+        chart.legend.font.size = Pt(LEGEND_FONT_SIZE)
+        chart.legend.include_in_layout = False
+
+    # Adjust font size for axis labels
+    category_axis.tick_labels.font.size = Pt(AXIS_LABEL_FONT_SIZE)
+    value_axis.tick_labels.font.size = Pt(AXIS_LABEL_FONT_SIZE)
+
+    if min_y is not None:
+        value_axis.minimum_scale = min_y  # Adjust this value as needed
+
+    if max_y is not None:
+        value_axis.maximum_scale = max_y  # Adjust this value as needed
+
+    plot = chart.plots[0]
+    plot.has_data_labels = True
+    data_labels = plot.data_labels
+
+    if label_type == 'per':
+        data_labels.number_format = '0"%"'
+    elif label_type == '$':
+        data_labels.number_format = '$[>=1000000]#,##0.0,,"M";$[>=1000]#,##0.0,"K";$#,##0'
+    elif label_type == '$1':
+        data_labels.number_format = '$[>=1000000]#,##0,,"M";$[>=1000]#,##0,"K";$#,##0'
+    elif label_type == 'M':
+        data_labels.number_format = '#0.0,,"M"'
+    elif label_type == 'M1':
+        data_labels.number_format = '#0.00,,"M"'
+    elif label_type == 'K':
+        data_labels.number_format = '#0.0,"K"'
+
+    data_labels.font.size = Pt(DATA_LABEL_FONT_SIZE)
+
+    chart_placeholder._element.getparent().remove(chart_placeholder._element)
+
+
+def line_chart(chart_placeholder, slide, chart_data, titles={}, min_y=None, max_y=None):
+    # Add the chart to the slide
+    x, y, cx, cy = chart_placeholder.left, chart_placeholder.top, chart_placeholder.width, chart_placeholder.height
+
+    chart = slide.shapes.add_chart(
+        XL_CHART_TYPE.LINE, x, y, cx, cy, chart_data
+    ).chart
+
+    chart.has_legend = True
+    chart.legend.position = XL_LEGEND_POSITION.BOTTOM
+    chart.legend.font.size = Pt(LEGEND_FONT_SIZE)
+
+    category_axis = chart.category_axis
+    value_axis = chart.value_axis
+
+    if min_y is not None:
+        value_axis.minimum_scale = min_y
+
+    if max_y is not None:
+        value_axis.maximum_scale = max_y
+
+    if min_y is not None and max_y is not None:
+        value_axis.major_unit = int((max_y - min_y) / 2)
+
+    if 'chart_title' in titles.keys():
+        chart.has_title = True
+        chart.chart_title.text_frame.text = titles['chart_title']
+        chart_title = chart.chart_title.text_frame.paragraphs[0].runs[0]
+        chart_title.font.size = Pt(CHART_TITLE_FONT_SIZE)
+
+    if 'x_axis' in titles.keys():
+        category_axis.has_title = True
+        category_axis.axis_title.text_frame.text = titles['x_axis']
+        category_title = category_axis.axis_title.text_frame.paragraphs[0].runs[0]
+        category_title.font.size = Pt(AXIS_TITLE_FONT_SIZE)
+
+    if 'y_axis' in titles.keys():
+        value_axis.has_title = True
+        value_axis.axis_title.text_frame.text = titles['y_axis']
+        value_title = value_axis.axis_title.text_frame.paragraphs[0].runs[0]
+        value_title.font.size = Pt(AXIS_TITLE_FONT_SIZE)
+
+    # Adjust font size for axis labels
+    category_axis.tick_labels.font.size = Pt(AXIS_LABEL_FONT_SIZE)
+    value_axis.tick_labels.font.size = Pt(AXIS_LABEL_FONT_SIZE)
+
+    plot = chart.plots[0]
+    series = plot.series[1]
+    line = series.format.line
+    line.color.rgb = RGBColor(141, 47, 0)
+
+    chart_placeholder._element.getparent().remove(chart_placeholder._element)
+
+
+def pie_chart(chart_placeholder, slide, chart_data, title):
+    # Add the chart to the slide
+    x, y, cx, cy = chart_placeholder.left, chart_placeholder.top, chart_placeholder.width, chart_placeholder.height
+
+    chart = slide.shapes.add_chart(
+        XL_CHART_TYPE.PIE, x, y, cx, cy, chart_data
+    ).chart
+
+    chart.has_legend = True
+    chart.legend.position = XL_LEGEND_POSITION.RIGHT
+    chart.legend.include_in_layout = False
+    chart.legend.font.size = Pt(PIE_LEGEND_FONT_SIZE)
+
+    chart.plots[0].has_data_labels = True
+    data_labels = chart.plots[0].data_labels
+    data_labels.number_format = '0%'
+    data_labels.position = XL_LABEL_POSITION.OUTSIDE_END
+    data_labels.font.size = Pt(DATA_LABEL_FONT_SIZE)
+
+    chart.has_title = True
+    chart.chart_title.text_frame.text = title
+    chart_title = chart.chart_title.text_frame.paragraphs[0].runs[0]
+    chart_title.font.size = Pt(CHART_TITLE_FONT_SIZE)
+
+    chart_placeholder._element.getparent().remove(chart_placeholder._element)
+
+
+def title_and_table(slide, title, df, column_width=None, custom_table_height=False):
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = title
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    table_placeholder = slide.placeholders[ph_idx[1]]
+
+    table_height = None
+    if custom_table_height:
+        if len(df) < 4:
+            table_height = int(np.ceil(table_placeholder.height / 2))
+
+    fill_table_placeholder(table_placeholder, slide, df, column_width, table_height)
+
+    # try:
+    #     font_size = 18  # default for 3*3
+    #     if cols < 3:
+    #         row_diff = 3 - rows
+    #         font_size = font_size + ((row_diff)*2) # 1 row less -> 2 pt font size increase & vice versa
+    #     else:
+    #         row_diff = 2 - rows
+    #         font_size = font_size + ((row_diff)*2)
+    #     for row in table.rows:
+    #         for cell in row.cells:
+    #             cell.text_frame.paragraphs[0].runs[0].font.size = Pt(font_size)
+    # except Exception as e :
+    #     print("**"*30)
+    #     print(e)
+    # else:
+    # except Exception as e:
+    #     print('table', e)
+    return slide
+
+
+def data_import(data, bin_dict):
+    import_df = pd.DataFrame(columns=['Category', 'Value'])
+
+    import_df.at[0, 'Category'] = 'Date Range'
+
+    date_start = data['date'].min().date()
+    date_end = data['date'].max().date()
+    import_df.at[0, 'Value'] = str(date_start) + ' - ' + str(date_end)
+
+    import_df.at[1, 'Category'] = 'Response Metrics'
+    import_df.at[1, 'Value'] = ', '.join(bin_dict['Response Metrics'])
+
+    import_df.at[2, 'Category'] = 'Media Variables'
+    import_df.at[2, 'Value'] = ', '.join(bin_dict['Media'])
+
+    import_df.at[3, 'Category'] = 'Spend Variables'
+    import_df.at[3, 'Value'] = ', '.join(bin_dict['Spends'])
+
+    if bin_dict['Exogenous'] != []:
+        import_df.at[4, 'Category'] = 'Exogenous Variables'
+        import_df.at[4, 'Value'] = ', '.join(bin_dict['Exogenous'])
+
+    return import_df
+
+
+def channel_groups_df(channel_groups_dct={}, bin_dict={}):
+    df = pd.DataFrame(columns=['Channel', 'Media Variables', 'Spend Variables'])
+    i = 0
+    for channel, vars in channel_groups_dct.items():
+        media_vars = ", ".join(list(set(vars).intersection(set(bin_dict["Media"]))))
+        spend_vars = ", ".join(list(set(vars).intersection(set(bin_dict["Spends"]))))
+        df.at[i, "Channel"] = channel
+        df.at[i, 'Media Variables'] = media_vars
+        df.at[i, 'Spend Variables'] = spend_vars
+        i += 1
+
+    return df
+
+
+def transformations(transform_dict):
+    transform_df = pd.DataFrame(columns=['Category', 'Transformation', 'Value'])
+    i = 0
+
+    for category in ['Media', 'Exogenous']:
+        transformations = f'transformation_{category}'
+        category_dict = transform_dict[category]
+        if transformations in category_dict.keys():
+            for transformation in category_dict[transformations]:
+                transform_df.at[i, 'Category'] = category
+                transform_df.at[i, 'Transformation'] = transformation
+                transform_df.at[i, 'Value'] = str(category_dict[transformation][0]) + ' - ' + str(
+                    category_dict[transformation][1])
+                i += 1
+    return transform_df
+
+
+def model_metrics(model_dict, is_panel):
+    metrics_df = pd.DataFrame(
+        columns=[
+            "Response Metric",
+            "Model",
+            "R2",
+            "ADJR2",
+            "Train MAPE",
+            "Test MAPE"
+        ]
+    )
+    i = 0
+    for key in model_dict.keys():
+        target = key.split("__")[1]
+        metrics_df.at[i, "Response Metric"] = format_response_metric(target)
+        metrics_df.at[i, "Model"] = key.split("__")[0]
+
+        y = model_dict[key]["X_train_tuned"][target]
+
+        feature_set = model_dict[key]["feature_set"]
+
+        if is_panel:
+            random_df = get_random_effects(
+                media_data, panel_col, model_dict[key]["Model_object"]
+            )
+            pred = mdf_predict(
+                model_dict[key]["X_train_tuned"],
+                model_dict[key]["Model_object"],
+                random_df,
+            )["pred"]
+        else:
+            pred = model_dict[key]["Model_object"].predict(model_dict[key]["X_train_tuned"][feature_set])
+
+        ytest = model_dict[key]["X_test_tuned"][target]
+        if is_panel:
+
+            predtest = mdf_predict(
+                model_dict[key]["X_test_tuned"],
+                model_dict[key]["Model_object"],
+                random_df,
+            )["pred"]
+
+        else:
+            predtest = model_dict[key]["Model_object"].predict(model_dict[key]["X_test_tuned"][feature_set])
+
+        metrics_df.at[i, "R2"] = np.round(r2_score(y, pred), 2)
+        adjr2 = 1 - (1 - metrics_df.loc[i, "R2"]) * (
+                len(y) - 1
+        ) / (len(y) - len(model_dict[key]["feature_set"]) - 1)
+        metrics_df.at[i, "ADJR2"] = np.round(adjr2, 2)
+        # y = np.where(np.abs(y) < 0.00001, 0.00001, y)
+        metrics_df.at[i, "Train MAPE"] = np.round(smape(y, pred), 2)
+        metrics_df.at[i, "Test MAPE"] = np.round(smape(ytest, predtest), 2)
+        i += 1
+    metrics_df = np.round(metrics_df, 2)
+
+    return metrics_df
+
+
+def model_result(slide, model_key, model_dict, model_metrics_df, date_col):
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = model_key.split('__')[0]
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+    target = model_key.split('__')[1]
+
+    metrics_table_placeholder = slide.placeholders[ph_idx[1]]
+    metrics_df = model_metrics_df[model_metrics_df['Model'] == model_key.split('__')[0]].reset_index(drop=True)
+
+    # Accuracy = 1-mape
+    metrics_df['Accuracy'] = 100 * (1 - metrics_df['Train MAPE'])
+    metrics_df['Accuracy'] = metrics_df['Accuracy'].apply(lambda x: f'{np.round(x, 0)}%')
+
+    ## Removing metrics as requested by Ioannis
+
+    metrics_df = metrics_df.drop(columns=['R2', 'ADJR2', 'Train MAPE', 'Test MAPE'])
+    fill_table_placeholder(metrics_table_placeholder, slide, metrics_df)
+
+    # coeff_table_placeholder = slide.placeholders[ph_idx[2]]
+    # coeff_df = pd.DataFrame(model_dict['Model_object'].params)
+    # coeff_df.reset_index(inplace=True)
+    # coeff_df.columns = ['Feature', 'Coefficent']
+    # fill_table_placeholder(coeff_table_placeholder, slide, coeff_df)
+
+    chart_placeholder = slide.placeholders[ph_idx[2]]
+    full_df = pd.concat([model_dict['X_train_tuned'], model_dict['X_test_tuned']])
+    full_df['Predicted'] = model_dict['Model_object'].predict(full_df[model_dict['feature_set']])
+    pred_df = full_df[[date_col, target, 'Predicted']]
+    pred_df.rename(columns={target: 'Actual'}, inplace=True)
+
+    # Create chart data
+    chart_data = CategoryChartData()
+    chart_data.categories = pred_df[date_col]
+    chart_data.add_series('Actual', pred_df['Actual'])
+    chart_data.add_series('Predicted', pred_df['Predicted'])
+
+    # Set range for y axis
+    min_y = np.floor(min(pred_df['Actual'].min(), pred_df['Predicted'].min()))
+    max_y = np.ceil(max(pred_df['Actual'].max(), pred_df['Predicted'].max()))
+
+    # Create the chart
+    line_chart(chart_placeholder=chart_placeholder,
+               slide=slide,
+               chart_data=chart_data,
+               titles={'chart_title': 'Actual VS Predicted',
+                       'x_axis': 'Date',
+                       'y_axis': target.title().replace('_', ' ')
+                       },
+               min_y=min_y,
+               max_y=max_y
+               )
+
+    return slide
+
+
+def metrics_contributions(slide, contributions_excels_dict, panel_col):
+    # Create data for metrics contributions
+    all_contribution_df = pd.DataFrame(columns=['Channel'])
+    target_sum_dict = {}
+    sort_support_dct = {}
+    for target in contributions_excels_dict.keys():
+        contribution_df = contributions_excels_dict[target]['CONTRIBUTION MMM'].copy()
+        if 'Date' in contribution_df.columns:
+            contribution_df.drop(columns=['Date'], inplace=True)
+        if panel_col in contribution_df.columns:
+            contribution_df.drop(columns=[panel_col], inplace=True)
+
+        contribution_df = pd.DataFrame(np.sum(contribution_df, axis=0)).reset_index()
+        contribution_df.columns = ['Channel', target]
+        target_sum = contribution_df[target].sum()
+        target_sum_dict[target] = target_sum
+        contribution_df[target] = 100 * contribution_df[target] / target_sum
+
+        all_contribution_df = pd.merge(all_contribution_df, contribution_df, on='Channel', how='outer')
+
+    sorted_target_sum_dict = sorted(target_sum_dict.items(), key=lambda kv: kv[1], reverse=True)
+    sorted_target_sum_keys = [kv[0] for kv in sorted_target_sum_dict]
+    if len([metric for metric in sorted_target_sum_keys if metric.lower() == 'revenue']) == 1:
+        rev_metric = [metric for metric in sorted_target_sum_keys if metric.lower() == 'revenue'][0]
+        sorted_target_sum_keys.remove(rev_metric)
+        sorted_target_sum_keys.append(rev_metric)
+    all_contribution_df = all_contribution_df[['Channel'] + sorted_target_sum_keys]
+
+    # for col in all_contribution_df.columns:
+    #     all_contribution_df[col]=all_contribution_df[col].apply(lambda x: round_off(x,1))
+
+    # Sort Data by Average contribution of the channels keeping base first <Removed>
+    # all_contribution_df['avg'] = np.mean(all_contribution_df[list(contributions_excels_dict.keys())],axis=1)
+    # all_contribution_df['rank'] = all_contribution_df['avg'].rank(ascending=False)
+
+    # Sort data by contribution of bottom funnel metric
+    bottom_funnel_metric = sorted_target_sum_keys[-1]
+    all_contribution_df['rank'] = all_contribution_df[bottom_funnel_metric].rank(ascending=False)
+    all_contribution_df.loc[all_contribution_df[all_contribution_df['Channel'] == 'base'].index, 'rank'] = 0
+    all_contribution_df = all_contribution_df.sort_values(by='rank')
+    all_contribution_df.drop(columns=['rank'], inplace=True)
+
+    # Add title
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = "Response Metrics Contributions"
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    for target in contributions_excels_dict.keys():
+        all_contribution_df[target] = all_contribution_df[target].astype(float)
+
+
+    # Create chart data
+    chart_data = CategoryChartData()
+    chart_data.categories = all_contribution_df['Channel']
+    for target in sorted_target_sum_keys:
+        chart_data.add_series(format_response_metric(target), all_contribution_df[target])
+    chart_placeholder = slide.placeholders[ph_idx[1]]
+
+    if isinstance(np.min(all_contribution_df.select_dtypes(exclude=['object', 'datetime'])), float):
+
+        # Add the chart to the slide
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Response Metrics Contributions',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'Contributions'},
+                  min_y=np.floor(np.min(all_contribution_df.select_dtypes(exclude=['object', 'datetime']))),
+                  max_y=np.ceil(np.max(all_contribution_df.select_dtypes(exclude=['object', 'datetime']))),
+                  type='V',
+                  label_type='per'
+                  )
+    else:
+
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Response Metrics Contributions',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'Contributions'},
+                  min_y=np.floor(np.min(all_contribution_df.select_dtypes(exclude=['object', 'datetime'])).values[0]),
+                  max_y=np.ceil(np.max(all_contribution_df.select_dtypes(exclude=['object', 'datetime'])).values[0]),
+                  type='V',
+                  label_type='per'
+                  )
+
+    return slide
+
+
+def model_media_performance(slide, target, contributions_excels_dict, date_col='Date', is_panel=False,
+                            panel_col='panel'):
+    # Add title
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = "Media Performance - " + target.title().replace("_", " ")
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    # CONTRIBUTION CHART
+    # Create contribution data
+    contribution_df = contributions_excels_dict[target]['CONTRIBUTION MMM']
+    if panel_col in contribution_df.columns:
+        contribution_df.drop(columns=[panel_col], inplace=True)
+    # contribution_df.drop(columns=[date_col], inplace=True)
+    contribution_df = pd.DataFrame(np.sum(contribution_df, axis=0)).reset_index()
+    contribution_df.columns = ['Channel', format_response_metric(target)]
+    contribution_df['Channel'] = contribution_df['Channel'].apply(lambda x: x.title())
+    target_sum = contribution_df[format_response_metric(target)].sum()
+    contribution_df[format_response_metric(target)] = contribution_df[format_response_metric(target)] / target_sum
+    contribution_df.sort_values(by=['Channel'], ascending=False, inplace=True)
+
+    # for col in contribution_df.columns:
+    #     contribution_df[col] = contribution_df[col].apply(lambda x : round_off(x))
+
+    # Create Chart Data
+    chart_data = ChartData()
+    chart_data.categories = contribution_df['Channel']
+    chart_data.add_series('Contribution', contribution_df[format_response_metric(target)])
+
+    chart_placeholder = slide.placeholders[ph_idx[2]]
+    pie_chart(chart_placeholder=chart_placeholder,
+              slide=slide,
+              chart_data=chart_data,
+              title='Contribution')
+
+    # SPENDS CHART
+
+    initialize_data(panel='aggregated', metrics=target)
+    scenario = st.session_state["scenario"]
+    spends_values = {
+        channel_name: round(
+            scenario.channels[channel_name].actual_total_spends
+            * scenario.channels[channel_name].conversion_rate,
+            1,
+        )
+        for channel_name in st.session_state["channels_list"]
+    }
+    spends_df = pd.DataFrame(columns=['Channel', 'Media Spend'])
+    spends_df['Channel'] = list(spends_values.keys())
+    spends_df['Media Spend'] = list(spends_values.values())
+    spends_sum = spends_df['Media Spend'].sum()
+    spends_df['Media Spend'] = spends_df['Media Spend'] / spends_sum
+    spends_df['Channel'] = spends_df['Channel'].apply(lambda x: x.title())
+    spends_df.sort_values(by='Channel', ascending=False, inplace=True)
+    # for col in spends_df.columns:
+    #     spends_df[col] = spends_df[col].apply(lambda x : round_off(x))
+
+    # Create Chart Data
+    spends_chart_data = ChartData()
+    spends_chart_data = ChartData()
+    spends_chart_data.categories = spends_df['Channel']
+    spends_chart_data.add_series('Media Spend', spends_df['Media Spend'])
+
+    spends_chart_placeholder = slide.placeholders[ph_idx[1]]
+    pie_chart(chart_placeholder=spends_chart_placeholder,
+              slide=slide,
+              chart_data=spends_chart_data,
+              title='Media Spend')
+    # spends_values.append(0)
+    return contribution_df, spends_df
+
+
+# def get_saved_scenarios_dict(project_path):
+#     # Path to the saved scenarios file
+#     saved_scenarios_dict_path = os.path.join(
+#         project_path, "saved_scenarios.pkl"
+#     )
+#
+#     # Load existing scenarios if the file exists
+#     if os.path.exists(saved_scenarios_dict_path):
+#         with open(saved_scenarios_dict_path, "rb") as f:
+#             saved_scenarios_dict = pickle.load(f)
+#     else:
+#         saved_scenarios_dict = OrderedDict()
+#
+#     return saved_scenarios_dict
+
+def optimization_summary(slide, scenario, scenario_name):
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Optimization Summary'  # + ' (Scenario: ' + scenario_name + ')'
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    multiplier = 1 / float(scenario['multiplier'])
+    # st.write(scenario['multiplier'], multiplier)
+    ## Multiplier is an indicator of selected time fram
+    ## Doesn't effect CPA
+
+    opt_on = scenario['optimization']
+    if opt_on.lower() == 'spends':
+        opt_on = 'Media Spend'
+
+    details_ph = slide.placeholders[ph_idx[3]]
+    details_ph.text = 'Scenario Name: ' + scenario_name + \
+                      '\nResponse Metric: ' + str(scenario['metrics_selected']).replace("_", " ").title() + \
+                      '\nOptimized on: ' + str(opt_on).replace("_", " ").title()
+
+    scenario_df = pd.DataFrame(columns=['Category', 'Actual', 'Simulated', 'Change'])
+    scenario_df.at[0, 'Category'] = 'Media Spend'
+
+    scenario_df.at[0, 'Actual'] = scenario['actual_total_spends'] * multiplier
+    scenario_df.at[0, 'Simulated'] = scenario['modified_total_spends'] * multiplier
+    scenario_df.at[0, 'Change'] = (scenario['modified_total_spends'] - scenario['actual_total_spends']) * multiplier
+
+    scenario_df.at[1, 'Category'] = scenario['metrics_selected'].replace("_", " ").title()
+    scenario_df.at[1, 'Actual'] = scenario['actual_total_sales'] * multiplier
+    scenario_df.at[1, 'Simulated'] = (scenario['modified_total_sales']) * multiplier
+    scenario_df.at[1, 'Change'] = (scenario['modified_total_sales'] - scenario['actual_total_sales']) * multiplier
+
+    scenario_df.at[2, 'Category'] = 'CPA'
+    actual_cpa = scenario['actual_total_spends'] / scenario['actual_total_sales']
+    modified_cpa = scenario['modified_total_spends'] / scenario['modified_total_sales']
+    scenario_df.at[2, 'Actual'] = actual_cpa
+    scenario_df.at[2, 'Simulated'] = modified_cpa
+    scenario_df.at[2, 'Change'] = modified_cpa - actual_cpa
+
+    scenario_df.at[3, 'Category'] = 'ROI'
+    act_roi = scenario['actual_total_sales'] / scenario['actual_total_spends']
+    opt_roi = scenario['modified_total_sales'] / scenario['modified_total_spends']
+    scenario_df.at[3, 'Actual'] = act_roi
+    scenario_df.at[3, 'Simulated'] = opt_roi
+    scenario_df.at[3, 'Change'] = opt_roi - act_roi
+
+    for col in scenario_df.columns:
+        scenario_df[col] = scenario_df[col].apply(lambda x: round_off(x, 1))
+        scenario_df[col] = scenario_df[col].apply(lambda x: convert_number_to_abbreviation(x))
+
+    table_placeholder = slide.placeholders[ph_idx[1]]
+    fill_table_placeholder(table_placeholder, slide, scenario_df)
+
+    channel_spends_df = pd.DataFrame(columns=['Channel', 'Actual Spends', 'Optimized Spends'])
+    for i, channel in enumerate(scenario['channels'].values()):
+        channel_spends_df.at[i, 'Channel'] = channel['name']
+        channel_conversion_rate = channel[
+            "conversion_rate"
+        ]
+        channel_spends_df.at[i, 'Actual Spends'] = (
+                                                           channel["actual_total_spends"]
+                                                           * channel_conversion_rate
+                                                   ) * multiplier
+        channel_spends_df.at[i, 'Optimized Spends'] = (
+                                                              channel["modified_total_spends"]
+                                                              * channel_conversion_rate
+                                                      ) * multiplier
+    channel_spends_df['Actual Spends'] = channel_spends_df['Actual Spends'].astype('float')
+    channel_spends_df['Optimized Spends'] = channel_spends_df['Optimized Spends'].astype('float')
+
+    for col in channel_spends_df.columns:
+        channel_spends_df[col] = channel_spends_df[col].apply(lambda x: round_off(x, 0))
+
+    # Sort data on Actual Spends
+    channel_spends_df.sort_values(by='Actual Spends', inplace=True, ascending=False)
+
+    # Create chart data
+    chart_data = CategoryChartData()
+    chart_data.categories = channel_spends_df['Channel']
+    for col in ['Actual Spends', 'Optimized Spends']:
+        chart_data.add_series(col, channel_spends_df[col])
+
+    chart_placeholder = slide.placeholders[ph_idx[2]]
+
+    # Add the chart to the slide
+    if isinstance(np.max(channel_spends_df.select_dtypes(exclude=['object', 'datetime'])),float):
+        bar_chart(chart_placeholder=chart_placeholder,
+              slide=slide,
+              chart_data=chart_data,
+              titles={'chart_title': 'Channel Wise Spends',
+                      # 'x_axis':'Channels',
+                      'y_axis': 'Spends'},
+              # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+              min_y=0,
+              max_y=np.ceil(np.max(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+              label_type='$'
+              )
+    else:
+        # Add the chart to the slide
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Channel Wise Spends',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'Spends'},
+                  # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+                  min_y=0,
+                  max_y=np.ceil(np.max(channel_spends_df.select_dtypes(exclude=['object', 'datetime'])).values[0]),
+                  label_type='$'
+                  )
+
+
+def channel_wise_spends(slide, scenario):
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Channel Spends and Impact'
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+    # print(scenario.keys())
+
+    multiplier = 1 / float(scenario['multiplier'])
+    channel_spends_df = pd.DataFrame(columns=['Channel', 'Actual Spends', 'Optimized Spends'])
+    for i, channel in enumerate(scenario['channels'].values()):
+        channel_spends_df.at[i, 'Channel'] = channel['name']
+        channel_conversion_rate = channel["conversion_rate"]
+        channel_spends_df.at[i, 'Actual Spends'] = (channel[
+                                                        "actual_total_spends"] * channel_conversion_rate) * multiplier
+        channel_spends_df.at[i, 'Optimized Spends'] = (channel[
+                                                           "modified_total_spends"] * channel_conversion_rate) * multiplier
+    channel_spends_df['Actual Spends'] = channel_spends_df['Actual Spends'].astype('float')
+    channel_spends_df['Optimized Spends'] = channel_spends_df['Optimized Spends'].astype('float')
+
+    actual_sum = channel_spends_df['Actual Spends'].sum()
+    opt_sum = channel_spends_df['Optimized Spends'].sum()
+
+    for col in channel_spends_df.columns:
+        channel_spends_df[col] = channel_spends_df[col].apply(lambda x: round_off(x, 0))
+
+    channel_spends_df['Actual Spends %'] = 100 * (channel_spends_df['Actual Spends'] / actual_sum)
+    channel_spends_df['Optimized Spends %'] = 100 * (channel_spends_df['Optimized Spends'] / opt_sum)
+    channel_spends_df['Actual Spends %'] = np.round(channel_spends_df['Actual Spends %'])
+    channel_spends_df['Optimized Spends %'] = np.round(channel_spends_df['Optimized Spends %'])
+
+    # Sort Data based on Actual Spends %
+    channel_spends_df.sort_values(by='Actual Spends %', inplace=True)
+
+    # Create chart data
+    chart_data = CategoryChartData()
+    chart_data.categories = channel_spends_df['Channel']
+    for col in ['Actual Spends %', 'Optimized Spends %']:
+        # for col in ['Actual Spends %']:
+        chart_data.add_series(col, channel_spends_df[col])
+    chart_placeholder = slide.placeholders[ph_idx[1]]
+
+    # Add the chart to the slide
+    if isinstance(np.max(channel_spends_df[['Actual Spends %', 'Optimized Spends %']]), float):
+        bar_chart(chart_placeholder=chart_placeholder,
+              slide=slide,
+              chart_data=chart_data,
+              titles={'chart_title': 'Spend Split %',
+                      # 'x_axis':'Channels',
+                      'y_axis': 'Spend %'},
+              min_y=0,
+              max_y=np.ceil(np.max(channel_spends_df[['Actual Spends %', 'Optimized Spends %']])),
+              type='H',
+              legend=True,
+              label_type='per',
+              xaxis_pos='low'
+              )
+    else:
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Spend Split %',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'Spend %'},
+                  min_y=0,
+                  max_y=np.ceil(np.max(channel_spends_df[['Actual Spends %', 'Optimized Spends %']]).values[0]),
+                  type='H',
+                  legend=True,
+                  label_type='per',
+                  xaxis_pos='low'
+                  )
+    #
+    # # Create chart data
+    # chart_data_1 = CategoryChartData()
+    # chart_data_1.categories = channel_spends_df['Channel']
+    # # for col in ['Actual Spends %', 'Optimized Spends %']:
+    # for col in ['Optimized Spends %']:
+    #     chart_data_1.add_series(col, channel_spends_df[col])
+    # chart_placeholder_1 = slide.placeholders[ph_idx[3]]
+    #
+    # # Add the chart to the slide
+    # bar_chart(chart_placeholder=chart_placeholder_1,
+    #           slide=slide,
+    #           chart_data=chart_data_1,
+    #           titles={'chart_title': 'Optimized Spends Split %',
+    #                   # 'x_axis':'Channels',
+    #                   'y_axis': 'Spends %'},
+    #           min_y=0,
+    #           max_y=np.ceil(np.max(channel_spends_df[['Actual Spends %', 'Optimized Spends %']])),
+    #           type='H',
+    #           legend=False,
+    #           label_type='per'
+    #           )
+
+    channel_spends_df['Delta %'] = 100 * (channel_spends_df['Optimized Spends'] - channel_spends_df['Actual Spends']) / \
+                                   channel_spends_df['Actual Spends']
+    channel_spends_df['Delta %'] = channel_spends_df['Delta %'].apply(lambda x: round_off(x, 0))
+
+    # Create chart data
+    delta_chart_data = CategoryChartData()
+    delta_chart_data.categories = channel_spends_df['Channel']
+    col = 'Delta %'
+    delta_chart_data.add_series(col, channel_spends_df[col])
+    delta_chart_placeholder = slide.placeholders[ph_idx[3]]
+
+    # Add the chart to the slide
+    if isinstance(np.min(channel_spends_df['Delta %']), float):
+        bar_chart(chart_placeholder=delta_chart_placeholder,
+                  slide=slide,
+                  chart_data=delta_chart_data,
+                  titles={'chart_title': 'Spend Delta %',
+                          'y_axis': 'Spend Delta %'},
+                  min_y=np.floor(np.min(channel_spends_df['Delta %'])),
+                  max_y=np.ceil(np.max(channel_spends_df['Delta %'])),
+                  type='H',
+                  legend=False,
+                  label_type='per',
+                  xaxis_pos='low'
+
+                  )
+    else:
+        bar_chart(chart_placeholder=delta_chart_placeholder,
+                  slide=slide,
+                  chart_data=delta_chart_data,
+                  titles={'chart_title': 'Spend Delta %',
+                          'y_axis': 'Spend Delta %'},
+                  min_y=np.floor(np.min(channel_spends_df['Delta %']).values[0]),
+                  max_y=np.ceil(np.max(channel_spends_df['Delta %']).values[0]),
+                  type='H',
+                  legend=False,
+                  label_type='per',
+                  xaxis_pos='low'
+
+                  )
+
+    # Incremental Impact
+    channel_inc_df = pd.DataFrame(columns=['Channel', 'Increment'])
+    for i, channel in enumerate(scenario['channels'].values()):
+        channel_inc_df.at[i, 'Channel'] = channel['name']
+        act_impact = channel['actual_total_sales']
+        opt_impact = channel['modified_total_sales']
+        impact = opt_impact - act_impact
+        impact = round_off(impact, 0)
+        impact = impact if abs(impact) > 0.0001 else 0
+        channel_inc_df.at[i, 'Increment'] = impact
+
+    channel_inc_df_1 = pd.merge(channel_spends_df, channel_inc_df, how='left', on='Channel')
+
+    # Create chart data
+    delta_chart_data = CategoryChartData()
+    delta_chart_data.categories = channel_inc_df_1['Channel']
+    col = 'Increment'
+    delta_chart_data.add_series(col, channel_inc_df_1[col])
+    delta_chart_placeholder = slide.placeholders[ph_idx[2]]
+
+    label_req = True
+    if min(np.abs(channel_inc_df_1[col])) > 100000:  # 0.1M
+        label_type = 'M'
+    elif min(np.abs(channel_inc_df_1[col])) > 10000 and max(np.abs(channel_inc_df_1[col])) > 1000000:
+        label_type = 'M1'
+    elif min(np.abs(channel_inc_df_1[col])) > 100 and max(np.abs(channel_inc_df_1[col])) > 1000:
+        label_type = 'K'
+    else:
+        label_req = False
+    # Add the chart to the slide
+    if label_req:
+        bar_chart(chart_placeholder=delta_chart_placeholder,
+                  slide=slide,
+                  chart_data=delta_chart_data,
+                  titles={'chart_title': 'Incremental Impact',
+                          'y_axis': format_response_metric(scenario['metrics_selected'])},
+                  # min_y=np.floor(np.min(channel_inc_df_1['Delta %'])),
+                  # max_y=np.ceil(np.max(channel_inc_df_1['Delta %'])),
+                  type='H',
+                  label_type=label_type,
+                  legend=False,
+                  xaxis_pos='low'
+                  )
+    else:
+        bar_chart(chart_placeholder=delta_chart_placeholder,
+                  slide=slide,
+                  chart_data=delta_chart_data,
+                  titles={'chart_title': 'Increment',
+                          'y_axis': scenario['metrics_selected']},
+                  # min_y=np.floor(np.min(channel_inc_df_1['Delta %'])),
+                  # max_y=np.ceil(np.max(channel_inc_df_1['Delta %'])),
+                  type='H',
+                  legend=False,
+                  xaxis_pos='low'
+                  )
+
+
+def channel_wise_roi(slide, scenario):
+    channel_roi_mroi = scenario['channel_roi_mroi']
+
+    # Add title
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Channel ROIs'
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    channel_roi_df = pd.DataFrame(columns=['Channel', 'Actual ROI', 'Optimized ROI'])
+    for i, channel in enumerate(channel_roi_mroi.keys()):
+        channel_roi_df.at[i, 'Channel'] = channel
+        channel_roi_df.at[i, 'Actual ROI'] = channel_roi_mroi[channel]['actual_roi']
+        channel_roi_df.at[i, 'Optimized ROI'] = channel_roi_mroi[channel]['optimized_roi']
+    channel_roi_df['Actual ROI'] = channel_roi_df['Actual ROI'].astype('float')
+    channel_roi_df['Optimized ROI'] = channel_roi_df['Optimized ROI'].astype('float')
+
+    for col in channel_roi_df.columns:
+        channel_roi_df[col] = channel_roi_df[col].apply(lambda x: round_off(x, 2))
+
+    # Create chart data
+    chart_data = CategoryChartData()
+    chart_data.categories = channel_roi_df['Channel']
+    for col in ['Actual ROI', 'Optimized ROI']:
+        chart_data.add_series(col, channel_roi_df[col])
+
+    chart_placeholder = slide.placeholders[ph_idx[1]]
+
+    # Add the chart to the slide
+    if isinstance(channel_roi_df.select_dtypes(exclude=['object', 'datetime']), float):
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Channel Wise ROI',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'ROI'},
+                  # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+                  min_y=0,
+                  max_y=np.max(channel_roi_df.select_dtypes(exclude=['object', 'datetime']))
+                  )
+    else:
+        bar_chart(chart_placeholder=chart_placeholder,
+                  slide=slide,
+                  chart_data=chart_data,
+                  titles={'chart_title': 'Channel Wise ROI',
+                          # 'x_axis':'Channels',
+                          'y_axis': 'ROI'},
+                  # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+                  min_y=0,
+                  max_y=np.max(channel_roi_df.select_dtypes(exclude=['object', 'datetime'])).values[0]
+                  )
+    # act_roi = scenario['actual_total_sales']/scenario['actual_total_spends']
+    # opt_roi = scenario['modified_total_sales']/scenario['modified_total_spends']
+    #
+    # act_roi_ph = slide.placeholders[ph_idx[2]]
+    # act_roi_ph.text = 'Actual ROI: ' + str(round_off(act_roi,2))
+    # opt_roi_ph = slide.placeholders[ph_idx[3]]
+    # opt_roi_ph.text = 'Optimized ROI: ' + str(round_off(opt_roi, 2))
+
+    ## Removing mroi chart as per Ioannis' feedback
+    # channel_mroi_df = pd.DataFrame(columns=['Channel', 'Actual mROI', 'Optimized mROI'])
+    # for i, channel in enumerate(channel_roi_mroi.keys()):
+    #     channel_mroi_df.at[i, 'Channel'] = channel
+    #     channel_mroi_df.at[i, 'Actual mROI'] = channel_roi_mroi[channel]['actual_mroi']
+    #     channel_mroi_df.at[i, 'Optimized mROI'] = channel_roi_mroi[channel]['optimized_mroi']
+    # channel_mroi_df['Actual mROI']=channel_mroi_df['Actual mROI'].astype('float')
+    # channel_mroi_df['Optimized mROI']=channel_mroi_df['Optimized mROI'].astype('float')
+    #
+    # for col in channel_mroi_df.columns:
+    #     channel_mroi_df[col]=channel_mroi_df[col].apply(lambda x: round_off(x))
+    #
+    # # Create chart data
+    # mroi_chart_data = CategoryChartData()
+    # mroi_chart_data.categories = channel_mroi_df['Channel']
+    # for col in ['Actual mROI', 'Optimized mROI']:
+    #     mroi_chart_data.add_series(col, channel_mroi_df[col])
+    #
+    # mroi_chart_placeholder=slide.placeholders[ph_idx[2]]
+    #
+    # # Add the chart to the slide
+    # bar_chart(chart_placeholder=mroi_chart_placeholder,
+    #           slide=slide,
+    #           chart_data=mroi_chart_data,
+    #           titles={'chart_title':'Channel Wise mROI',
+    #                   # 'x_axis':'Channels',
+    #                   'y_axis':'mROI'},
+    #           # min_y=np.floor(np.min(channel_mroi_df.select_dtypes(exclude=['object', 'datetime']))),
+    #           min_y=0,
+    #           max_y=np.ceil(np.max(channel_mroi_df.select_dtypes(exclude=['object', 'datetime'])))
+    #           )
+
+
+def effictiveness_efficiency(slide, final_data, bin_dct, scenario):
+    # Add title
+    placeholders = slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Effectiveness and Efficiency'
+    title_ph.text_frame.paragraphs[0].font.size = Pt(TITLE_FONT_SIZE)
+
+    response_metrics = bin_dct['Response Metrics']
+
+    kpi_df = final_data[response_metrics].sum(axis=0).reset_index()
+    kpi_df.columns = ['Response Metric', 'Effectiveness']
+    kpi_df['Efficiency'] = kpi_df['Effectiveness'] / scenario['modified_total_spends']
+    kpi_df['Efficiency'] = kpi_df['Efficiency'].apply(lambda x: round_off(x, 1))
+    kpi_df.sort_values(by='Effectiveness', inplace=True)
+    kpi_df['Response Metric'] = kpi_df['Response Metric'].apply(lambda x: format_response_metric(x))
+
+    # Create chart data for effectiveness
+    chart_data = CategoryChartData()
+    chart_data.categories = kpi_df['Response Metric']
+    chart_data.add_series('Effectiveness', kpi_df['Effectiveness'])
+
+    chart_placeholder = slide.placeholders[ph_idx[1]]
+
+    # Add the chart to the slide
+    bar_chart(chart_placeholder=chart_placeholder,
+              slide=slide,
+              chart_data=chart_data,
+              titles={'chart_title': 'Effectiveness',
+                      # 'x_axis':'Channels',
+                      # 'y_axis': 'ROI'
+                      },
+              # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+              min_y=0,
+              # max_y=np.max(channel_roi_df.select_dtypes(exclude=['object', 'datetime'])),
+              type='H',
+              label_type='M'
+              )
+
+    # Create chart data for efficiency
+    chart_data_1 = CategoryChartData()
+    chart_data_1.categories = kpi_df['Response Metric']
+    chart_data_1.add_series('Efficiency', kpi_df['Efficiency'])
+
+    chart_placeholder_1 = slide.placeholders[ph_idx[2]]
+
+    # Add the chart to the slide
+    bar_chart(chart_placeholder=chart_placeholder_1,
+              slide=slide,
+              chart_data=chart_data_1,
+              titles={'chart_title': 'Efficiency',
+                      # 'x_axis':'Channels',
+                      # 'y_axis': 'ROI'
+                      },
+              # min_y=np.floor(np.min(channel_spends_df.select_dtypes(exclude=['object', 'datetime']))),
+              min_y=0,
+              # max_y=np.max(channel_roi_df.select_dtypes(exclude=['object', 'datetime'])),
+              type='H'
+              )
+
+    definition_ph_1 = slide.placeholders[ph_idx[3]]
+    definition_ph_1.text = 'Effectiveness is measured as the total sum of the Response Metric'
+    definition_ph_2 = slide.placeholders[ph_idx[4]]
+    definition_ph_2.text = 'Efficiency is measured as the ratio of sum of the Response Metric and sum of Media Spend'
+
+
+def load_pickle(path):
+    with open(path, "rb") as f:
+        file_data = pickle.load(f)
+    return file_data
+
+
+def read_all_files():
+    files=[]
+
+    # Read data and bin dictionary
+    if st.session_state["project_dct"]["data_import"]["imputed_tool_df"] is not None:
+        final_df_loaded = st.session_state["project_dct"]["data_import"]["imputed_tool_df"].copy()
+        bin_dict_loaded = st.session_state["project_dct"]["data_import"]["category_dict"].copy()
+
+        files.append(final_df_loaded)
+        files.append(bin_dict_loaded)
+
+        if "group_dict" in st.session_state["project_dct"]["data_import"].keys():
+            channels = st.session_state["project_dct"]["data_import"]["group_dict"]
+            files.append(channels)
+
+
+        if st.session_state["project_dct"]["transformations"]["final_df"] is not None:
+            transform_dict = st.session_state["project_dct"]["transformations"]
+            files.append(transform_dict)
+            if retrieve_pkl_object_without_warning(st.session_state['project_number'], "Model_Tuning", "tuned_model", schema) is not None:
+                tuned_model_dict = retrieve_pkl_object_without_warning(st.session_state['project_number'], "Model_Tuning",
+                                                       "tuned_model", schema)  # db
+
+                files.append(tuned_model_dict)
+            else:
+                files.append(None)
+        else:
+            files.append(None)
+
+        if len(list(st.session_state["project_dct"]["current_media_performance"]["model_outputs"].keys()))>0: # check if there are model outputs for at least one metric
+            metrics_list = list(st.session_state["project_dct"]["current_media_performance"]["model_outputs"].keys())
+            contributions_excels_dict = {}
+            for metrics in metrics_list:
+                # raw_df = st.session_state["project_dct"]["current_media_performance"]["model_outputs"][metrics]["raw_data"]
+                # spend_df = st.session_state["project_dct"]["current_media_performance"]["model_outputs"][metrics]["spends_data"]
+                contribution_df = st.session_state["project_dct"]["current_media_performance"]["model_outputs"][metrics]["contribution_data"]
+                contributions_excels_dict[metrics] = {'CONTRIBUTION MMM':contribution_df}
+            files.append(contributions_excels_dict)
+
+            # Get Saved Scenarios
+            if len(list(st.session_state["project_dct"]["saved_scenarios"]["saved_scenarios_dict"].keys()))>0:
+                files.append(st.session_state["project_dct"]["saved_scenarios"]["saved_scenarios_dict"])
+
+                    # saved_scenarios_loaded = get_saved_scenarios_dict(project_path)
+
+
+    return files
+
+
+
+'''
+
+    Template Layout
+
+    0 : Title
+    1 : Data Details Section {no changes required}
+    2 : Data Import
+    3 : Data Import - Channel Groups
+    4 : Model Results {Duplicate for each model}  
+    5 : Metrics Contribution
+    6 : Media performance {Duplicate for each model} 
+    7 : Media performance Tabular View {Duplicate for each model} 
+    8 : Optimization Section {no changes}
+    9 : Optimization Summary {Duplicate for each section}
+    10 : Channel Spends {Duplicate for each model}
+    11 : Channel Wise ROI {Duplicate for each model}
+    12 : Efficiency & Efficacy
+    13 : Appendix
+    14 : Transformations
+    15 : Model Summary
+    16 : Thank You Slide
+
+'''
+
+
+def create_ppt(project_name, username, panel_col):
+    # Read saved files
+    files = read_all_files()
+    transform_dict, tuned_model_dict, contributions_excels_dict, saved_scenarios_loaded = None, None, None, None
+
+    if len(files)>0:
+        # saved_data = files[0]
+        data = files[0]
+        bin_dict = files[1]
+
+        channel_groups_dct = files[2]
+        try:
+            transform_dict = files[3]
+            tuned_model_dict = files[4]
+            contributions_excels_dict = files[5]
+            saved_scenarios_loaded = files[6]
+        except Exception as e:
+            print(e)
+
+    else:
+        return False
+
+    is_panel = True if data[panel_col].nunique()>1 else False
+
+    template_path = 'ppt/template.pptx'
+    # ppt_path = os.path.join('ProjectSummary.pptx')
+
+    prs = Presentation(template_path)
+    num_slides = len(prs.slides)
+    slides = prs.slides
+
+    # Title Slide
+    title_slide_layout = slides[0].slide_layout
+    title_slide = prs.slides.add_slide(title_slide_layout)
+
+    # Add title & project name
+    placeholders = title_slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = title_slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Media Mix Optimization Summary'
+    txt_ph = title_slide.placeholders[ph_idx[1]]
+    txt_ph.text = 'Project Name: ' + project_name + '\nCreated By: ' + username
+
+    # Model Details Section
+    model_section_slide_layout = slides[1].slide_layout
+    model_section_slide = prs.slides.add_slide(model_section_slide_layout)
+
+    ## Add title
+    placeholders = model_section_slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = model_section_slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Model Details'
+    section_ph = model_section_slide.placeholders[ph_idx[1]]
+    section_ph.text = 'Section 1'
+
+    # Data Import
+    data_import_slide_layout = slides[2].slide_layout
+    data_import_slide = prs.slides.add_slide(data_import_slide_layout)
+    data_import_slide = title_and_table(slide=data_import_slide,
+                                        title='Data Import',
+                                        df=data_import(data, bin_dict),
+                                        column_width={0: 2, 1: 7}
+                                        )
+
+    # Channel Groups
+    channel_group_slide_layout = slides[3].slide_layout
+    channel_group_slide = prs.slides.add_slide(channel_group_slide_layout)
+    channel_group_slide = title_and_table(slide=channel_group_slide,
+                                          title='Channels - Media and Spend',
+                                          df=channel_groups_df(channel_groups_dct, bin_dict),
+                                          column_width={0: 2, 1: 5, 2: 2}
+                                          )
+
+    if tuned_model_dict is not None:
+        model_metrics_df = model_metrics(tuned_model_dict, False)
+
+        # Model Results
+        for model_key, model_dict in tuned_model_dict.items():
+            model_result_slide_layout = slides[4].slide_layout
+            model_result_slide = prs.slides.add_slide(model_result_slide_layout)
+            model_result_slide = model_result(slide=model_result_slide,
+                                              model_key=model_key,
+                                              model_dict=model_dict,
+                                              model_metrics_df=model_metrics_df,
+                                              date_col='date')
+
+    if contributions_excels_dict is not None:
+
+        # Metrics Contributions
+        metrics_contributions_slide_layout = slides[5].slide_layout
+        metrics_contributions_slide = prs.slides.add_slide(metrics_contributions_slide_layout)
+        metrics_contributions_slide = metrics_contributions(slide=metrics_contributions_slide,
+                                                            contributions_excels_dict=contributions_excels_dict,
+                                                            panel_col=panel_col
+                                                            )
+
+        # Media Performance
+        for target in contributions_excels_dict.keys():
+
+            # Chart
+            model_media_perf_slide_layout = slides[6].slide_layout
+            model_media_perf_slide = prs.slides.add_slide(model_media_perf_slide_layout)
+            contribution_df, spends_df = model_media_performance(slide=model_media_perf_slide,
+                                                                 target=target,
+                                                                 contributions_excels_dict=contributions_excels_dict
+                                                                 )
+
+            # Tabular View
+            contri_spends_df = pd.merge(spends_df, contribution_df, on='Channel', how='outer')
+            contri_spends_df.fillna(0, inplace=True)
+
+            for col in [c for c in contri_spends_df.columns if c != 'Channel']:
+                contri_spends_df[col] = contri_spends_df[col].apply(lambda x: safe_num_to_per(x))
+
+            media_performance_table_slide_layout = slides[7].slide_layout
+            media_performance_table_slide = prs.slides.add_slide(media_performance_table_slide_layout)
+            media_performance_table_slide = title_and_table(slide=media_performance_table_slide,
+                                                            title='Media and Spends Channels Tabular View',
+                                                            df=contri_spends_df,
+                                                            # column_width={0:2, 1:5, 2:2}
+                                                            )
+
+    if saved_scenarios_loaded is not None:
+        # Optimization Details
+        opt_section_slide_layout = slides[8].slide_layout
+        opt_section_slide = prs.slides.add_slide(opt_section_slide_layout)
+
+        ## Add title
+        placeholders = opt_section_slide.placeholders
+        ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+        title_ph = opt_section_slide.placeholders[ph_idx[0]]
+        title_ph.text = 'Optimizations Details'
+        section_ph = opt_section_slide.placeholders[ph_idx[1]]
+        section_ph.text = 'Section 2'
+
+        # Optimization
+        for scenario_name, scenario in saved_scenarios_loaded.items():
+            opt_summary_slide_layout = slides[9].slide_layout
+            opt_summary_slide = prs.slides.add_slide(opt_summary_slide_layout)
+            optimization_summary(opt_summary_slide, scenario, scenario_name)
+
+            channel_spends_slide_layout = slides[10].slide_layout
+            channel_spends_slide = prs.slides.add_slide(channel_spends_slide_layout)
+            channel_wise_spends(channel_spends_slide, scenario)
+
+            channel_roi_slide_layout = slides[11].slide_layout
+            channel_roi_slide = prs.slides.add_slide(channel_roi_slide_layout)
+            channel_wise_roi(channel_roi_slide, scenario)
+
+            effictiveness_efficiency_slide_layout = slides[12].slide_layout
+            effictiveness_efficiency_slide = prs.slides.add_slide(effictiveness_efficiency_slide_layout)
+            effictiveness_efficiency(effictiveness_efficiency_slide,
+                                     data,
+                                     bin_dict,
+                                     scenario)
+
+    # Appendix Section
+    appendix_section_slide_layout = slides[13].slide_layout
+    appendix_section_slide = prs.slides.add_slide(appendix_section_slide_layout)
+
+    if tuned_model_dict is not None:
+
+        ## Add title
+        placeholders = appendix_section_slide.placeholders
+        ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+        title_ph = appendix_section_slide.placeholders[ph_idx[0]]
+        title_ph.text = 'Appendix'
+        section_ph = appendix_section_slide.placeholders[ph_idx[1]]
+        section_ph.text = 'Section 3'
+
+        # Add transformations
+        # if transform_dict is not None:
+        #     # Transformations
+        #     transformation_slide_layout = slides[14].slide_layout
+        #     transformation_slide = prs.slides.add_slide(transformation_slide_layout)
+        #     transformation_slide = title_and_table(slide=transformation_slide,
+        #                                            title='Transformations',
+        #                                            df=transformations(transform_dict),
+        #                                            custom_table_height=True
+        #                                            )
+
+        # Add model summary
+        # Model Summary
+        model_metrics_df = model_metrics(tuned_model_dict, False)
+        model_summary_slide_layout = slides[15].slide_layout
+        model_summary_slide = prs.slides.add_slide(model_summary_slide_layout)
+        model_summary_slide = title_and_table(slide=model_summary_slide,
+                                              title='Model Summary',
+                                              df=model_metrics_df,
+                                              custom_table_height=True
+                                              )
+
+    # Last Slide
+    last_slide_layout = slides[num_slides - 1].slide_layout
+    last_slide = prs.slides.add_slide(last_slide_layout)
+
+    # Add title
+    placeholders = last_slide.placeholders
+    ph_idx = [ph.placeholder_format.idx for ph in placeholders]
+    title_ph = last_slide.placeholders[ph_idx[0]]
+    title_ph.text = 'Thank You'
+
+    # Remove template slides
+    xml_slides = prs.slides._sldIdLst
+    slides = list(xml_slides)
+    for index in range(num_slides):
+        xml_slides.remove(slides[index])
+
+    # prs.save(ppt_path)
+
+    # save the output into binary form
+    binary_output = BytesIO()
+    prs.save(binary_output)
+
+    return binary_output

requirements.txt

@@ -0,0 +1,96 @@
+altair	==			4.2.0
+attrs	==			23.1.0
+bcrypt	==			4.0.1
+blinker	==			1.6.2
+cachetools	==		5.3.1
+certifi	==			2023.7.22
+charset-normalizer	==	3.2.0
+click	==			8.1.7
+colorama	==		0.4.6
+contourpy	==		1.1.1
+cycler	==			0.11.0
+dacite	==			1.8.1
+entrypoints	==		0.4
+et-xmlfile	==		1.1.0
+extra-streamlit-components ==	0.1.56
+fonttools	==		4.42.1
+gitdb	==			4.0.10
+GitPython	==		3.1.35
+htmlmin	==			0.1.12
+idna	==			3.4
+ImageHash	==		4.3.1
+importlib-metadata	==	6.8.0
+importlib-resources	==	6.1.0
+Jinja2	==			3.1.2
+joblib	==			1.3.2
+jsonschema	==		4.19.0
+jsonschema-specifications==	2023.7.1
+kaleido	==			0.2.1
+kiwisolver	==		1.4.5
+markdown-it-py	==		3.0.0
+MarkupSafe	==		2.1.3
+matplotlib	==		3.7.0
+mdurl	==			0.1.2
+networkx	==		3.1
+numerize	==		0.12
+numpy	==			1.23.5
+openpyxl>=3.1.0
+packaging	==		23.1
+pandas	==			1.5.2
+pandas-profiling	==	3.6.6
+patsy	==			0.5.3
+phik	==			0.12.3
+Pillow	==			10.0.0
+pip	==			23.2.1
+plotly	==			5.11.0
+protobuf	==		3.20.3
+pyarrow	==			13.0.0
+pydantic	==		1.10.13
+pydeck	==			0.8.1b0
+Pygments	==		2.16.1
+PyJWT	==			2.8.0
+Pympler	==			1.0.1
+pyparsing	==		3.1.1
+python-dateutil	==		2.8.2
+python-decouple	==		3.8
+pytz	==			2023.3.post1
+PyWavelets	==		1.4.1
+PyYAML	==			6.0.1
+referencing	==		0.30.2
+requests	==		2.31.0
+rich	==			13.5.2
+rpds-py	==			0.10.2
+scikit-learn	==		1.1.3
+scipy	==			1.9.3
+seaborn	==			0.12.2
+semver	==			3.0.1
+setuptools	==		68.1.2
+six	==			1.16.0
+smmap	==			5.0.0
+statsmodels	==		0.14.0
+streamlit	==		1.28.0
+streamlit-aggrid	==	0.3.4.post3
+streamlit-authenticator	==	0.2.1
+streamlit-pandas-profiling==	0.1.3
+sweetviz	==		2.2.1
+tangled-up-in-unicode	==	0.2.0
+tenacity	==		8.2.3
+threadpoolctl	==		3.2.0
+toml	==			0.10.2
+toolz	==			0.12.0
+tornado	==			6.3.3
+tqdm	==			4.66.1
+typeguard	==		2.13.3
+typing_extensions	==	4.7.1
+tzdata	==			2023.3
+tzlocal	==			5.0.1
+urllib3	==			2.0.4
+validators	==		0.22.0
+visions	==			0.7.5
+watchdog	==		3.0.0
+wheel	==			0.41.2
+wordcloud	==		1.9.2
+ydata-profiling	==		4.5.1
+zipp	==			3.16.2
+psycopg2 == 2.9.9
+python-pptx == 0.6.21

scenario.py

@@ -0,0 +1,763 @@
+import numpy as np
+from scipy.optimize import minimize, LinearConstraint, NonlinearConstraint
+from collections import OrderedDict
+import pandas as pd
+from decimal import Decimal
+
+
+def round_num(n, decimal=2):
+    n = Decimal(n)
+    return n.to_integral() if n == n.to_integral() else round(n.normalize(), decimal)
+
+
+def numerize(n, decimal=2):
+    # 60 sufixes
+    sufixes = [
+        "",
+        "K",
+        "M",
+        "B",
+        "T",
+        "Qa",
+        "Qu",
+        "S",
+        "Oc",
+        "No",
+        "D",
+        "Ud",
+        "Dd",
+        "Td",
+        "Qt",
+        "Qi",
+        "Se",
+        "Od",
+        "Nd",
+        "V",
+        "Uv",
+        "Dv",
+        "Tv",
+        "Qv",
+        "Qx",
+        "Sx",
+        "Ox",
+        "Nx",
+        "Tn",
+        "Qa",
+        "Qu",
+        "S",
+        "Oc",
+        "No",
+        "D",
+        "Ud",
+        "Dd",
+        "Td",
+        "Qt",
+        "Qi",
+        "Se",
+        "Od",
+        "Nd",
+        "V",
+        "Uv",
+        "Dv",
+        "Tv",
+        "Qv",
+        "Qx",
+        "Sx",
+        "Ox",
+        "Nx",
+        "Tn",
+        "x",
+        "xx",
+        "xxx",
+        "X",
+        "XX",
+        "XXX",
+        "END",
+    ]
+
+    sci_expr = [
+        1e0,
+        1e3,
+        1e6,
+        1e9,
+        1e12,
+        1e15,
+        1e18,
+        1e21,
+        1e24,
+        1e27,
+        1e30,
+        1e33,
+        1e36,
+        1e39,
+        1e42,
+        1e45,
+        1e48,
+        1e51,
+        1e54,
+        1e57,
+        1e60,
+        1e63,
+        1e66,
+        1e69,
+        1e72,
+        1e75,
+        1e78,
+        1e81,
+        1e84,
+        1e87,
+        1e90,
+        1e93,
+        1e96,
+        1e99,
+        1e102,
+        1e105,
+        1e108,
+        1e111,
+        1e114,
+        1e117,
+        1e120,
+        1e123,
+        1e126,
+        1e129,
+        1e132,
+        1e135,
+        1e138,
+        1e141,
+        1e144,
+        1e147,
+        1e150,
+        1e153,
+        1e156,
+        1e159,
+        1e162,
+        1e165,
+        1e168,
+        1e171,
+        1e174,
+        1e177,
+    ]
+    minus_buff = n
+    n = abs(n)
+
+    if n < 1:
+        return f"{round(n/1000, decimal)}K"
+
+    for x in range(len(sci_expr)):
+        try:
+            if n >= sci_expr[x] and n < sci_expr[x + 1]:
+                sufix = sufixes[x]
+                if n >= 1e3:
+                    num = str(round_num(n / sci_expr[x], decimal))
+                else:
+                    num = str(round_num(n, decimal))
+                return num + sufix if minus_buff > 0 else "-" + num + sufix
+        except IndexError:
+            pass
+
+
+def class_to_dict(class_instance):
+    attr_dict = {}
+    if isinstance(class_instance, Channel):
+        attr_dict["type"] = "Channel"
+        attr_dict["name"] = class_instance.name
+        attr_dict["dates"] = class_instance.dates
+        attr_dict["spends"] = class_instance.actual_spends
+        attr_dict["conversion_rate"] = class_instance.conversion_rate
+        attr_dict["modified_spends"] = class_instance.modified_spends
+        attr_dict["modified_sales"] = class_instance.modified_sales
+        attr_dict["response_curve_type"] = class_instance.response_curve_type
+        attr_dict["response_curve_params"] = class_instance.response_curve_params
+        attr_dict["penalty"] = class_instance.penalty
+        attr_dict["bounds"] = class_instance.bounds
+        attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+        attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+        attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+        attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+        attr_dict["actual_mroi"] = class_instance.get_marginal_roi("actual")
+        attr_dict["modified_mroi"] = class_instance.get_marginal_roi("modified")
+        attr_dict["freeze"] = class_instance.freeze
+        attr_dict["correction"] = class_instance.correction
+
+    elif isinstance(class_instance, Scenario):
+        attr_dict["type"] = "Scenario"
+        attr_dict["name"] = class_instance.name
+        attr_dict["bounds"] = class_instance.bounds
+        channels = []
+        for channel in class_instance.channels.values():
+            channels.append(class_to_dict(channel))
+        attr_dict["channels"] = channels
+        attr_dict["constant"] = class_instance.constant
+        attr_dict["correction"] = class_instance.correction
+        attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+        attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+        attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+        attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+
+    return attr_dict
+
+
+# def class_convert_to_dict(class_instance):
+#     attr_dict = {}
+#     if isinstance(class_instance, Channel):
+#         attr_dict["type"] = "Channel"
+#         attr_dict["name"] = class_instance.name
+#         attr_dict["dates"] = class_instance.dates
+#         attr_dict["spends"] = class_instance.actual_spends
+#         attr_dict["conversion_rate"] = class_instance.conversion_rate
+#         attr_dict["modified_spends"] = class_instance.modified_spends
+#         attr_dict["modified_sales"] = class_instance.modified_sales
+#         attr_dict["response_curve_type"] = class_instance.response_curve_type
+#         attr_dict["response_curve_params"] = class_instance.response_curve_params
+#         # attr_dict["penalty"] = class_instance.penalty
+#         attr_dict["bounds"] = class_instance.bounds
+#         attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+#         attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+#         attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+#         attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+#         # attr_dict["actual_mroi"] = class_instance.get_marginal_roi("actual")
+#         # attr_dict["modified_mroi"] = class_instance.get_marginal_roi("modified")
+
+#         attr_dict["freeze"] = class_instance.freeze
+#         attr_dict["correction"] = class_instance.correction
+
+#     elif isinstance(class_instance, Scenario):
+#         attr_dict["type"] = "Scenario"
+#         attr_dict["name"] = class_instance.name
+#         channels = {}
+#         for channel in class_instance.channels.values():
+#             channels[channel.name] = class_to_dict(channel)
+#         attr_dict["channels"] = channels
+#         attr_dict["constant"] = class_instance.constant
+#         attr_dict["correction"] = class_instance.correction
+#         attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+#         attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+#         attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+#         attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+
+#         attr_dict["bound_type"] = class_instance.bound_type
+
+#         attr_dict["bounds"] = class_instance.bounds
+
+#     return attr_dict.copy()
+
+# Function to convert class instance to dictionary
+def class_convert_to_dict(class_instance):
+    attr_dict = {}
+
+    if isinstance(class_instance, Channel):
+        # Convert Channel instance to dictionary
+        attr_dict["type"] = "Channel"
+        attr_dict["name"] = class_instance.name
+        attr_dict["dates"] = class_instance.dates
+        attr_dict["spends"] = list(class_instance.actual_spends)
+        attr_dict["conversion_rate"] = class_instance.conversion_rate
+        attr_dict["modified_spends"] = list(class_instance.modified_spends)
+        attr_dict["modified_sales"] = list(class_instance.modified_sales)
+        attr_dict["response_curve_type"] = class_instance.response_curve_type
+        attr_dict["response_curve_params"] = class_instance.response_curve_params.copy()
+        attr_dict["bounds"] = class_instance.bounds.copy()
+        attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+        attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+        attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+        attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+        attr_dict["freeze"] = class_instance.freeze
+        attr_dict["correction"] = class_instance.correction.copy()
+
+    elif isinstance(class_instance, Scenario):
+        # Convert Scenario instance to dictionary
+        attr_dict["type"] = "Scenario"
+        attr_dict["name"] = class_instance.name
+
+        channels = {}
+        for channel in class_instance.channels.values():
+            channels[channel.name] = class_convert_to_dict(channel)
+        attr_dict["channels"] = channels
+
+        attr_dict["constant"] = list(class_instance.constant)
+        attr_dict["correction"] = list(class_instance.correction)
+        attr_dict["actual_total_spends"] = class_instance.actual_total_spends
+        attr_dict["actual_total_sales"] = class_instance.actual_total_sales
+        attr_dict["modified_total_spends"] = class_instance.modified_total_spends
+        attr_dict["modified_total_sales"] = class_instance.modified_total_sales
+        attr_dict["bound_type"] = class_instance.bound_type
+        attr_dict["bounds"] = class_instance.bounds.copy()
+
+    return attr_dict
+
+
+def class_from_dict(attr_dict):
+    if attr_dict["type"] == "Channel":
+        return Channel.from_dict(attr_dict)
+    elif attr_dict["type"] == "Scenario":
+        return Scenario.from_dict(attr_dict)
+
+
+class Channel:
+    def __init__(
+        self,
+        name,
+        dates,
+        spends,
+        response_curve_type,
+        response_curve_params,
+        bounds,
+        correction,
+        conversion_rate=1,
+        modified_spends=None,
+        penalty=True,
+        freeze=False,
+    ):
+        self.name = name
+        self.dates = dates
+        self.conversion_rate = conversion_rate
+        self.actual_spends = spends.copy()
+        self.correction = correction
+
+        if modified_spends is None:
+            self.modified_spends = self.actual_spends.copy()
+        else:
+            self.modified_spends = modified_spends
+
+        self.response_curve_type = response_curve_type
+        self.response_curve_params = response_curve_params
+        self.bounds = bounds
+        self.penalty = penalty
+        self.freeze = freeze
+
+        self.upper_limit = self.actual_spends.max() + self.actual_spends.std()
+        self.power = np.ceil(np.log(self.actual_spends.max()) / np.log(10)) - 3
+        # self.actual_sales = None
+        # self.actual_sales = self.response_curve(self.actual_spends)
+        self.actual_total_spends = self.actual_spends.sum()
+        self.actual_total_sales = self.actual_sales.sum()
+        self.modified_sales = self.calculate_sales()
+        self.modified_total_spends = self.modified_spends.sum()
+        self.modified_total_sales = self.modified_sales.sum()
+        self.delta_spends = self.modified_total_spends - self.actual_total_spends
+        self.delta_sales = self.modified_total_sales - self.actual_total_sales
+
+    @property
+    def actual_sales(self):
+        return self.response_curve(self.actual_spends) + self.correction
+
+    def update_penalty(self, penalty):
+        self.penalty = penalty
+
+    def _modify_spends(self, spends_array, total_spends):
+        return spends_array * total_spends / spends_array.sum()
+
+    def modify_spends(self, total_spends):
+        self.modified_spends = (
+            self.modified_spends * total_spends / self.modified_spends.sum()
+        )
+
+    def calculate_sales(self):
+        return self.response_curve(self.modified_spends) + self.correction
+
+    def response_curve(self, x):
+        if self.penalty:
+            x = np.where(
+                x < self.upper_limit,
+                x,
+                self.upper_limit + (x - self.upper_limit) * self.upper_limit / x,
+            )
+        if self.response_curve_type == "s-curve":
+            if self.power >= 0:
+                x = x / 10**self.power
+            x = x.astype("float64")
+            K = self.response_curve_params["K"]
+            b = self.response_curve_params["b"]
+            a = self.response_curve_params["a"]
+            x0 = self.response_curve_params["x0"]
+            sales = K / (1 + b * np.exp(-a * (x - x0)))
+        if self.response_curve_type == "linear":
+            beta = self.response_curve_params["beta"]
+            sales = beta * x
+
+        return sales
+
+    def get_marginal_roi(self, flag):
+        K = self.response_curve_params["K"]
+        a = self.response_curve_params["a"]
+        # x = self.modified_total_spends
+        # if self.power >= 0 :
+        #     x = x / 10**self.power
+        # x = x.astype('float64')
+        # return K*b*a*np.exp(-a*(x-x0)) / (1 + b * np.exp(-a*(x - x0)))**2
+        if flag == "actual":
+            y = self.response_curve(self.actual_spends)
+            # spends_array = self.actual_spends
+            # total_spends = self.actual_total_spends
+            # total_sales = self.actual_total_sales
+
+        else:
+            y = self.response_curve(self.modified_spends)
+            # spends_array = self.modified_spends
+            # total_spends = self.modified_total_spends
+            # total_sales = self.modified_total_sales
+
+        # spends_inc_1 = self._modify_spends(spends_array, total_spends+1)
+        mroi = a * (y) * (1 - y / K)
+        return mroi.sum() / len(self.modified_spends)
+        # spends_inc_1 = self.spends_array + 1
+        # new_total_sales = self.response_curve(spends_inc_1).sum()
+        # return (new_total_sales - total_sales) / len(self.modified_spends)
+
+    def update(self, total_spends):
+        self.modify_spends(total_spends)
+        self.modified_sales = self.calculate_sales()
+        self.modified_total_spends = self.modified_spends.sum()
+        self.modified_total_sales = self.modified_sales.sum()
+        self.delta_spends = self.modified_total_spends - self.actual_total_spends
+        self.delta_sales = self.modified_total_sales - self.actual_total_sales
+
+    def intialize(self):
+        self.new_spends = self.old_spends
+
+    def __str__(self):
+        return f"{self.name},{self.actual_total_sales}, {self.modified_total_spends}"
+
+    @classmethod
+    def from_dict(cls, attr_dict):
+        return Channel(
+            name=attr_dict["name"],
+            dates=attr_dict["dates"],
+            spends=attr_dict["spends"],
+            bounds=attr_dict["bounds"],
+            modified_spends=attr_dict["modified_spends"],
+            response_curve_type=attr_dict["response_curve_type"],
+            response_curve_params=attr_dict["response_curve_params"],
+            penalty=attr_dict["penalty"],
+            correction=attr_dict["correction"],
+        )
+
+    def update_response_curves(self, response_curve_params):
+        self.response_curve_params = response_curve_params
+
+
+class Scenario:
+    def __init__(
+        self, name, channels, constant, correction, bound_type=False, bounds=[-10, 10]
+    ):
+        self.name = name
+        self.channels = channels
+        self.constant = constant
+        self.correction = correction
+        self.bound_type = bound_type
+        self.bounds = bounds
+
+        self.actual_total_spends = self.calculate_modified_total_spends()
+        self.actual_total_sales = self.calculate_actual_total_sales()
+        self.modified_total_sales = self.calculate_modified_total_sales()
+        self.modified_total_spends = self.calculate_modified_total_spends()
+        self.delta_spends = self.modified_total_spends - self.actual_total_spends
+        self.delta_sales = self.modified_total_sales - self.actual_total_sales
+
+    def update_penalty(self, value):
+        for channel in self.channels.values():
+            channel.update_penalty(value)
+
+    def calculate_modified_total_spends(self):
+        total_actual_spends = 0.0
+        for channel in self.channels.values():
+            total_actual_spends += channel.actual_total_spends * channel.conversion_rate
+        return total_actual_spends
+
+    def calculate_modified_total_spends(self):
+        total_modified_spends = 0.0
+        for channel in self.channels.values():
+            # import streamlit as st
+            # st.write(channel.modified_total_spends )
+            total_modified_spends += (
+                channel.modified_total_spends * channel.conversion_rate
+            )
+        return total_modified_spends
+
+    def calculate_actual_total_sales(self):
+        total_actual_sales = self.constant.sum() + self.correction.sum()
+        for channel in self.channels.values():
+            total_actual_sales += channel.actual_total_sales
+        return total_actual_sales
+
+    def calculate_modified_total_sales(self):
+        total_modified_sales = self.constant.sum() + self.correction.sum()
+        for channel in self.channels.values():
+            total_modified_sales += channel.modified_total_sales
+        return total_modified_sales
+
+    def update(self, channel_name, modified_spends):
+        self.channels[channel_name].update(modified_spends)
+        self.modified_total_sales = self.calculate_modified_total_sales()
+        self.modified_total_spends = self.calculate_modified_total_spends()
+        self.delta_spends = self.modified_total_spends - self.actual_total_spends
+        self.delta_sales = self.modified_total_sales - self.actual_total_sales
+
+    # def optimize_spends(self, sales_percent, channels_list, algo="COBYLA"):
+    #     desired_sales = self.actual_total_sales * (1 + sales_percent / 100.0)
+
+    #     def constraint(x):
+    #         for ch, spends in zip(channels_list, x):
+    #             self.update(ch, spends)
+    #         return self.modified_total_sales - desired_sales
+
+    #     bounds = []
+    #     for ch in channels_list:
+    #         bounds.append(
+    #             (1 + np.array([-50.0, 100.0]) / 100.0)
+    #             * self.channels[ch].actual_total_spends
+    #         )
+
+    #     initial_point = []
+    #     for bound in bounds:
+    #         initial_point.append(bound[0])
+
+    #     power = np.ceil(np.log(sum(initial_point)) / np.log(10))
+
+    #     constraints = [NonlinearConstraint(constraint, -1.0, 1.0)]
+
+    #     res = minimize(
+    #         lambda x: sum(x) / 10 ** (power),
+    #         bounds=bounds,
+    #         x0=initial_point,
+    #         constraints=constraints,
+    #         method=algo,
+    #         options={"maxiter": int(2e7), "catol": 1},
+    #     )
+
+    #     for channel_name, modified_spends in zip(channels_list, res.x):
+    #         self.update(channel_name, modified_spends)
+
+    #     return zip(channels_list, res.x)
+
+    def optimize_spends(self, sales_percent, channels_list, algo="trust-constr"):
+        desired_sales = self.actual_total_sales * (1 + sales_percent / 100.0)
+
+        def constraint(x):
+            for ch, spends in zip(channels_list, x):
+                self.update(ch, spends)
+            return self.modified_total_sales - desired_sales
+
+        bounds = []
+        for ch in channels_list:
+            bounds.append(
+                (1 + np.array([-50.0, 100.0]) / 100.0)
+                * self.channels[ch].actual_total_spends
+            )
+
+        initial_point = []
+        for bound in bounds:
+            initial_point.append(bound[0])
+
+        power = np.ceil(np.log(sum(initial_point)) / np.log(10))
+
+        constraints = [NonlinearConstraint(constraint, -1.0, 1.0)]
+
+        res = minimize(
+            lambda x: sum(x) / 10 ** (power),
+            bounds=bounds,
+            x0=initial_point,
+            constraints=constraints,
+            method=algo,
+            options={"maxiter": int(2e7), "xtol": 100},
+        )
+
+        for channel_name, modified_spends in zip(channels_list, res.x):
+            self.update(channel_name, modified_spends)
+
+        return zip(channels_list, res.x)
+
+    def optimize(self, spends_percent, channels_list):
+        # channels_list = self.channels.keys()
+        num_channels = len(channels_list)
+        spends_constant = []
+        spends_constraint = 0.0
+        for channel_name in channels_list:
+            # spends_constraint += self.channels[channel_name].modified_total_spends
+            spends_constant.append(self.channels[channel_name].conversion_rate)
+            spends_constraint += (
+                self.channels[channel_name].actual_total_spends
+                * self.channels[channel_name].conversion_rate
+            )
+        spends_constraint = spends_constraint * (1 + spends_percent / 100)
+        # constraint= LinearConstraint(np.ones((num_channels,)), lb = spends_constraint, ub = spends_constraint)
+        constraint = LinearConstraint(
+            np.array(spends_constant),
+            lb=spends_constraint,
+            ub=spends_constraint,
+        )
+        bounds = []
+        old_spends = []
+        for channel_name in channels_list:
+            _channel_class = self.channels[channel_name]
+            channel_bounds = _channel_class.bounds
+            channel_actual_total_spends = _channel_class.actual_total_spends * (
+                (1 + spends_percent / 100)
+            )
+            old_spends.append(channel_actual_total_spends)
+            bounds.append((1 + channel_bounds / 100) * channel_actual_total_spends)
+
+        def objective_function(x):
+            for channel_name, modified_spends in zip(channels_list, x):
+                self.update(channel_name, modified_spends)
+            return -1 * self.modified_total_sales
+
+        power = np.ceil(np.log(self.modified_total_sales) / np.log(10))
+  
+        res = minimize(
+            lambda x: objective_function(x) / 10 ** (power - 1),
+            method="trust-constr",
+            x0=old_spends,
+            constraints=constraint,
+            bounds=bounds,
+            options={"maxiter": int(1e7), "xtol": 0.1},
+        )
+        # res = dual_annealing(
+        # objective_function,
+        # x0=old_spends,
+        # mi
+        # constraints=constraint,
+        # bounds=bounds,
+        # tol=1e-16
+   
+        for channel_name, modified_spends in zip(channels_list, res.x):
+            self.update(channel_name, modified_spends)
+
+        return zip(channels_list, res.x)
+
+    def save(self):
+        details = {}
+        actual_list = []
+        modified_list = []
+        data = {}
+        channel_data = []
+
+        summary_rows = []
+        actual_list.append(
+            {
+                "name": "Total",
+                "Spends": self.actual_total_spends,
+                "Sales": self.actual_total_sales,
+            }
+        )
+        modified_list.append(
+            {
+                "name": "Total",
+                "Spends": self.modified_total_spends,
+                "Sales": self.modified_total_sales,
+            }
+        )
+        for channel in self.channels.values():
+            name_mod = channel.name.replace("_", " ")
+            if name_mod.lower().endswith(" imp"):
+                name_mod = name_mod.replace("Imp", " Impressions")
+            summary_rows.append(
+                [
+                    name_mod,
+                    channel.actual_total_spends,
+                    channel.modified_total_spends,
+                    channel.actual_total_sales,
+                    channel.modified_total_sales,
+                    round(
+                        channel.actual_total_sales / channel.actual_total_spends,
+                        2,
+                    ),
+                    round(
+                        channel.modified_total_sales / channel.modified_total_spends,
+                        2,
+                    ),
+                    channel.get_marginal_roi("actual"),
+                    channel.get_marginal_roi("modified"),
+                ]
+            )
+            data[channel.name] = channel.modified_spends
+            data["Date"] = channel.dates
+            data["Sales"] = (
+                data.get("Sales", np.zeros((len(channel.dates),)))
+                + channel.modified_sales
+            )
+            actual_list.append(
+                {
+                    "name": channel.name,
+                    "Spends": channel.actual_total_spends,
+                    "Sales": channel.actual_total_sales,
+                    "ROI": round(
+                        channel.actual_total_sales / channel.actual_total_spends,
+                        2,
+                    ),
+                }
+            )
+            modified_list.append(
+                {
+                    "name": channel.name,
+                    "Spends": channel.modified_total_spends,
+                    "Sales": channel.modified_total_sales,
+                    "ROI": round(
+                        channel.modified_total_sales / channel.modified_total_spends,
+                        2,
+                    ),
+                    "Marginal ROI": channel.get_marginal_roi("modified"),
+                }
+            )
+
+            channel_data.append(
+                {
+                    "channel": channel.name,
+                    "spends_act": channel.actual_total_spends,
+                    "spends_mod": channel.modified_total_spends,
+                    "sales_act": channel.actual_total_sales,
+                    "sales_mod": channel.modified_total_sales,
+                }
+            )
+        summary_rows.append(
+            [
+                "Total",
+                self.actual_total_spends,
+                self.modified_total_spends,
+                self.actual_total_sales,
+                self.modified_total_sales,
+                round(self.actual_total_sales / self.actual_total_spends, 2),
+                round(self.modified_total_sales / self.modified_total_spends, 2),
+                0.0,
+                0.0,
+            ]
+        )
+        details["Actual"] = actual_list
+        details["Modified"] = modified_list
+        columns_index = pd.MultiIndex.from_product(
+            [[""], ["Channel"]], names=["first", "second"]
+        )
+        columns_index = columns_index.append(
+            pd.MultiIndex.from_product(
+                [["Spends", "NRPU", "ROI", "MROI"], ["Actual", "Simulated"]],
+                names=["first", "second"],
+            )
+        )
+        details["Summary"] = pd.DataFrame(summary_rows, columns=columns_index)
+        data_df = pd.DataFrame(data)
+        channel_list = list(self.channels.keys())
+        data_df = data_df[["Date", *channel_list, "Sales"]]
+
+        details["download"] = {
+            "data_df": data_df,
+            "channels_df": pd.DataFrame(channel_data),
+            "total_spends_act": self.actual_total_spends,
+            "total_sales_act": self.actual_total_sales,
+            "total_spends_mod": self.modified_total_spends,
+            "total_sales_mod": self.modified_total_sales,
+        }
+
+        return details
+
+    @classmethod
+    def from_dict(cls, attr_dict):
+        channels_list = attr_dict["channels"]
+        channels = {
+            channel["name"]: class_from_dict(channel) for channel in channels_list
+        }
+        return Scenario(
+            name=attr_dict["name"],
+            channels=channels,
+            constant=attr_dict["constant"],
+            correction=attr_dict["correction"],
+        )

styles.css

@@ -0,0 +1,97 @@
+html {
+    margin: 0;
+}
+
+
+#MainMenu {
+    
+    visibility: collapse;
+}
+
+footer {
+    visibility: collapse;
+}
+
+div.block-container{
+    padding: 2rem 3rem;
+}
+
+div[data-testid="stExpander"]{
+    border: 1px solid '#739FAE';
+}
+
+a[data-testid="stPageLink-NavLink"]{
+    border: 1px solid rgba(0, 110, 192, 0.2);
+    margin: 0;
+    padding: 0.25rem 0.5rem;
+    border-radius: 0.5rem;
+}
+
+
+
+hr{
+    margin: 0;
+    padding: 0;
+}
+
+hr.spends-heading-seperator {
+    background-color : #11B6BD;
+    height: 2px;
+}
+
+.spends-header{
+    font-size: 1rem;
+    font-weight: bold;
+    margin: 0;
+
+}
+
+td {
+    max-width: 100px;
+    /* white-space:nowrap; */
+}
+
+.main-header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    
+}
+.blend-logo {
+    max-height: 64px;
+    /* max-width: 300px; */
+    object-fit: cover;
+}
+
+table {
+    width: 90%;
+}
+
+.lime-logo {
+    margin: 0;
+    padding: 0;
+    display: flex;
+    align-items: center ;
+    max-height: 64px;
+}
+
+.lime-text {
+    color: #00EDED;
+    font-size: 30px;
+    margin: 0;
+    padding: 0;
+    line-height: 0%;
+}
+
+.lime-img {
+    max-height: 30px;
+    /* max-height: 64px; */
+    /* max-width: 300px; */
+    object-fit: cover;
+}
+
+img {
+    margin: 0;
+    padding: 0;
+}

utilities.py

@@ -0,0 +1,2155 @@
+import streamlit as st
+import pandas as pd
+import json
+from scenario import Channel, Scenario
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from scenario import class_to_dict
+from collections import OrderedDict
+import io
+import plotly
+from pathlib import Path
+import pickle
+import yaml
+from yaml import SafeLoader
+from streamlit.components.v1 import html
+import smtplib
+from scipy.optimize import curve_fit
+from sklearn.metrics import r2_score
+from scenario import class_from_dict, class_convert_to_dict
+import os
+import base64
+import sqlite3
+import datetime
+from scenario import numerize
+import psycopg2
+
+#
+import re
+import bcrypt
+import os
+import json
+import glob
+import pickle
+import streamlit as st
+import streamlit as st
+import pandas as pd
+import json
+from scenario import Channel, Scenario
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from scenario import class_to_dict
+from collections import OrderedDict
+import io
+import plotly
+from pathlib import Path
+import pickle
+import yaml
+from yaml import SafeLoader
+from streamlit.components.v1 import html
+import smtplib
+from scipy.optimize import curve_fit
+from sklearn.metrics import r2_score
+from scenario import class_from_dict, class_convert_to_dict
+import os
+import base64
+import sqlite3
+import datetime
+from scenario import numerize
+import sqlite3
+
+# # schema = db_cred["schema"]
+
+color_palette = [
+    "#F3F3F0",
+    "#5E7D7E",
+    "#2FA1FF",
+    "#00EDED",
+    "#00EAE4",
+    "#304550",
+    "#EDEBEB",
+    "#7FBEFD",
+    "#003059",
+    "#A2F3F3",
+    "#E1D6E2",
+    "#B6B6B6",
+]
+
+
+CURRENCY_INDICATOR = "$"
+db_cred = None
+# database_file = r"DB/User.db"
+
+# conn = sqlite3.connect(database_file, check_same_thread=False)  # connection with sql db
+# c = conn.cursor()
+
+
+# def query_excecuter_postgres(
+#     query,
+#     db_cred,
+#     params=None,
+#     insert=True,
+#     insert_retrieve=False,
+# ):
+#     """
+#     Executes a SQL query on a PostgreSQL database, handling both insert and select operations.
+
+#     Parameters:
+#     query (str): The SQL query to be executed.
+#     params (tuple, optional): Parameters to pass into the SQL query for parameterized execution.
+#     insert (bool, default=True): Flag to determine if the query is an insert operation (default) or a select operation.
+#     insert_retrieve (bool, default=False): Flag to determine if the query should insert and then return the inserted ID.
+
+#     """
+#     # Database connection parameters
+#     dbname = db_cred["dbname"]
+#     user = db_cred["user"]
+#     password = db_cred["password"]
+#     host = db_cred["host"]
+#     port = db_cred["port"]
+
+#     try:
+#         # Establish connection to the PostgreSQL database
+#         conn = psycopg2.connect(
+#             dbname=dbname, user=user, password=password, host=host, port=port
+#         )
+#     except psycopg2.Error as e:
+#         st.warning(f"Unable to connect to the database: {e}")
+#         st.stop()
+
+#     # Create a cursor object to interact with the database
+#     c = conn.cursor()
+
+#     try:
+#         # Execute the query with or without parameters
+#         if params:
+#             c.execute(query, params)
+#         else:
+#             c.execute(query)
+
+#         if not insert:
+#             # If not an insert operation, fetch and return the results
+#             results = c.fetchall()
+#             return results
+#         elif insert_retrieve:
+#             # If insert and retrieve operation, fetch and return the results
+#             conn.commit()
+#             return c.fetchall()
+#         else:
+#             conn.commit()
+
+#     except Exception as e:
+#         st.write(f"Error executing query: {e}")
+#     finally:
+#         conn.close()
+
+
+db_path = os.path.join("imp_db.db")
+
+
+def query_excecuter_postgres(
+    query, db_path=None, params=None, insert=True, insert_retrieve=False, db_cred=None
+):
+    """
+    Executes a SQL query on a SQLite database, handling both insert and select operations.
+
+    Parameters:
+    query (str): The SQL query to be executed.
+    db_path (str): Path to the SQLite database file.
+    params (tuple, optional): Parameters to pass into the SQL query for parameterized execution.
+    insert (bool, default=True): Flag to determine if the query is an insert operation (default) or a select operation.
+    insert_retrieve (bool, default=False): Flag to determine if the query should insert and then return the inserted ID.
+
+    """
+    try:
+        # Construct a cross-platform path to the database
+        db_dir = os.path.join("db")
+        os.makedirs(db_dir, exist_ok=True)  # Make sure the directory exists
+        db_path = os.path.join(db_dir, "imp_db.db")
+
+        # Establish connection to the SQLite database
+        conn = sqlite3.connect(db_path)
+    except sqlite3.Error as e:
+        st.warning(f"Unable to connect to the SQLite database: {e}")
+        st.stop()
+
+    # Create a cursor object to interact with the database
+    c = conn.cursor()
+
+    # Prepare the query with proper placeholders
+    if params:
+        # Handle the `IN (?)` clause dynamically
+        query = query.replace("IN (?)", f"IN ({','.join(['?' for _ in params])})")
+        c.execute(query, params)
+    else:
+        c.execute(query)
+
+    try:
+        if not insert:
+            # If not an insert operation, fetch and return the results
+            results = c.fetchall()
+            return results
+        elif insert_retrieve:
+            # If insert and retrieve operation, commit and return the last inserted row ID
+            conn.commit()
+            return c.lastrowid
+        else:
+            # For standard insert operations, commit the transaction
+            conn.commit()
+
+    except Exception as e:
+        st.write(f"Error executing query: {e}")
+    finally:
+        conn.close()
+
+
+def update_summary_df():
+    """
+    Updates the 'project_summary_df' in the session state with the latest project
+    summary information based on the most recent updates.
+
+    This function executes a SQL query to retrieve project metadata from a database
+    and stores the result in the session state.
+
+    Uses:
+    - query_excecuter_postgres(query, params=params, insert=False): A function that
+      executes the provided SQL query on a PostgreSQL database.
+
+    Modifies:
+    - st.session_state['project_summary_df']: Updates the dataframe with columns:
+      'Project Number', 'Project Name', 'Last Modified Page', 'Last Modified Time'.
+    """
+
+    query = f"""
+            WITH LatestUpdates AS (
+                SELECT
+                    prj_id,
+                    page_nam,
+                    updt_dt_tm,
+                    ROW_NUMBER() OVER (PARTITION BY prj_id ORDER BY updt_dt_tm DESC) AS rn
+                FROM
+                    mmo_project_meta_data
+            )
+            SELECT
+                p.prj_id,
+                p.prj_nam AS prj_nam,
+                lu.page_nam,
+                lu.updt_dt_tm
+            FROM
+                LatestUpdates lu
+            RIGHT JOIN
+                mmo_projects p ON lu.prj_id = p.prj_id
+            WHERE
+                p.prj_ownr_id = ? AND lu.rn = 1
+            """
+
+    params = (st.session_state["emp_id"],)  # Parameters for the SQL query
+
+    # Execute the query and retrieve project summary data
+    project_summary = query_excecuter_postgres(
+        query, db_cred, params=params, insert=False
+    )
+
+    # Update the session state with the project summary dataframe
+    st.session_state["project_summary_df"] = pd.DataFrame(
+        project_summary,
+        columns=[
+            "Project Number",
+            "Project Name",
+            "Last Modified Page",
+            "Last Modified Time",
+        ],
+    )
+
+    st.session_state["project_summary_df"] = st.session_state[
+        "project_summary_df"
+    ].sort_values(by=["Last Modified Time"], ascending=False)
+
+    return st.session_state["project_summary_df"]
+
+
+from constants import default_dct
+
+
+def ensure_project_dct_structure(session_state, default_dct):
+    for key, value in default_dct.items():
+        if key not in session_state:
+            session_state[key] = value
+        elif isinstance(value, dict):
+            ensure_project_dct_structure(session_state[key], value)
+
+
+def project_selection():
+
+    emp_id = st.text_input("employee id", key="emp1111").lower()
+    password = st.text_input("Password", max_chars=15, type="password")
+
+    if st.button("Login"):
+
+        if "unique_ids" not in st.session_state:
+            unique_users_query = f"""
+                    SELECT DISTINCT emp_id, emp_nam, emp_typ from mmo_users;
+                    """
+            unique_users_result = query_excecuter_postgres(
+                unique_users_query, db_cred, insert=False
+            )  # retrieves all the users who has access to MMO TOOL
+            st.session_state["unique_ids"] = {
+                emp_id: (emp_nam, emp_type)
+                for emp_id, emp_nam, emp_type in unique_users_result
+            }
+
+        if emp_id not in st.session_state["unique_ids"].keys() or len(password) == 0:
+            st.warning("invalid id or password!")
+            st.stop()
+
+        if not is_pswrd_flag_set(emp_id):
+            st.warning("Reset password in home page to continue")
+            st.stop()
+
+        elif not verify_password(emp_id, password):
+            st.warning("Invalid user name or password")
+            st.stop()
+
+        else:
+            st.session_state["emp_id"] = emp_id
+            st.session_state["username"] = st.session_state["unique_ids"][
+                st.session_state["emp_id"]
+            ][0]
+
+        with st.spinner("Loading Saved Projects"):
+            st.session_state["project_summary_df"] = update_summary_df()
+
+            # st.write(st.session_state["project_name"][0])
+        if len(st.session_state["project_summary_df"]) == 0:
+            st.warning("No projects found please create a project in Home page")
+            st.stop()
+
+        else:
+
+            try:
+                st.session_state["project_name"] = (
+                    st.session_state["project_summary_df"]
+                    .loc[
+                        st.session_state["project_summary_df"]["Project Number"]
+                        == st.session_state["project_summary_df"].iloc[0, 0],
+                        "Project Name",
+                    ]
+                    .values[0]
+                )  # fetching project name from project number stored in summary df
+
+                poroject_dct_query = f""" 
+                
+                SELECT pkl_obj FROM  mmo_project_meta_data WHERE prj_id = ? AND file_nam=?;
+                
+                """
+                # Execute the query and retrieve the result
+
+                project_number = int(st.session_state["project_summary_df"].iloc[0, 0])
+
+                st.session_state["project_number"] = project_number
+
+                project_dct_retrieved = query_excecuter_postgres(
+                    poroject_dct_query,
+                    db_cred,
+                    params=(project_number, "project_dct"),
+                    insert=False,
+                )
+                # retrieves project dict (meta data)  stored in db
+
+                st.session_state["project_dct"] = pickle.loads(
+                    project_dct_retrieved[0][0]
+                )  # converting bytes data to original objet using pickle
+                ensure_project_dct_structure(
+                    st.session_state["project_dct"], default_dct
+                )
+
+                st.success("Project Loded")
+                st.rerun()
+
+            except Exception as e:
+
+                st.write(
+                    "Failed to load project meta data from db please create new project!"
+                )
+                st.stop()
+
+
+def update_db(prj_id, page_nam, file_nam, pkl_obj, resp_mtrc="", schema=""):
+
+    # Check if an entry already exists
+
+    check_query = f"""
+    SELECT 1 FROM mmo_project_meta_data
+    WHERE prj_id = ? AND file_nam =?;
+    """
+
+    check_params = (prj_id, file_nam)
+    result = query_excecuter_postgres(
+        check_query, db_cred, params=check_params, insert=False
+    )
+
+    # If entry exists, perform an update
+    if result is not None and result:
+
+        update_query = f"""
+        UPDATE mmo_project_meta_data
+        SET file_nam = ?, pkl_obj = ?, page_nam=? ,updt_dt_tm = datetime('now')
+
+        WHERE prj_id = ? AND file_nam = ?;
+        """
+
+        update_params = (file_nam, pkl_obj, page_nam, prj_id, file_nam)
+
+        query_excecuter_postgres(
+            update_query, db_cred, params=update_params, insert=True
+        )
+
+    # If entry does not exist, perform an insert
+    else:
+
+        insert_query = f"""
+        INSERT INTO mmo_project_meta_data
+        (prj_id, page_nam, file_nam, pkl_obj,crte_by_uid, crte_dt_tm, updt_dt_tm)
+        VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'));
+        """
+
+        insert_params = (
+            prj_id,
+            page_nam,
+            file_nam,
+            pkl_obj,
+            st.session_state["emp_id"],
+        )
+
+        query_excecuter_postgres(
+            insert_query, db_cred, params=insert_params, insert=True
+        )
+
+        # st.success(f"Inserted project meta data for project {prj_id}, page {page_nam}")
+
+
+def retrieve_pkl_object(prj_id, page_nam, file_nam, schema=""):
+
+    query = f"""
+    SELECT pkl_obj FROM mmo_project_meta_data
+    WHERE prj_id = ? AND page_nam = ? AND file_nam = ?;
+    """
+
+    params = (prj_id, page_nam, file_nam)
+    result = query_excecuter_postgres(
+        query, db_cred=db_cred, params=params, insert=False
+    )
+
+    if result and result[0] and result[0][0]:
+        pkl_obj = result[0][0]
+        # Deserialize the pickle object
+        return pickle.loads(pkl_obj)
+    else:
+        return None
+
+
+def validate_text(input_text):
+
+    # Check the length of the text
+    if len(input_text) < 2:
+        return False, "Input should be at least 2 characters long."
+    if len(input_text) > 30:
+        return False, "Input should not exceed 30 characters."
+
+    # Check if the text contains only allowed characters
+    if not re.match(r"^[A-Za-z0-9_]+$", input_text):
+        return (
+            False,
+            "Input contains invalid characters. Only letters, numbers and underscores are allowed.",
+        )
+
+    return True, "Input is valid."
+
+
+def delete_entries(prj_id, page_names, db_cred=None, schema=None):
+    """
+    Deletes all entries from the project_meta_data table based on prj_id and a list of page names.
+
+    Parameters:
+    prj_id (int): The project ID.
+    page_names (list): A list of page names.
+    db_cred (dict): Database credentials with keys 'dbname', 'user', 'password', 'host', 'port'.
+    schema (str): The schema name.
+    """
+    # Create placeholders for each page name in the list
+    placeholders = ", ".join(["?"] * len(page_names))
+    query = f"""
+    DELETE FROM mmo_project_meta_data
+    WHERE prj_id = ? AND page_nam IN ({placeholders});
+    """
+
+    # Combine prj_id and page_names into one list of parameters
+    params = (prj_id, *page_names)
+
+    query_excecuter_postgres(query, db_cred, params=params, insert=True)
+
+
+# st.success(f"Deleted entries for project {prj_id}, page {page_name}")
+def store_hashed_password(
+    user_id,
+    plain_text_password,
+):
+    """
+    Hashes a plain text password using bcrypt, converts it to a UTF-8 string, and stores it as text.
+
+    Parameters:
+    plain_text_password (str): The plain text password to be hashed.
+    db_cred (dict): The database credentials including dbname, user, password, host, and port.
+    """
+    # Hash the plain text password
+    hashed_password = bcrypt.hashpw(
+        plain_text_password.encode("utf-8"), bcrypt.gensalt()
+    )
+
+    # Convert the byte string to a regular string for storage
+    hashed_password_str = hashed_password.decode("utf-8")
+
+    # SQL query to update the pswrd_key for the specified user_id
+    query = f"""
+    UPDATE mmo_users
+    SET pswrd_key = ?
+    WHERE emp_id = ?;
+    """
+
+    # Execute the query using the existing query_excecuter_postgres function
+    query_excecuter_postgres(
+        query=query, db_cred=db_cred, params=(hashed_password_str, user_id), insert=True
+    )
+
+
+def verify_password(user_id, plain_text_password):
+    """
+    Verifies the plain text password against the stored hashed password for the specified user_id.
+
+    Parameters:
+    user_id (int): The ID of the user whose password is being verified.
+    plain_text_password (str): The plain text password to verify.
+    db_cred (dict): The database credentials including dbname, user, password, host, and port.
+    """
+    # SQL query to retrieve the hashed password for the user_id
+    query = f"""
+    SELECT pswrd_key FROM mmo_users WHERE emp_id = ?;
+    """
+
+    # Execute the query using the existing query_excecuter_postgres function
+    result = query_excecuter_postgres(
+        query=query, db_cred=db_cred, params=(user_id,), insert=False
+    )
+
+    if result:
+
+        stored_hashed_password_str = result[0][0]
+        # Convert the stored string back to bytes
+        stored_hashed_password = stored_hashed_password_str.encode("utf-8")
+
+        if bcrypt.checkpw(plain_text_password.encode("utf-8"), stored_hashed_password):
+
+            return True
+        else:
+
+            return False
+    else:
+
+        return False
+
+
+def update_password_in_db(user_id, plain_text_password):
+    """
+    Hashes the plain text password and updates the `pswrd_key`
+    column for the given `emp_id` in the `mmo_users` table.
+
+    Parameters:
+    emp_id (var): The ID of the user whose password needs to be updated.
+    plain_text_password (str): The plain text password to be hashed and stored.
+    db_cred (dict): Database credentials required to connect to the database.
+    """
+    # Hash the plain text password using bcrypt
+    hashed_password = bcrypt.hashpw(
+        plain_text_password.encode("utf-8"), bcrypt.gensalt()
+    )
+
+    # Convert the hashed password from bytes to a string for storage
+    hashed_password_str = hashed_password.decode("utf-8")
+
+    # SQL query to update the password in the database
+    query = f"""
+    UPDATE mmo_users
+    SET pswrd_key = ?
+    WHERE emp_id = ?
+    """
+
+    # Parameters for the query
+    params = (hashed_password_str, user_id)
+
+    # Execute the query using the query_excecuter_postgres function
+    query_excecuter_postgres(query, db_cred, params=params, insert=True)
+
+
+def is_pswrd_flag_set(user_id):
+    query = f"""
+    SELECT pswrd_flag 
+    FROM mmo_users 
+    WHERE emp_id = ?;
+    """
+
+    # Execute the query
+    result = query_excecuter_postgres(query, db_cred, params=(user_id,), insert=False)
+
+    # Return True if the flag is 1, otherwise return False
+    if result and result[0][0] == 1:
+        return True
+    else:
+        return False
+
+
+def set_pswrd_flag(user_id):
+    query = f"""
+    UPDATE mmo_users
+    SET pswrd_flag = 1
+    WHERE emp_id = ?;
+    """
+
+    # Execute the update query
+    query_excecuter_postgres(query, db_cred, params=(user_id,), insert=True)
+
+
+def retrieve_pkl_object_without_warning(prj_id, page_nam, file_nam, schema):
+
+    query = f"""
+    SELECT pkl_obj FROM mmo_project_meta_data
+    WHERE prj_id = ? AND page_nam = ? AND file_nam = ?;
+    """
+
+    params = (prj_id, page_nam, file_nam)
+    result = query_excecuter_postgres(
+        query, db_cred=db_cred, params=params, insert=False
+    )
+
+    if result and result[0] and result[0][0]:
+        pkl_obj = result[0][0]
+        # Deserialize the pickle object
+        return pickle.loads(pkl_obj)
+    else:
+        # st.warning(
+        #     "Pickle object not found for the given project ID, page name, and file name."
+        # )
+        return None
+
+
+color_palette = [
+    "#F3F3F0",
+    "#5E7D7E",
+    "#2FA1FF",
+    "#00EDED",
+    "#00EAE4",
+    "#304550",
+    "#EDEBEB",
+    "#7FBEFD",
+    "#003059",
+    "#A2F3F3",
+    "#E1D6E2",
+    "#B6B6B6",
+]
+
+
+CURRENCY_INDICATOR = "$"
+
+
+# database_file = r"DB/User.db"
+
+# conn = sqlite3.connect(database_file, check_same_thread=False)  # connection with sql db
+# c = conn.cursor()
+
+
+# def load_authenticator():
+#     with open("config.yaml") as file:
+#         config = yaml.load(file, Loader=SafeLoader)
+#         st.session_state["config"] = config
+#     authenticator = stauth.Authenticate(
+#         credentials=config["credentials"],
+#         cookie_name=config["cookie"]["name"],
+#         key=config["cookie"]["key"],
+#         cookie_expiry_days=config["cookie"]["expiry_days"],
+#         preauthorized=config["preauthorized"],
+#     )
+#     st.session_state["authenticator"] = authenticator
+#     return authenticator
+
+
+# Authentication
+# def authenticator():
+#     for k, v in st.session_state.items():
+#         if k not in ["logout", "login", "config"] and not k.startswith(
+#             "FormSubmitter"
+#         ):
+#             st.session_state[k] = v
+#     with open("config.yaml") as file:
+#         config = yaml.load(file, Loader=SafeLoader)
+#         st.session_state["config"] = config
+#     authenticator = stauth.Authenticate(
+#         config["credentials"],
+#         config["cookie"]["name"],
+#         config["cookie"]["key"],
+#         config["cookie"]["expiry_days"],
+#         config["preauthorized"],
+#     )
+#     st.session_state["authenticator"] = authenticator
+#     name, authentication_status, username = authenticator.login(
+#         "Login", "main"
+#     )
+#     auth_status = st.session_state.get("authentication_status")
+
+#     if auth_status == True:
+#         authenticator.logout("Logout", "main")
+#         is_state_initiaized = st.session_state.get("initialized", False)
+
+#         if not is_state_initiaized:
+
+#             if "session_name" not in st.session_state:
+#                 st.session_state["session_name"] = None
+
+#     return name
+
+
+# def authentication():
+#     with open("config.yaml") as file:
+#         config = yaml.load(file, Loader=SafeLoader)
+
+#         authenticator = stauth.Authenticate(
+#             config["credentials"],
+#             config["cookie"]["name"],
+#             config["cookie"]["key"],
+#             config["cookie"]["expiry_days"],
+#             config["preauthorized"],
+#         )
+
+#     name, authentication_status, username = authenticator.login(
+#         "Login", "main"
+#     )
+#     return authenticator, name, authentication_status, username
+
+
+def nav_page(page_name, timeout_secs=3):
+    nav_script = """
+        <script type="text/javascript">
+            function attempt_nav_page(page_name, start_time, timeout_secs) {
+                var links = window.parent.document.getElementsByTagName("a");
+                for (var i = 0; i < links.length; i++) {
+                    if (links[i].href.toLowerCase().endsWith("/" + page_name.toLowerCase())) {
+                        links[i].click();
+                        return;
+                    }
+                }
+                var elasped = new Date() - start_time;
+                if (elasped < timeout_secs * 1000) {
+                    setTimeout(attempt_nav_page, 100, page_name, start_time, timeout_secs);
+                } else {
+                    alert("Unable to navigate to page '" + page_name + "' after " + timeout_secs + " second(s).");
+                }
+            }
+            window.addEventListener("load", function() {
+                attempt_nav_page("%s", new Date(), %d);
+            });
+        </script>
+    """ % (
+        page_name,
+        timeout_secs,
+    )
+    html(nav_script)
+
+
+# def load_local_css(file_name):
+#     with open(file_name) as f:
+#         st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)
+
+
+# def set_header():
+#     return st.markdown(f"""<div class='main-header'>
+#                     <h1>MMM LiME</h1>
+#                     <img src="https://assets-global.website-files.com/64c8fffb0e95cbc525815b79/64df84637f83a891c1473c51_Vector%20(Stroke).svg   ">
+#             </div>""", unsafe_allow_html=True)
+
+path = os.path.dirname(__file__)
+
+file_ = open(f"{path}/logo.png", "rb")
+
+contents = file_.read()
+
+data_url = base64.b64encode(contents).decode("utf-8")
+
+file_.close()
+
+
+DATA_PATH = "./data"
+
+IMAGES_PATH = "./data/images_224_224"
+
+
+def load_local_css(file_name):
+
+    with open(file_name) as f:
+
+        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+
+
+# def set_header():
+
+#     return st.markdown(f"""<div class='main-header'>
+
+#                     <h1>H & M Recommendations</h1>
+
+#                     <img src="data:image;base64,{data_url}", alt="Logo">
+
+#             </div>""", unsafe_allow_html=True)
+path1 = os.path.dirname(__file__)
+
+# file_1 = open(f"{path}/willbank.png", "rb")
+
+# contents1 = file_1.read()
+
+# data_url1 = base64.b64encode(contents1).decode("utf-8")
+
+# file_1.close()
+
+
+DATA_PATH1 = "./data"
+
+IMAGES_PATH1 = "./data/images_224_224"
+
+
+def set_header():
+    return st.markdown(
+        f"""<div class='main-header'>
+                    <!-- <h1></h1> -->
+                       <div >
+                    <img class='blend-logo' src="data:image;base64,{data_url}", alt="Logo">
+            </div>""",
+        unsafe_allow_html=True,
+    )
+
+
+# def set_header():
+#     logo_path = "./path/to/your/local/LIME_logo.png"  # Replace with the actual file path
+#     text = "LiME"
+#     return st.markdown(f"""<div class='main-header'>
+#                     <img src="data:image/png;base64,{data_url}" alt="Logo" style="float: left; margin-right: 10px; width: 100px; height: auto;">
+#                     <h1>{text}</h1>
+#             </div>""", unsafe_allow_html=True)
+
+
+def s_curve(x, K, b, a, x0):
+    return K / (1 + b * np.exp(-a * (x - x0)))
+
+
+def panel_level(input_df, date_column="Date"):
+    # Ensure 'Date' is set as the index
+    if date_column not in input_df.index.names:
+        input_df = input_df.set_index(date_column)
+
+    # Select numeric columns only (excluding 'Date' since it's now the index)
+    numeric_columns_df = input_df.select_dtypes(include="number")
+
+    # Group by 'Date' (which is the index) and sum the numeric columns
+    aggregated_df = numeric_columns_df.groupby(input_df.index).sum()
+
+    # Reset the index to bring the 'Date' column
+    aggregated_df = aggregated_df.reset_index()
+
+    return aggregated_df
+
+
+def fetch_actual_data(
+    panel=None,
+    target_file="Overview_data_test.xlsx",
+    updated_rcs=None,
+    metrics=None,
+):
+    excel = pd.read_excel(Path(target_file), sheet_name=None)
+
+    # Extract dataframes for raw data, spend input, and contribution MMM
+    raw_df = excel["RAW DATA MMM"]
+    spend_df = excel["SPEND INPUT"]
+    contri_df = excel["CONTRIBUTION MMM"]
+
+    # Check if the panel is not None
+    if panel is not None and panel != "Aggregated":
+        raw_df = raw_df[raw_df["Panel"] == panel].drop(columns=["Panel"])
+        spend_df = spend_df[spend_df["Panel"] == panel].drop(columns=["Panel"])
+        contri_df = contri_df[contri_df["Panel"] == panel].drop(columns=["Panel"])
+    elif panel == "Aggregated":
+        raw_df = panel_level(raw_df, date_column="Date")
+        spend_df = panel_level(spend_df, date_column="Week")
+        contri_df = panel_level(contri_df, date_column="Date")
+
+    # Revenue_df = excel['Revenue']
+
+    ## remove sesonalities, indices etc ...
+    unnamed_cols = [col for col in raw_df.columns if col.lower().startswith("unnamed")]
+    ## remove sesonalities, indices etc ...
+
+    exclude_columns = [
+        "Date",
+        "Region",
+        "Controls_Grammarly_Index_SeasonalAVG",
+        "Controls_Quillbot_Index",
+        "Daily_Positive_Outliers",
+        "External_RemoteClass_Index",
+        "Intervals ON 20190520-20190805 | 20200518-20200803 | 20210517-20210802",
+        "Intervals ON 20190826-20191209 | 20200824-20201207 | 20210823-20211206",
+        "Intervals ON 20201005-20201019",
+        "Promotion_PercentOff",
+        "Promotion_TimeBased",
+        "Seasonality_Indicator_Chirstmas",
+        "Seasonality_Indicator_NewYears_Days",
+        "Seasonality_Indicator_Thanksgiving",
+        "Trend 20200302 / 20200803",
+    ] + unnamed_cols
+
+    raw_df["Date"] = pd.to_datetime(raw_df["Date"])
+    contri_df["Date"] = pd.to_datetime(contri_df["Date"])
+    input_df = raw_df.sort_values(by="Date")
+    output_df = contri_df.sort_values(by="Date")
+    spend_df["Week"] = pd.to_datetime(
+        spend_df["Week"], format="%Y-%m-%d", errors="coerce"
+    )
+    spend_df.sort_values(by="Week", inplace=True)
+
+    # spend_df['Week'] = pd.to_datetime(spend_df['Week'], errors='coerce')
+    # spend_df = spend_df.sort_values(by='Week')
+
+    channel_list = [col for col in input_df.columns if col not in exclude_columns]
+    channel_list = list(set(channel_list) - set(["fb_level_achieved_tier_1", "ga_app"]))
+
+    infeasible_channels = [
+        c
+        for c in contri_df.select_dtypes(include=["float", "int"]).columns
+        if contri_df[c].sum() <= 0
+    ]
+    # st.write(channel_list)
+    channel_list = list(set(channel_list) - set(infeasible_channels))
+
+    upper_limits = {}
+    output_cols = []
+    actual_output_dic = {}
+    actual_input_dic = {}
+
+    for inp_col in channel_list:
+        # st.write(inp_col)
+        spends = input_df[inp_col].values
+        x = spends.copy()
+        # upper limit for penalty
+        upper_limits[inp_col] = 2 * x.max()
+
+        # contribution
+        # out_col = [_col for _col in output_df.columns if _col.startswith(inp_col)][0]
+        out_col = inp_col
+        y = output_df[out_col].values.copy()
+        actual_output_dic[inp_col] = y.copy()
+        actual_input_dic[inp_col] = x.copy()
+        ##output cols aggregation
+        output_cols.append(out_col)
+
+    return pd.DataFrame(actual_input_dic), pd.DataFrame(actual_output_dic)
+
+
+# Function to initialize model results data
+def initialize_data(panel=None, metrics=None):
+    # Extract dataframes for raw data, spend input, and contribution data
+    raw_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][metrics]["raw_data"].copy()
+    spend_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][metrics]["spends_data"].copy()
+    contribution_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][metrics]["contribution_data"].copy()
+
+    # Check if 'Panel' or 'panel' is in the columns
+    panel_column = None
+    if "Panel" in raw_df.columns:
+        panel_column = "Panel"
+    elif "panel" in raw_df.columns:
+        panel_column = "panel"
+
+    # Filter data by panel if provided
+    if panel and panel.lower() != "aggregated":
+        raw_df = raw_df[raw_df[panel_column] == panel].drop(columns=[panel_column])
+        spend_df = spend_df[spend_df[panel_column] == panel].drop(
+            columns=[panel_column]
+        )
+        contribution_df = contribution_df[contribution_df[panel_column] == panel].drop(
+            columns=[panel_column]
+        )
+    else:
+        raw_df = panel_level(raw_df, date_column="Date")
+        spend_df = panel_level(spend_df, date_column="Date")
+        contribution_df = panel_level(contribution_df, date_column="Date")
+
+    # Remove unnecessary columns
+    unnamed_cols = [col for col in raw_df.columns if col.lower().startswith("unnamed")]
+    exclude_columns = ["Date"] + unnamed_cols
+
+    # Convert Date columns to datetime
+    for df in [raw_df, spend_df, contribution_df]:
+        df["Date"] = pd.to_datetime(df["Date"], format="%Y-%m-%d", errors="coerce")
+
+    # Sort data by Date
+    input_df = raw_df.sort_values(by="Date")
+    contribution_df = contribution_df.sort_values(by="Date")
+    spend_df.sort_values(by="Date", inplace=True)
+
+    # Extract channels excluding unwanted columns
+    channel_list = [col for col in input_df.columns if col not in exclude_columns]
+
+    # Filter out channels with non-positive contributions
+    negative_contributions = [
+        col
+        for col in contribution_df.select_dtypes(include=["float", "int"]).columns
+        if contribution_df[col].sum() <= 0
+    ]
+    channel_list = list(set(channel_list) - set(negative_contributions))
+
+    # Initialize dictionaries for metrics and response curves
+    response_curves, mapes, rmses, upper_limits = {}, {}, {}, {}
+    r2_scores, powers, conversion_rates, actual_output, actual_input = (
+        {},
+        {},
+        {},
+        {},
+        {},
+    )
+    channels = {}
+    sales = None
+    dates = input_df["Date"].values
+
+    # Fit s-curve for each channel
+    for channel in channel_list:
+        spends = input_df[channel].values
+        x = spends.copy()
+        upper_limits[channel] = 2 * x.max()
+
+        # Get corresponding output column
+        output_col = [
+            _col for _col in contribution_df.columns if _col.startswith(channel)
+        ][0]
+        y = contribution_df[output_col].values.copy()
+        actual_output[channel] = y.copy()
+        actual_input[channel] = x.copy()
+
+        # Scale input data
+        power = np.ceil(np.log(x.max()) / np.log(10)) - 3
+        if power >= 0:
+            x = x / 10**power
+        x, y = x.astype("float64"), y.astype("float64")
+
+        # Set bounds for curve fitting
+        if y.max() <= 0.01:
+            bounds = (
+                (0, 0, 0, 0),
+                (3 * 0.01, 1000, 1, x.max() if x.max() > 0 else 0.01),
+            )
+        else:
+            bounds = ((0, 0, 0, 0), (3 * y.max(), 1000, 1, x.max()))
+
+        # Set y to 0 where x is 0
+        y[x == 0] = 0
+
+        # Fit s-curve and calculate metrics
+        # params, _ = curve_fit(
+        #     s_curve,
+        #     x
+        #     y,
+        #     p0=(2 * y.max(), 0.01, 1e-5, x.max()),
+        #     bounds=bounds,
+        #     maxfev=int(1e6),
+        # )
+        params, _ = curve_fit(
+            s_curve,
+            list(x) + [0] * len(x),
+            list(y) + [0] * len(y),
+            p0=(2 * y.max(), 0.01, 1e-5, x.max()),
+            bounds=bounds,
+            maxfev=int(1e6),
+        )
+
+        mape = (100 * abs(1 - s_curve(x, *params) / y.clip(min=1))).mean()
+        rmse = np.sqrt(((y - s_curve(x, *params)) ** 2).mean())
+        r2_score_ = r2_score(y, s_curve(x, *params))
+
+        # Store metrics and parameters
+        response_curves[channel] = {
+            "K": params[0],
+            "b": params[1],
+            "a": params[2],
+            "x0": params[3],
+        }
+        mapes[channel] = mape
+        rmses[channel] = rmse
+        r2_scores[channel] = r2_score_
+        powers[channel] = power
+
+        conversion_rate = spend_df[channel].sum() / max(input_df[channel].sum(), 1e-9)
+        conversion_rates[channel] = conversion_rate
+        correction = y - s_curve(x, *params)
+
+        # Initialize Channel object
+        channel_obj = Channel(
+            name=channel,
+            dates=dates,
+            spends=spends,
+            conversion_rate=conversion_rate,
+            response_curve_type="s-curve",
+            response_curve_params={
+                "K": params[0],
+                "b": params[1],
+                "a": params[2],
+                "x0": params[3],
+            },
+            bounds=np.array([-10, 10]),
+            correction=correction,
+        )
+        channels[channel] = channel_obj
+        if sales is None:
+            sales = channel_obj.actual_sales
+        else:
+            sales += channel_obj.actual_sales
+
+    # Calculate other contributions
+    other_contributions = (
+        contribution_df.drop(columns=[*response_curves.keys()])
+        .sum(axis=1, numeric_only=True)
+        .values
+    )
+
+    # Initialize Scenario object
+    scenario = Scenario(
+        name="default",
+        channels=channels,
+        constant=other_contributions,
+        correction=np.array([]),
+    )
+
+    # Set session state variables
+    st.session_state.update(
+        {
+            "initialized": True,
+            "actual_df": input_df,
+            "raw_df": raw_df,
+            "contri_df": contribution_df,
+            "default_scenario_dict": class_to_dict(scenario),
+            "scenario": scenario,
+            "channels_list": channel_list,
+            "optimization_channels": {
+                channel_name: False for channel_name in channel_list
+            },
+            "rcs": response_curves.copy(),
+            "powers": powers,
+            "actual_contribution_df": pd.DataFrame(actual_output),
+            "actual_input_df": pd.DataFrame(actual_input),
+            "xlsx_buffer": io.BytesIO(),
+            "saved_scenarios": (
+                pickle.load(open("../saved_scenarios.pkl", "rb"))
+                if Path("../saved_scenarios.pkl").exists()
+                else OrderedDict()
+            ),
+            "disable_download_button": True,
+        }
+    )
+
+    for channel in channels.values():
+        st.session_state[channel.name] = numerize(
+            channel.actual_total_spends * channel.conversion_rate, 1
+        )
+
+    # Prepare response curve data for output
+    response_curve_data = {}
+    for channel, params in st.session_state["rcs"].items():
+        x = st.session_state["actual_input_df"][channel].values.astype(float)
+        y = st.session_state["actual_contribution_df"][channel].values.astype(float)
+        power = float(np.ceil(np.log(max(x)) / np.log(10)) - 3)
+        x_plot = list(np.linspace(0, 5 * max(x), 100))
+
+        response_curve_data[channel] = {
+            "K": float(params["K"]),
+            "b": float(params["b"]),
+            "a": float(params["a"]),
+            "x0": float(params["x0"]),
+            "power": power,
+            "x": list(x),
+            "y": list(y),
+            "x_plot": x_plot,
+        }
+
+    return response_curve_data, scenario
+
+
+# def initialize_data(panel=None, metrics=None):
+#     # Extract dataframes for raw data, spend input, and contribution data
+#     raw_df = st.session_state["project_dct"]["current_media_performance"][
+#         "model_outputs"
+#     ][metrics]["raw_data"]
+#     spend_df = st.session_state["project_dct"]["current_media_performance"][
+#         "model_outputs"
+#     ][metrics]["spends_data"]
+#     contri_df = st.session_state["project_dct"]["current_media_performance"][
+#         "model_outputs"
+#     ][metrics]["contribution_data"]
+
+#     # Check if the panel is not None
+#     if panel is not None and panel.lower() != "aggregated":
+#         raw_df = raw_df[raw_df["Panel"] == panel].drop(columns=["Panel"])
+#         spend_df = spend_df[spend_df["Panel"] == panel].drop(columns=["Panel"])
+#         contri_df = contri_df[contri_df["Panel"] == panel].drop(columns=["Panel"])
+#     elif panel.lower() == "aggregated":
+#         raw_df = panel_level(raw_df, date_column="Date")
+#         spend_df = panel_level(spend_df, date_column="Date")
+#         contri_df = panel_level(contri_df, date_column="Date")
+
+#     ## remove sesonalities, indices etc ...
+#     unnamed_cols = [col for col in raw_df.columns if col.lower().startswith("unnamed")]
+
+#     ## remove sesonalities, indices etc ...
+#     exclude_columns = ["Date"] + unnamed_cols
+
+#     raw_df["Date"] = pd.to_datetime(raw_df["Date"], format="%Y-%m-%d", errors="coerce")
+#     contri_df["Date"] = pd.to_datetime(
+#         contri_df["Date"], format="%Y-%m-%d", errors="coerce"
+#     )
+#     spend_df["Date"] = pd.to_datetime(
+#         spend_df["Date"], format="%Y-%m-%d", errors="coerce"
+#     )
+
+#     input_df = raw_df.sort_values(by="Date")
+#     output_df = contri_df.sort_values(by="Date")
+#     spend_df.sort_values(by="Date", inplace=True)
+
+#     channel_list = [col for col in input_df.columns if col not in exclude_columns]
+
+#     negative_contribution = [
+#         c
+#         for c in contri_df.select_dtypes(include=["float", "int"]).columns
+#         if contri_df[c].sum() <= 0
+#     ]
+#     channel_list = list(set(channel_list) - set(negative_contribution))
+
+#     response_curves = {}
+#     mapes = {}
+#     rmses = {}
+#     upper_limits = {}
+#     powers = {}
+#     r2 = {}
+#     conv_rates = {}
+#     output_cols = []
+#     channels = {}
+#     sales = None
+#     dates = input_df.Date.values
+#     actual_output_dic = {}
+#     actual_input_dic = {}
+
+#     for inp_col in channel_list:
+#         spends = input_df[inp_col].values
+#         x = spends.copy()
+#         # upper limit for penalty
+#         upper_limits[inp_col] = 2 * x.max()
+
+#         # contribution
+#         out_col = [_col for _col in output_df.columns if _col.startswith(inp_col)][0]
+#         y = output_df[out_col].values.copy()
+#         actual_output_dic[inp_col] = y.copy()
+#         actual_input_dic[inp_col] = x.copy()
+#         ##output cols aggregation
+#         output_cols.append(out_col)
+
+#         ## scale the input
+#         power = np.ceil(np.log(x.max()) / np.log(10)) - 3
+#         if power >= 0:
+#             x = x / 10**power
+
+#         x = x.astype("float64")
+#         y = y.astype("float64")
+
+#         if y.max() <= 0.01:
+#             if x.max() <= 0.0:
+#                 bounds = ((0, 0, 0, 0), (3 * 0.01, 1000, 1, 0.01))
+
+#             else:
+#                 bounds = ((0, 0, 0, 0), (3 * 0.01, 1000, 1, x.max()))
+#         else:
+#             bounds = ((0, 0, 0, 0), (3 * y.max(), 1000, 1, x.max()))
+
+#         params, _ = curve_fit(
+#             s_curve,
+#             x,
+#             y,
+#             p0=(2 * y.max(), 0.01, 1e-5, x.max()),
+#             bounds=bounds,
+#             maxfev=int(1e5),
+#         )
+#         mape = (100 * abs(1 - s_curve(x, *params) / y.clip(min=1))).mean()
+#         rmse = np.sqrt(((y - s_curve(x, *params)) ** 2).mean())
+#         r2_ = r2_score(y, s_curve(x, *params))
+
+#         response_curves[inp_col] = {
+#             "K": params[0],
+#             "b": params[1],
+#             "a": params[2],
+#             "x0": params[3],
+#         }
+
+#         mapes[inp_col] = mape
+#         rmses[inp_col] = rmse
+#         r2[inp_col] = r2_
+#         powers[inp_col] = power
+
+#         conv = spend_df[inp_col].sum() / max(input_df[inp_col].sum(), 1e-9)
+#         conv_rates[inp_col] = conv
+
+#         correction = y - s_curve(x, *params)
+
+#         channel = Channel(
+#             name=inp_col,
+#             dates=dates,
+#             spends=spends,
+#             conversion_rate=conv_rates[inp_col],
+#             response_curve_type="s-curve",
+#             response_curve_params={
+#                 "K": params[0],
+#                 "b": params[1],
+#                 "a": params[2],
+#                 "x0": params[3],
+#             },
+#             bounds=np.array([-10, 10]),
+#             correction=correction,
+#         )
+
+#         channels[inp_col] = channel
+#         if sales is None:
+#             sales = channel.actual_sales
+#         else:
+#             sales += channel.actual_sales
+
+#     other_contributions = (
+#         output_df.drop([*output_cols], axis=1).sum(axis=1, numeric_only=True).values
+#     )
+
+#     scenario = Scenario(
+#         name="default",
+#         channels=channels,
+#         constant=other_contributions,
+#         correction=np.array([]),
+#     )
+
+#     ## setting session variables
+#     st.session_state["initialized"] = True
+#     st.session_state["actual_df"] = input_df
+#     st.session_state["raw_df"] = raw_df
+#     st.session_state["contri_df"] = output_df
+#     default_scenario_dict = class_to_dict(scenario)
+#     st.session_state["default_scenario_dict"] = default_scenario_dict
+#     st.session_state["scenario"] = scenario
+#     st.session_state["channels_list"] = channel_list
+#     st.session_state["optimization_channels"] = {
+#         channel_name: False for channel_name in channel_list
+#     }
+#     st.session_state["rcs"] = response_curves.copy()
+
+#     st.session_state["powers"] = powers
+#     st.session_state["actual_contribution_df"] = pd.DataFrame(actual_output_dic)
+#     st.session_state["actual_input_df"] = pd.DataFrame(actual_input_dic)
+
+#     for channel in channels.values():
+#         st.session_state[channel.name] = numerize(
+#             channel.actual_total_spends * channel.conversion_rate, 1
+#         )
+
+#     st.session_state["xlsx_buffer"] = io.BytesIO()
+
+#     if Path("../saved_scenarios.pkl").exists():
+#         with open("../saved_scenarios.pkl", "rb") as f:
+#             st.session_state["saved_scenarios"] = pickle.load(f)
+#     else:
+#         st.session_state["saved_scenarios"] = OrderedDict()
+
+#     # st.session_state["total_spends_change"] = 0
+#     st.session_state["optimization_channels"] = {
+#         channel_name: False for channel_name in channel_list
+#     }
+#     st.session_state["disable_download_button"] = True
+
+#     rcs_data = {}
+#     for channel in st.session_state["rcs"]:
+#         # Convert to native Python lists and types
+#         x = list(st.session_state["actual_input_df"][channel].values.astype(float))
+#         y = list(
+#             st.session_state["actual_contribution_df"][channel].values.astype(float)
+#         )
+#         power = float(np.ceil(np.log(max(x)) / np.log(10)) - 3)
+#         x_plot = list(np.linspace(0, 5 * max(x), 100))
+
+#         rcs_data[channel] = {
+#             "K": float(st.session_state["rcs"][channel]["K"]),
+#             "b": float(st.session_state["rcs"][channel]["b"]),
+#             "a": float(st.session_state["rcs"][channel]["a"]),
+#             "x0": float(st.session_state["rcs"][channel]["x0"]),
+#             "power": power,
+#             "x": x,
+#             "y": y,
+#             "x_plot": x_plot,
+#         }
+
+#     return rcs_data, scenario
+
+
+# def initialize_data():
+#     # fetch data from excel
+#     output = pd.read_excel('data.xlsx',sheet_name=None)
+#     raw_df = output['RAW DATA MMM']
+#     contribution_df = output['CONTRIBUTION MMM']
+#     Revenue_df = output['Revenue']
+
+#     ## channels to be shows
+#     channel_list = []
+#     for col in raw_df.columns:
+#         if 'click' in col.lower() or 'spend' in col.lower() or 'imp' in col.lower():
+#             channel_list.append(col)
+#         else:
+#             pass
+
+#     ## NOTE : Considered only Desktop spends for all calculations
+#     acutal_df = raw_df[raw_df.Region == 'Desktop'].copy()
+#     ## NOTE : Considered one year of data
+#     acutal_df = acutal_df[acutal_df.Date>'2020-12-31']
+#     actual_df = acutal_df.drop('Region',axis=1).sort_values(by='Date')[[*channel_list,'Date']]
+
+#     ##load response curves
+#     with open('./grammarly_response_curves.json','r') as f:
+#         response_curves = json.load(f)
+
+#     ## create channel dict for scenario creation
+#     dates = actual_df.Date.values
+#     channels = {}
+#     rcs = {}
+#     constant = 0.
+#     for i,info_dict in enumerate(response_curves):
+#         name = info_dict.get('name')
+#         response_curve_type = info_dict.get('response_curve')
+#         response_curve_params = info_dict.get('params')
+#         rcs[name] = response_curve_params
+#         if name != 'constant':
+#             spends = actual_df[name].values
+#             channel = Channel(name=name,dates=dates,
+#                             spends=spends,
+#                             response_curve_type=response_curve_type,
+#                             response_curve_params=response_curve_params,
+#                             bounds=np.array([-30,30]))
+
+#             channels[name] = channel
+#         else:
+#             constant = info_dict.get('value',0.) * len(dates)
+
+#     ## create scenario
+#     scenario = Scenario(name='default', channels=channels, constant=constant)
+#     default_scenario_dict = class_to_dict(scenario)
+
+
+#     ## setting session variables
+#     st.session_state['initialized'] = True
+#     st.session_state['actual_df'] = actual_df
+#     st.session_state['raw_df'] = raw_df
+#     st.session_state['default_scenario_dict'] = default_scenario_dict
+#     st.session_state['scenario'] = scenario
+#     st.session_state['channels_list'] = channel_list
+#     st.session_state['optimization_channels'] = {channel_name : False for channel_name in channel_list}
+#     st.session_state['rcs'] = rcs
+#     for channel in channels.values():
+#         if channel.name not in st.session_state:
+#             st.session_state[channel.name] = float(channel.actual_total_spends)
+
+#     if 'xlsx_buffer' not in st.session_state:
+#         st.session_state['xlsx_buffer'] = io.BytesIO()
+
+#     ## for saving scenarios
+#     if 'saved_scenarios' not in st.session_state:
+#         if Path('../saved_scenarios.pkl').exists():
+#             with open('../saved_scenarios.pkl','rb') as f:
+#                 st.session_state['saved_scenarios'] = pickle.load(f)
+
+#         else:
+#             st.session_state['saved_scenarios'] = OrderedDict()
+
+#     if 'total_spends_change' not in st.session_state:
+#         st.session_state['total_spends_change'] = 0
+
+#     if 'optimization_channels' not in st.session_state:
+#         st.session_state['optimization_channels'] = {channel_name : False for channel_name in channel_list}
+
+#     if 'disable_download_button' not in st.session_state:
+#         st.session_state['disable_download_button'] = True
+
+
+def create_channel_summary(scenario):
+
+    # Provided data
+    data = {
+        "Channel": [
+            "Paid Search",
+            "Ga will cid baixo risco",
+            "Digital tactic others",
+            "Fb la tier 1",
+            "Fb la tier 2",
+            "Paid social others",
+            "Programmatic",
+            "Kwai",
+            "Indicacao",
+            "Infleux",
+            "Influencer",
+        ],
+        "Spends": [
+            "$ 11.3K",
+            "$ 155.2K",
+            "$ 50.7K",
+            "$ 125.4K",
+            "$ 125.2K",
+            "$ 105K",
+            "$ 3.3M",
+            "$ 47.5K",
+            "$ 55.9K",
+            "$ 632.3K",
+            "$ 48.3K",
+        ],
+        "Revenue": [
+            "558.0K",
+            "3.5M",
+            "5.2M",
+            "3.1M",
+            "3.1M",
+            "2.1M",
+            "20.8M",
+            "1.6M",
+            "728.4K",
+            "22.9M",
+            "4.8M",
+        ],
+    }
+
+    # Create DataFrame
+    df = pd.DataFrame(data)
+
+    # Convert currency strings to numeric values
+    df["Spends"] = (
+        df["Spends"]
+        .replace({"\$": "", "K": "*1e3", "M": "*1e6"}, regex=True)
+        .map(pd.eval)
+        .astype(int)
+    )
+    df["Revenue"] = (
+        df["Revenue"]
+        .replace({"\$": "", "K": "*1e3", "M": "*1e6"}, regex=True)
+        .map(pd.eval)
+        .astype(int)
+    )
+
+    # Calculate ROI
+    df["ROI"] = (df["Revenue"] - df["Spends"]) / df["Spends"]
+
+    # Format columns
+    format_currency = lambda x: f"${x:,.1f}"
+    format_roi = lambda x: f"{x:.1f}"
+
+    df["Spends"] = [
+        "$ 11.3K",
+        "$ 155.2K",
+        "$ 50.7K",
+        "$ 125.4K",
+        "$ 125.2K",
+        "$ 105K",
+        "$ 3.3M",
+        "$ 47.5K",
+        "$ 55.9K",
+        "$ 632.3K",
+        "$ 48.3K",
+    ]
+    df["Revenue"] = [
+        "$ 536.3K",
+        "$ 3.4M",
+        "$ 5M",
+        "$ 3M",
+        "$ 3M",
+        "$ 2M",
+        "$ 20M",
+        "$ 1.5M",
+        "$ 7.1M",
+        "$ 22M",
+        "$ 4.6M",
+    ]
+    df["ROI"] = df["ROI"].apply(format_roi)
+
+    return df
+
+
+# @st.cache(allow_output_mutation=True)
+# def create_contribution_pie(scenario):
+#     #c1f7dc
+#     colors_map = {col:color for col,color in zip(st.session_state['channels_list'],plotly.colors.n_colors(plotly.colors.hex_to_rgb('#BE6468'), plotly.colors.hex_to_rgb('#E7B8B7'),23))}
+#     total_contribution_fig = make_subplots(rows=1, cols=2,subplot_titles=['Spends','Revenue'],specs=[[{"type": "pie"}, {"type": "pie"}]])
+#     total_contribution_fig.add_trace(
+#                 go.Pie(labels=[channel_name_formating(channel_name) for channel_name in st.session_state['channels_list']] + ['Non Media'],
+#                     values= [round(scenario.channels[channel_name].actual_total_spends * scenario.channels[channel_name].conversion_rate,1) for channel_name in st.session_state['channels_list']] + [0],
+#                     marker=dict(colors = [plotly.colors.label_rgb(colors_map[channel_name]) for channel_name in st.session_state['channels_list']] + ['#F0F0F0']),
+#                         hole=0.3),
+#                 row=1, col=1)
+
+#     total_contribution_fig.add_trace(
+#                 go.Pie(labels=[channel_name_formating(channel_name) for channel_name in st.session_state['channels_list']] + ['Non Media'],
+#                     values= [scenario.channels[channel_name].actual_total_sales for channel_name in st.session_state['channels_list']] + [scenario.correction.sum() + scenario.constant.sum()],
+#                         hole=0.3),
+#                 row=1, col=2)
+
+#     total_contribution_fig.update_traces(textposition='inside',texttemplate='%{percent:.1%}')
+#     total_contribution_fig.update_layout(uniformtext_minsize=12,title='Channel contribution', uniformtext_mode='hide')
+#     return total_contribution_fig
+
+# @st.cache(allow_output_mutation=True)
+
+# def create_contribuion_stacked_plot(scenario):
+#     weekly_contribution_fig = make_subplots(rows=1, cols=2,subplot_titles=['Spends','Revenue'],specs=[[{"type": "bar"}, {"type": "bar"}]])
+#     raw_df = st.session_state['raw_df']
+#     df = raw_df.sort_values(by='Date')
+#     x = df.Date
+#     weekly_spends_data = []
+#     weekly_sales_data = []
+#     for channel_name in st.session_state['channels_list']:
+#         weekly_spends_data.append((go.Bar(x=x,
+#                                           y=scenario.channels[channel_name].actual_spends * scenario.channels[channel_name].conversion_rate,
+#                                           name=channel_name_formating(channel_name),
+#                                           hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+#                                           legendgroup=channel_name)))
+#         weekly_sales_data.append((go.Bar(x=x,
+#                                          y=scenario.channels[channel_name].actual_sales,
+#                                          name=channel_name_formating(channel_name),
+#                                          hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+#                                          legendgroup=channel_name, showlegend=False)))
+#     for _d in weekly_spends_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=1)
+#     for _d in weekly_sales_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=2)
+#     weekly_contribution_fig.add_trace(go.Bar(x=x,
+#                                          y=scenario.constant + scenario.correction,
+#                                          name='Non Media',
+#                                          hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), row=1, col=2)
+#     weekly_contribution_fig.update_layout(barmode='stack', title='Channel contribuion by week', xaxis_title='Date')
+#     weekly_contribution_fig.update_xaxes(showgrid=False)
+#     weekly_contribution_fig.update_yaxes(showgrid=False)
+#     return weekly_contribution_fig
+
+# @st.cache(allow_output_mutation=True)
+# def create_channel_spends_sales_plot(channel):
+#     if channel is not None:
+#         x = channel.dates
+#         _spends = channel.actual_spends * channel.conversion_rate
+#         _sales = channel.actual_sales
+#         channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+#         channel_sales_spends_fig.add_trace(go.Bar(x=x, y=_sales,marker_color='#c1f7dc',name='Revenue', hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), secondary_y = False)
+#         channel_sales_spends_fig.add_trace(go.Scatter(x=x, y=_spends,line=dict(color='#005b96'),name='Spends',hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}"), secondary_y = True)
+#         channel_sales_spends_fig.update_layout(xaxis_title='Date',yaxis_title='Revenue',yaxis2_title='Spends ($)',title='Channel spends and Revenue week wise')
+#         channel_sales_spends_fig.update_xaxes(showgrid=False)
+#         channel_sales_spends_fig.update_yaxes(showgrid=False)
+#     else:
+#         raw_df = st.session_state['raw_df']
+#         df = raw_df.sort_values(by='Date')
+#         x = df.Date
+#         scenario = class_from_dict(st.session_state['default_scenario_dict'])
+#         _sales = scenario.constant + scenario.correction
+#         channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+#         channel_sales_spends_fig.add_trace(go.Bar(x=x, y=_sales,marker_color='#c1f7dc',name='Revenue', hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), secondary_y = False)
+#         # channel_sales_spends_fig.add_trace(go.Scatter(x=x, y=_spends,line=dict(color='#15C39A'),name='Spends',hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}"), secondary_y = True)
+#         channel_sales_spends_fig.update_layout(xaxis_title='Date',yaxis_title='Revenue',yaxis2_title='Spends ($)',title='Channel spends and Revenue week wise')
+#         channel_sales_spends_fig.update_xaxes(showgrid=False)
+#         channel_sales_spends_fig.update_yaxes(showgrid=False)
+#     return channel_sales_spends_fig
+
+
+# Define a shared color palette
+
+
+def create_contribution_pie():
+    color_palette = [
+        "#F3F3F0",
+        "#5E7D7E",
+        "#2FA1FF",
+        "#00EDED",
+        "#00EAE4",
+        "#304550",
+        "#EDEBEB",
+        "#7FBEFD",
+        "#003059",
+        "#A2F3F3",
+        "#E1D6E2",
+        "#B6B6B6",
+    ]
+    total_contribution_fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=["Spends", "Revenue"],
+        specs=[[{"type": "pie"}, {"type": "pie"}]],
+    )
+
+    channels_list = [
+        "Paid Search",
+        "Ga will cid baixo risco",
+        "Digital tactic others",
+        "Fb la tier 1",
+        "Fb la tier 2",
+        "Paid social others",
+        "Programmatic",
+        "Kwai",
+        "Indicacao",
+        "Infleux",
+        "Influencer",
+        "Non Media",
+    ]
+
+    # Assign colors from the limited palette to channels
+    colors_map = {
+        col: color_palette[i % len(color_palette)]
+        for i, col in enumerate(channels_list)
+    }
+    colors_map["Non Media"] = color_palette[
+        5
+    ]  # Assign fixed green color for 'Non Media'
+
+    # Hardcoded values for Spends and Revenue
+    spends_values = [0.5, 3.36, 1.1, 2.7, 2.7, 2.27, 70.6, 1, 1, 13.7, 1, 0]
+    revenue_values = [1, 4, 5, 3, 3, 2, 50.8, 1.5, 0.7, 13, 0, 16]
+
+    # Add trace for Spends pie chart
+    total_contribution_fig.add_trace(
+        go.Pie(
+            labels=[channel_name for channel_name in channels_list],
+            values=spends_values,
+            marker=dict(
+                colors=[colors_map[channel_name] for channel_name in channels_list]
+            ),
+            hole=0.3,
+        ),
+        row=1,
+        col=1,
+    )
+
+    # Add trace for Revenue pie chart
+    total_contribution_fig.add_trace(
+        go.Pie(
+            labels=[channel_name for channel_name in channels_list],
+            values=revenue_values,
+            marker=dict(
+                colors=[colors_map[channel_name] for channel_name in channels_list]
+            ),
+            hole=0.3,
+        ),
+        row=1,
+        col=2,
+    )
+
+    total_contribution_fig.update_traces(
+        textposition="inside", texttemplate="%{percent:.1%}"
+    )
+    total_contribution_fig.update_layout(
+        uniformtext_minsize=12,
+        title="Channel contribution",
+        uniformtext_mode="hide",
+    )
+    return total_contribution_fig
+
+
+def create_contribuion_stacked_plot(scenario):
+    weekly_contribution_fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=["Spends", "Revenue"],
+        specs=[[{"type": "bar"}, {"type": "bar"}]],
+    )
+    raw_df = st.session_state["raw_df"]
+    df = raw_df.sort_values(by="Date")
+    x = df.Date
+    weekly_spends_data = []
+    weekly_sales_data = []
+
+    for i, channel_name in enumerate(st.session_state["channels_list"]):
+        color = color_palette[i % len(color_palette)]
+
+        weekly_spends_data.append(
+            go.Bar(
+                x=x,
+                y=scenario.channels[channel_name].actual_spends
+                * scenario.channels[channel_name].conversion_rate,
+                name=channel_name_formating(channel_name),
+                hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+                legendgroup=channel_name,
+                marker_color=color,
+            )
+        )
+
+        weekly_sales_data.append(
+            go.Bar(
+                x=x,
+                y=scenario.channels[channel_name].actual_sales,
+                name=channel_name_formating(channel_name),
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+                legendgroup=channel_name,
+                showlegend=False,
+                marker_color=color,
+            )
+        )
+
+    for _d in weekly_spends_data:
+        weekly_contribution_fig.add_trace(_d, row=1, col=1)
+    for _d in weekly_sales_data:
+        weekly_contribution_fig.add_trace(_d, row=1, col=2)
+
+    weekly_contribution_fig.add_trace(
+        go.Bar(
+            x=x,
+            y=scenario.constant + scenario.correction,
+            name="Non Media",
+            hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            marker_color=color_palette[-1],
+        ),
+        row=1,
+        col=2,
+    )
+
+    weekly_contribution_fig.update_layout(
+        barmode="stack",
+        title="Channel contribution by week",
+        xaxis_title="Date",
+    )
+    weekly_contribution_fig.update_xaxes(showgrid=False)
+    weekly_contribution_fig.update_yaxes(showgrid=False)
+    return weekly_contribution_fig
+
+
+def create_channel_spends_sales_plot(channel):
+    if channel is not None:
+        x = channel.dates
+        _spends = channel.actual_spends * channel.conversion_rate
+        _sales = channel.actual_sales
+        channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+        channel_sales_spends_fig.add_trace(
+            go.Bar(
+                x=x,
+                y=_sales,
+                marker_color=color_palette[
+                    3
+                ],  # You can choose a color from the palette
+                name="Revenue",
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            ),
+            secondary_y=False,
+        )
+
+        channel_sales_spends_fig.add_trace(
+            go.Scatter(
+                x=x,
+                y=_spends,
+                line=dict(
+                    color=color_palette[2]
+                ),  # You can choose another color from the palette
+                name="Spends",
+                hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+            ),
+            secondary_y=True,
+        )
+
+        channel_sales_spends_fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title="Revenue",
+            yaxis2_title="Spends ($)",
+            title="Channel spends and Revenue week-wise",
+        )
+        channel_sales_spends_fig.update_xaxes(showgrid=False)
+        channel_sales_spends_fig.update_yaxes(showgrid=False)
+    else:
+        raw_df = st.session_state["raw_df"]
+        df = raw_df.sort_values(by="Date")
+        x = df.Date
+        scenario = class_from_dict(st.session_state["default_scenario_dict"])
+        _sales = scenario.constant + scenario.correction
+        channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+        channel_sales_spends_fig.add_trace(
+            go.Bar(
+                x=x,
+                y=_sales,
+                marker_color=color_palette[
+                    0
+                ],  # You can choose a color from the palette
+                name="Revenue",
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            ),
+            secondary_y=False,
+        )
+
+        channel_sales_spends_fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title="Revenue",
+            yaxis2_title="Spends ($)",
+            title="Channel spends and Revenue week-wise",
+        )
+        channel_sales_spends_fig.update_xaxes(showgrid=False)
+        channel_sales_spends_fig.update_yaxes(showgrid=False)
+
+    return channel_sales_spends_fig
+
+
+def format_numbers(value, n_decimals=1, include_indicator=True):
+    if value is None:
+        return None
+    _value = value if value < 1 else numerize(value, n_decimals)
+    if include_indicator:
+        return f"{CURRENCY_INDICATOR} {_value}"
+    else:
+        return f"{_value}"
+
+
+def decimal_formater(num_string, n_decimals=1):
+    parts = num_string.split(".")
+    if len(parts) == 1:
+        return num_string + "." + "0" * n_decimals
+    else:
+        to_be_padded = n_decimals - len(parts[-1])
+        if to_be_padded > 0:
+            return num_string + "0" * to_be_padded
+        else:
+            return num_string
+
+
+def channel_name_formating(channel_name):
+    name_mod = channel_name.replace("_", " ")
+    if name_mod.lower().endswith(" imp"):
+        name_mod = name_mod.replace("Imp", "Spend")
+    elif name_mod.lower().endswith(" clicks"):
+        name_mod = name_mod.replace("Clicks", "Spend")
+    return name_mod
+
+
+def send_email(email, message):
+    s = smtplib.SMTP("smtp.gmail.com", 587)
+    s.starttls()
+    s.login("[email protected]", "jgydhpfusuremcol")
+    s.sendmail("[email protected]", email, message)
+    s.quit()
+
+
+# if __name__ == "__main__":
+#     initialize_data()
+
+
+#############################################################################################################
+
+import os
+import json
+import streamlit as st
+
+
+# Function to get panels names
+def get_panels_names(file_selected):
+    raw_data_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][file_selected]["raw_data"]
+
+    if "panel" in raw_data_df.columns:
+        panel = list(set(raw_data_df["panel"]))
+    elif "Panel" in raw_data_df.columns:
+        panel = list(set(raw_data_df["Panel"]))
+    else:
+        panel = []
+
+    return panel + ["aggregated"]
+
+
+# Function to get metrics names
+def get_metrics_names():
+    return list(
+        st.session_state["project_dct"]["current_media_performance"][
+            "model_outputs"
+        ].keys()
+    )
+
+
+# Function to load the original and modified rcs metadata files into dictionaries
+def load_rcs_metadata_files():
+    original_data = st.session_state["project_dct"]["response_curves"][
+        "original_metadata_file"
+    ]
+    modified_data = st.session_state["project_dct"]["response_curves"][
+        "modified_metadata_file"
+    ]
+
+    return original_data, modified_data
+
+
+# Function to format name
+def name_formating(name):
+    # Replace underscores with spaces
+    name_mod = name.replace("_", " ")
+
+    # Capitalize the first letter of each word
+    name_mod = name_mod.title()
+
+    return name_mod
+
+
+# Function to load the original and modified scenario metadata files into dictionaries
+def load_scenario_metadata_files():
+    original_data = st.session_state["project_dct"]["scenario_planner"][
+        "original_metadata_file"
+    ]
+    modified_data = st.session_state["project_dct"]["scenario_planner"][
+        "modified_metadata_file"
+    ]
+
+    return original_data, modified_data
+
+
+# Function to generate RCS data and store it as dictionary
+def generate_rcs_data():
+    # Retrieve the list of all metric names from the specified directory
+    metrics_list = get_metrics_names()
+
+    # Dictionary to store RCS data for all metrics and their respective panels
+    all_rcs_data_original = {}
+    all_rcs_data_modified = {}
+
+    # Iterate over each metric in the metrics list
+    for metric in metrics_list:
+        # Retrieve the list of panel names from the current metric's Excel file
+        panel_list = get_panels_names(file_selected=metric)
+
+        # Check if rcs_data_modified exist
+        if (
+            st.session_state["project_dct"]["response_curves"]["modified_metadata_file"]
+            is not None
+        ):
+            modified_data = st.session_state["project_dct"]["response_curves"][
+                "modified_metadata_file"
+            ]
+
+        # Iterate over each panel in the panel list
+        for panel in panel_list:
+            # Initialize the original RCS data for the current panel and metric
+            rcs_dict_original, scenario = initialize_data(
+                panel=panel,
+                metrics=metric,
+            )
+
+            # Ensure the dictionary has the metric as a key for original data
+            if metric not in all_rcs_data_original:
+                all_rcs_data_original[metric] = {}
+
+            # Store the original RCS data under the corresponding panel for the current metric
+            all_rcs_data_original[metric][panel] = rcs_dict_original
+
+            # Ensure the dictionary has the metric as a key for modified data
+            if metric not in all_rcs_data_modified:
+                all_rcs_data_modified[metric] = {}
+
+            # Store the modified RCS data under the corresponding panel for the current metric
+            for channel in rcs_dict_original:
+                all_rcs_data_modified[metric][panel] = all_rcs_data_modified[
+                    metric
+                ].get(panel, {})
+
+                try:
+                    updated_rcs_dict = modified_data[metric][panel][channel]
+                except:
+                    updated_rcs_dict = {
+                        "K": rcs_dict_original[channel]["K"],
+                        "b": rcs_dict_original[channel]["b"],
+                        "a": rcs_dict_original[channel]["a"],
+                        "x0": rcs_dict_original[channel]["x0"],
+                    }
+
+                all_rcs_data_modified[metric][panel][channel] = updated_rcs_dict
+
+    # Write the original RCS data
+    st.session_state["project_dct"]["response_curves"][
+        "original_metadata_file"
+    ] = all_rcs_data_original
+
+    # Write the modified RCS data
+    st.session_state["project_dct"]["response_curves"][
+        "modified_metadata_file"
+    ] = all_rcs_data_modified
+
+
+# Function to generate scenario data and store it as dictionary
+def generate_scenario_data():
+    # Retrieve the list of all metric names from the specified directory
+    metrics_list = get_metrics_names()
+
+    # Dictionary to store scenario data for all metrics and their respective panels
+    all_scenario_data_original = {}
+    all_scenario_data_modified = {}
+
+    # Iterate over each metric in the metrics list
+    for metric in metrics_list:
+        # Retrieve the list of panel names from the current metric's Excel file
+        panel_list = get_panels_names(metric)
+
+        # Check if scenario_data_modified exist
+        if (
+            st.session_state["project_dct"]["scenario_planner"][
+                "modified_metadata_file"
+            ]
+            is not None
+        ):
+            modified_data = st.session_state["project_dct"]["scenario_planner"][
+                "modified_metadata_file"
+            ]
+
+        # Iterate over each panel in the panel list
+        for panel in panel_list:
+            # Initialize the original scenario data for the current panel and metric
+            rcs_dict_original, scenario = initialize_data(
+                panel=panel,
+                metrics=metric,
+            )
+
+            # Ensure the dictionary has the metric as a key for original data
+            if metric not in all_scenario_data_original:
+                all_scenario_data_original[metric] = {}
+
+            # Store the original scenario data under the corresponding panel for the current metric
+            all_scenario_data_original[metric][panel] = class_convert_to_dict(scenario)
+
+            # Ensure the dictionary has the metric as a key for modified data
+            if metric not in all_scenario_data_modified:
+                all_scenario_data_modified[metric] = {}
+
+            # Store the modified scenario data under the corresponding panel for the current metric
+            try:
+                all_scenario_data_modified[metric][panel] = modified_data[metric][panel]
+            except:
+                all_scenario_data_modified[metric][panel] = class_convert_to_dict(
+                    scenario
+                )
+
+    # Write the original scenario data
+    st.session_state["project_dct"]["scenario_planner"][
+        "original_metadata_file"
+    ] = all_scenario_data_original
+
+    # Write the modified scenario data
+    st.session_state["project_dct"]["scenario_planner"][
+        "modified_metadata_file"
+    ] = all_scenario_data_modified
+
+
+#############################################################################################################

utilities_with_panel.py

@@ -0,0 +1,1520 @@
+from scenario import numerize
+import streamlit as st
+import pandas as pd
+import json
+from scenario import Channel, Scenario
+import numpy as np
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+from scenario import class_to_dict
+from collections import OrderedDict
+import io
+import plotly
+from pathlib import Path
+import pickle
+import yaml
+from yaml import SafeLoader
+from streamlit.components.v1 import html
+import smtplib
+from scipy.optimize import curve_fit
+from sklearn.metrics import r2_score
+from scenario import class_from_dict
+from utilities import retrieve_pkl_object
+import os
+import base64
+
+
+
+
+# # schema = db_cred["schema"]
+
+color_palette = [
+    "#F3F3F0",
+    "#5E7D7E",
+    "#2FA1FF",
+    "#00EDED",
+    "#00EAE4",
+    "#304550",
+    "#EDEBEB",
+    "#7FBEFD",
+    "#003059",
+    "#A2F3F3",
+    "#E1D6E2",
+    "#B6B6B6",
+]
+
+
+CURRENCY_INDICATOR = "$"
+
+
+if "project_dct" not in st.session_state or "project_number" not in st.session_state:
+    st.error(
+        "No tuned model available. Please build and tune a model to generate response curves."
+    )
+    st.stop()
+
+tuned_model = retrieve_pkl_object(
+    st.session_state["project_number"], "Model_Tuning", "tuned_model"
+)
+
+if tuned_model is None:
+    st.error(
+        "No tuned model available. Please build and tune a model to generate response curves."
+    )
+    st.stop()
+
+
+def load_authenticator():
+    with open("config.yaml") as file:
+        config = yaml.load(file, Loader=SafeLoader)
+        st.session_state["config"] = config
+    authenticator = stauth.Authenticate(
+        config["credentials"],
+        config["cookie"]["name"],
+        config["cookie"]["key"],
+        config["cookie"]["expiry_days"],
+        config["preauthorized"],
+    )
+    st.session_state["authenticator"] = authenticator
+    return authenticator
+
+
+def nav_page(page_name, timeout_secs=3):
+    nav_script = """
+        <script type="text/javascript">
+            function attempt_nav_page(page_name, start_time, timeout_secs) {
+                var links = window.parent.document.getElementsByTagName("a");
+                for (var i = 0; i < links.length; i++) {
+                    if (links[i].href.toLowerCase().endsWith("/" + page_name.toLowerCase())) {
+                        links[i].click();
+                        return;
+                    }
+                }
+                var elasped = new Date() - start_time;
+                if (elasped < timeout_secs * 1000) {
+                    setTimeout(attempt_nav_page, 100, page_name, start_time, timeout_secs);
+                } else {
+                    alert("Unable to navigate to page '" + page_name + "' after " + timeout_secs + " second(s).");
+                }
+            }
+            window.addEventListener("load", function() {
+                attempt_nav_page("?", new Date(), %d);
+            });
+        </script>
+    """ % (
+        page_name,
+        timeout_secs,
+    )
+    html(nav_script)
+
+
+# def load_local_css(file_name):
+#     with open(file_name) as f:
+#         st.markdown(f'<style>{f.read()}</style>', unsafe_allow_html=True)
+
+
+# def set_header():
+#     return st.markdown(f"""<div class='main-header'>
+#                     <h1>MMM LiME</h1>
+#                     <img src="https://assets-global.website-files.com/64c8fffb0e95cbc525815b79/64df84637f83a891c1473c51_Vector%20(Stroke).svg   ">
+#             </div>""", unsafe_allow_html=True)
+
+path = os.path.dirname(__file__)
+
+file_ = open(f"{path}/logo.png", "rb")
+
+contents = file_.read()
+
+data_url = base64.b64encode(contents).decode("utf-8")
+
+file_.close()
+
+
+DATA_PATH = "./data"
+
+IMAGES_PATH = "./data/images_224_224"
+
+
+# is_panel = True if len(panel_col) > 0 else False
+
+# manoj
+
+is_panel = False
+date_col = "Date"
+# is_panel = False # flag if set to true - do panel level response curves
+
+
+def load_local_css(file_name):
+
+    with open(file_name) as f:
+
+        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+
+
+# def set_header():
+
+#     return st.markdown(f"""<div class='main-header'>
+
+#                     <h1>H & M Recommendations</h1>
+
+#                     <img src="data:image;base64,{data_url}", alt="Logo">
+
+#             </div>""", unsafe_allow_html=True)
+path1 = os.path.dirname(__file__)
+
+# file_1 = open(f"{path}/willbank.png", "rb")
+
+# contents1 = file_1.read()
+
+# data_url1 = base64.b64encode(contents1).decode("utf-8")
+
+# file_1.close()
+
+
+DATA_PATH1 = "./data"
+
+IMAGES_PATH1 = "./data/images_224_224"
+
+
+def set_header():
+    return st.markdown(
+        f"""<div class='main-header'>
+                    <!-- <h1></h1> -->
+                       <div >
+                       <img class='blend-logo' src="data:image;base64,{data_url}", alt="Logo">
+                       </div>
+                    <img class='blend-logo' src="data:image;base64,{data_url}", alt="Logo">
+            </div>""",
+        unsafe_allow_html=True,
+    )
+
+
+# def set_header():
+#     logo_path = "./path/to/your/local/LIME_logo.png"  # Replace with the actual file path
+#     text = "LiME"
+#     return st.markdown(f"""<div class='main-header'>
+#                     <img src="data:image/png;base64,{data_url}" alt="Logo" style="float: left; margin-right: 10px; width: 100px; height: auto;">
+#                     <h1>{text}</h1>
+#             </div>""", unsafe_allow_html=True)
+
+
+def s_curve(x, K, b, a, x0):
+    return K / (1 + b * np.exp(-a * (x - x0)))
+
+
+def overview_test_data_prep_panel(X, df, spends_X, date_col, panel_col, target_col):
+    """
+    function to create the data which is used in initialize data fn
+    X : X test with contributions
+    df : originally uploaded data (media data) which has raw vars
+    spends_X : spends of dates in X test
+    """
+
+    channels = st.session_state["channels"]
+    channel_list = channels.keys()
+
+    # map transformed variable to raw variable name & channel name
+    # mapping eg : paid_search_clicks_lag_2 (transformed var) --> paid_search_clicks (raw var) --> paid_search (channel)
+    variables = {}
+    channel_and_variables = {}
+    new_variables = {}
+    new_channels_and_variables = {}
+
+    for transformed_var in [
+        col
+        for col in X.drop(columns=[date_col, panel_col, target_col, "pred"]).columns
+        if "_contr" not in col
+    ]:
+        if len([col for col in df.columns if col in transformed_var]) == 1:
+            raw_var = [col for col in df.columns if col in transformed_var][0]
+
+            variables[transformed_var] = raw_var
+
+            # Check if the list comprehension result is not empty before accessing the first element
+            channels_list = [
+                channel for channel, raw_vars in channels.items() if raw_var in raw_vars
+            ]
+            if channels_list:
+                channel_and_variables[raw_var] = channels_list[0]
+            else:
+                # Handle the case where channels_list is empty
+                # You might want to set a default value or handle it according to your use case
+                channel_and_variables[raw_var] = None
+        else:
+            new_variables[transformed_var] = transformed_var
+            new_channels_and_variables[transformed_var] = "base"
+
+    # Raw DF
+    raw_X = pd.merge(
+        X[[date_col, panel_col]],
+        df[[date_col, panel_col] + list(variables.values())],
+        how="left",
+        on=[date_col, panel_col],
+    )
+    assert len(raw_X) == len(X)
+
+    raw_X_cols = []
+    for i in raw_X.columns:
+        if i in channel_and_variables.keys():
+            raw_X_cols.append(channel_and_variables[i])
+        else:
+            raw_X_cols.append(i)
+    raw_X.columns = raw_X_cols
+
+    # Contribution DF
+    contr_X = X[
+        [date_col, panel_col]
+        + [col for col in X.columns if "_contr" in col and "sum_" not in col]
+    ].copy()
+    # if "base_contr" in contr_X.columns:
+    #     contr_X.rename(columns={'base_contr':'const_contr'},inplace=True)
+    # # new_variables = [
+    #     col
+    #     for col in contr_X.columns
+    #     if "_flag" in col.lower() or "trend" in col.lower() or "sine" in col.lower()
+    # ]
+    # if len(new_variables) > 0:
+    #     contr_X["const"] = contr_X[["panel_effect"] + new_variables].sum(axis=1)
+    #     contr_X.drop(columns=["panel_effect"], inplace=True)
+    #     contr_X.drop(columns=new_variables, inplace=True)
+    # else:
+    #     contr_X.rename(columns={"panel_effect": "const"}, inplace=True)
+
+    new_contr_X_cols = []
+    for col in contr_X.columns:
+        col_clean = col.replace("_contr", "")
+        new_contr_X_cols.append(col_clean)
+    contr_X.columns = new_contr_X_cols
+
+    contr_X_cols = []
+    for i in contr_X.columns:
+        if i in variables.keys():
+            contr_X_cols.append(channel_and_variables[variables[i]])
+        else:
+            contr_X_cols.append(i)
+    contr_X.columns = contr_X_cols
+
+    # Spends DF
+    spends_X.columns = [col.replace("_cost", "") for col in spends_X.columns]
+
+    raw_X.rename(columns={"date": "Date"}, inplace=True)
+    contr_X.rename(columns={"date": "Date"}, inplace=True)
+    spends_X.rename(columns={"date": "Week"}, inplace=True)
+
+    spends_X.columns = [
+        col.replace("spends_", "") if col.startswith("spends_") else col
+        for col in spends_X.columns
+    ]
+
+    # Rename column to 'Date'
+    spends_X.rename(columns={"Week": "Date"}, inplace=True)
+
+    # Remove "response_metric_" from the start and "_total" from the end
+    if str(target_col).startswith("response_metric_"):
+        target_col = target_col.replace("response_metric_", "", 1)
+
+    # Remove the last 6 characters (length of "_total")
+    if str(target_col).endswith("_total"):
+        target_col = target_col[:-6]
+
+    # Rename column to 'Date'
+    spends_X.rename(columns={"Week": "Date"}, inplace=True)
+
+    # Save raw, spends and contribution data
+    st.session_state["project_dct"]["current_media_performance"]["model_outputs"][
+        target_col
+    ] = {
+        "raw_data": raw_X,
+        "contribution_data": contr_X,
+        "spends_data": spends_X,
+    }
+
+    # Clear page metadata
+    st.session_state["project_dct"]["scenario_planner"]["original_metadata_file"] = None
+    st.session_state["project_dct"]["response_curves"]["original_metadata_file"] = None
+
+    # st.session_state["project_dct"]["scenario_planner"]["modified_metadata_file"] = None
+    # st.session_state["project_dct"]["response_curves"]["modified_metadata_file"] = None
+
+
+def overview_test_data_prep_nonpanel(X, df, spends_X, date_col, target_col):
+    """
+    function to create the data which is used in initialize data fn
+    """
+
+    # with open(
+    #     os.path.join(st.session_state["project_path"], "channel_groups.pkl"), "rb"
+    # ) as f:
+    #     channels = pickle.load(f)
+
+    # channel_list = list(channels.keys())
+    channels = st.session_state["channels"]
+    channel_list = channels.keys()
+
+    # map transformed variable to raw variable name & channel name
+    # mapping eg : paid_search_clicks_lag_2 (transformed var) --> paid_search_clicks (raw var) --> paid_search (channel)
+    variables = {}
+    channel_and_variables = {}
+    new_variables = {}
+    new_channels_and_variables = {}
+
+    cols_to_del = list(
+        set([date_col, target_col, "pred"]).intersection((set(X.columns)))
+    )
+
+    # remove exog cols from RAW data (exog cols are part of base, raw data needs media vars only)
+    all_exog_vars = st.session_state["bin_dict"]["Exogenous"]
+    all_exog_vars = [
+        var.lower()
+        .replace(".", "_")
+        .replace("@", "_")
+        .replace(" ", "_")
+        .replace("-", "")
+        .replace(":", "")
+        .replace("__", "_")
+        for var in all_exog_vars
+    ]
+    exog_cols = []
+    if len(all_exog_vars) > 0:
+        for col in X.columns:
+            if len([exog_var for exog_var in all_exog_vars if exog_var in col]) > 0:
+                exog_cols.append(col)
+    cols_to_del = cols_to_del + exog_cols
+    for transformed_var in [
+        col for col in X.drop(columns=cols_to_del).columns if "_contr" not in col
+    ]:  # also has 'const'
+
+        if (
+            len([col for col in df.columns if col in transformed_var]) == 1
+        ):  # col is raw var
+            raw_var = [col for col in df.columns if col in transformed_var][0]
+            variables[transformed_var] = raw_var
+            channel_and_variables[raw_var] = [
+                channel for channel, raw_vars in channels.items() if raw_var in raw_vars
+            ][0]
+        else:  # when no corresponding raw var then base
+            new_variables[transformed_var] = transformed_var
+            new_channels_and_variables[transformed_var] = "base"
+
+    # Raw DF
+    raw_X = pd.merge(
+        X[[date_col]],
+        df[[date_col] + list(variables.values())],
+        how="left",
+        on=[date_col],
+    )
+    assert len(raw_X) == len(X)
+
+    raw_X_cols = []
+    for i in raw_X.columns:
+        if i in channel_and_variables.keys():
+            raw_X_cols.append(channel_and_variables[i])
+        else:
+            raw_X_cols.append(i)
+    raw_X.columns = raw_X_cols
+
+    # Contribution DF
+    contr_X = X[
+        [date_col] + [col for col in X.columns if "_contr" in col and "sum_" not in col]
+    ].copy()
+
+    new_variables = [
+        col
+        for col in contr_X.columns
+        if "_flag" in col.lower() or "trend" in col.lower() or "sine" in col.lower()
+    ]
+    if (
+        len(new_variables) > 0
+    ):  # if new vars are available, their contributions should be added to base (called const)
+        contr_X["const_contr"] = contr_X[["const_contr"] + new_variables].sum(axis=1)
+        contr_X.drop(columns=new_variables, inplace=True)
+
+    new_contr_X_cols = []
+    for col in contr_X.columns:
+        col_clean = col.replace("_contr", "")
+        new_contr_X_cols.append(col_clean)
+    contr_X.columns = new_contr_X_cols
+
+    contr_X_cols = []
+    for i in contr_X.columns:
+        if i in variables.keys():
+            contr_X_cols.append(channel_and_variables[variables[i]])
+        else:
+            contr_X_cols.append(i)
+    contr_X.columns = contr_X_cols
+
+    # Spends DF
+    # spends_X.columns = [
+    #     col.replace("_cost", "").replace("_spends", "").replace("_spend", "")
+    #     for col in spends_X.columns
+    # ]
+    spends_X_col_map = {
+        col: bucket
+        for col in spends_X.columns
+        for bucket in channels.keys()
+        if col in channels[bucket]
+    }
+    spends_X.rename(columns=spends_X_col_map, inplace=True)
+
+    raw_X.rename(columns={"date": "Date"}, inplace=True)
+    contr_X.rename(columns={"date": "Date"}, inplace=True)
+    spends_X.rename(columns={"date": "Week"}, inplace=True)
+
+    spends_X.columns = [
+        col.replace("spends_", "") if col.startswith("spends_") else col
+        for col in spends_X.columns
+    ]
+
+    # Rename column to 'Date'
+    spends_X.rename(columns={"Week": "Date"}, inplace=True)
+
+    # Remove "response_metric_" from the start and "_total" from the end
+    if str(target_col).startswith("response_metric_"):
+        target_col = target_col.replace("response_metric_", "", 1)
+
+    # Remove the last 6 characters (length of "_total")
+    if str(target_col).endswith("_total"):
+        target_col = target_col[:-6]
+
+    # Rename column to 'Date'
+    spends_X.rename(columns={"Week": "Date"}, inplace=True)
+
+    # Save raw, spends and contribution data
+    st.session_state["project_dct"]["current_media_performance"]["model_outputs"][
+        target_col
+    ] = {
+        "raw_data": raw_X,
+        "contribution_data": contr_X,
+        "spends_data": spends_X,
+    }
+
+    # Clear page metadata
+    st.session_state["project_dct"]["scenario_planner"]["original_metadata_file"] = None
+    st.session_state["project_dct"]["response_curves"]["original_metadata_file"] = None
+
+    # st.session_state["project_dct"]["scenario_planner"]["modified_metadata_file"] = None
+    # st.session_state["project_dct"]["response_curves"]["modified_metadata_file"] = None
+
+
+def initialize_data_cmp(target_col, is_panel, panel_col, start_date, end_date):
+    start_date = pd.to_datetime(start_date)
+    end_date = pd.to_datetime(end_date)
+
+    # Remove "response_metric_" from the start and "_total" from the end
+    if str(target_col).startswith("response_metric_"):
+        target_col = target_col.replace("response_metric_", "", 1)
+
+    # Remove the last 6 characters (length of "_total")
+    if str(target_col).endswith("_total"):
+        target_col = target_col[:-6]
+
+    # Extract dataframes for raw data, spend input, and contribution data
+    raw_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][target_col]["raw_data"]
+    spend_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][target_col]["spends_data"]
+    contri_df = st.session_state["project_dct"]["current_media_performance"][
+        "model_outputs"
+    ][target_col]["contribution_data"]
+
+    # Remove unnecessary columns
+    unnamed_cols = [col for col in raw_df.columns if col.lower().startswith("unnamed")]
+    exclude_columns = ["Date"] + unnamed_cols
+
+    if is_panel:
+        exclude_columns = exclude_columns + [panel_col]
+
+    # Aggregate all 3 dfs to date level (from date-panel level)
+    raw_df[date_col] = pd.to_datetime(raw_df[date_col])
+    raw_df = raw_df[raw_df[date_col] >= start_date]
+    raw_df = raw_df[raw_df[date_col] <= end_date].reset_index(drop=True)
+    raw_df_aggregations = {c: "sum" for c in raw_df.columns if c not in exclude_columns}
+    raw_df = raw_df.groupby(date_col).agg(raw_df_aggregations).reset_index()
+
+    contri_df[date_col] = pd.to_datetime(contri_df[date_col])
+    contri_df = contri_df[contri_df[date_col] >= start_date]
+    contri_df = contri_df[contri_df[date_col] <= end_date].reset_index(drop=True)
+    contri_df_aggregations = {
+        c: "sum" for c in contri_df.columns if c not in exclude_columns
+    }
+    contri_df = contri_df.groupby(date_col).agg(contri_df_aggregations).reset_index()
+
+    input_df = raw_df.sort_values(by=[date_col])
+
+    output_df = contri_df.sort_values(by=[date_col])
+    spend_df["Date"] = pd.to_datetime(
+        spend_df["Date"], format="%Y-%m-%d", errors="coerce"
+    )
+    spend_df = spend_df[spend_df["Date"] >= start_date]
+    spend_df = spend_df[spend_df["Date"] <= end_date].reset_index(drop=True)
+    spend_df_aggregations = {
+        c: "sum" for c in spend_df.columns if c not in exclude_columns
+    }
+    spend_df = spend_df.groupby("Date").agg(spend_df_aggregations).reset_index()
+    # spend_df['Week'] = pd.to_datetime(spend_df['Week'], errors='coerce')
+    # spend_df = spend_df.sort_values(by='Week')
+
+    channel_list = [col for col in input_df.columns if col not in exclude_columns]
+
+    response_curves = {}
+    mapes = {}
+    rmses = {}
+    upper_limits = {}
+    powers = {}
+    r2 = {}
+    conv_rates = {}
+    output_cols = []
+    channels = {}
+    sales = None
+    dates = input_df.Date.values
+    actual_output_dic = {}
+    actual_input_dic = {}
+
+    # channel_list=['programmatic']
+    infeasible_channels = [
+        c
+        for c in contri_df.select_dtypes(include=["float", "int"]).columns
+        if contri_df[c].sum() <= 0
+    ]
+
+    channel_list = list(set(channel_list) - set(infeasible_channels))
+
+    for inp_col in channel_list:
+        spends = input_df[inp_col].values
+
+        x = spends.copy()
+
+        # upper limit for penalty
+        upper_limits[inp_col] = 2 * x.max()
+
+        out_col = inp_col
+        y = output_df[out_col].values.copy()
+
+        actual_output_dic[inp_col] = y.copy()
+        actual_input_dic[inp_col] = x.copy()
+        ##output cols aggregation
+        output_cols.append(out_col)
+
+        ## scale the input
+        power = np.ceil(np.log(x.max()) / np.log(10)) - 3
+        if power >= 0:
+            x = x / 10**power
+
+        x = x.astype("float64")
+        y = y.astype("float64")
+
+        if y.max() <= 0.01:
+            bounds = (
+                (0, 0, 0, 0),
+                (3 * 0.01, 1000, 1, x.max() if x.max() > 0 else 0.01),
+            )
+        else:
+            bounds = ((0, 0, 0, 0), (3 * y.max(), 1000, 1, x.max()))
+
+        params, _ = curve_fit(
+            s_curve,
+            x,
+            y,
+            p0=(2 * y.max(), 0.01, 1e-5, x.max()),
+            bounds=bounds,
+            maxfev=int(1e5),
+        )
+        mape = (100 * abs(1 - s_curve(x, *params) / y.clip(min=1))).mean()
+        rmse = np.sqrt(((y - s_curve(x, *params)) ** 2).mean())
+        r2_ = r2_score(y, s_curve(x, *params))
+
+        response_curves[inp_col] = {
+            "K": params[0],
+            "b": params[1],
+            "a": params[2],
+            "x0": params[3],
+        }
+        mapes[inp_col] = mape
+        rmses[inp_col] = rmse
+        r2[inp_col] = r2_
+        powers[inp_col] = power
+
+        conv = spend_df[inp_col].sum() / max(input_df[inp_col].sum(), 1e-9)
+        conv_rates[inp_col] = conv
+
+        channel = Channel(
+            name=inp_col,
+            dates=dates,
+            spends=spends,
+            # conversion_rate = np.mean(list(conv_rates[inp_col].values())),
+            conversion_rate=conv_rates[inp_col],
+            response_curve_type="s-curve",
+            response_curve_params={
+                "K": params[0],
+                "b": params[1],
+                "a": params[2],
+                "x0": params[3],
+            },
+            bounds=np.array([-10, 10]),
+            correction=y - s_curve(x, *params),
+        )
+        channels[inp_col] = channel
+        if sales is None:
+            sales = channel.actual_sales
+        else:
+            sales += channel.actual_sales
+
+    other_contributions = (
+        output_df.drop([*output_cols], axis=1).sum(axis=1, numeric_only=True).values
+    )
+    correction = output_df.drop(["Date"], axis=1).sum(axis=1).values - (
+        sales + other_contributions
+    )
+
+    # Testing
+    # scenario_test_df = pd.DataFrame(
+    #     columns=["other_contributions", "correction", "sales"]
+    # )
+    # scenario_test_df["other_contributions"] = other_contributions
+    # scenario_test_df["correction"] = correction
+    # scenario_test_df["sales"] = sales
+    # scenario_test_df.to_csv("test\scenario_test_df.csv", index=False)
+    # output_df.to_csv("test\output_df.csv", index=False)
+
+    scenario = Scenario(
+        name="default",
+        channels=channels,
+        constant=other_contributions,
+        correction=correction,
+    )
+    ## setting session variables
+    st.session_state["initialized"] = True
+    st.session_state["actual_df"] = input_df
+    st.session_state["raw_df"] = raw_df
+    st.session_state["contri_df"] = output_df
+    default_scenario_dict = class_to_dict(scenario)
+    st.session_state["default_scenario_dict"] = default_scenario_dict
+    st.session_state["scenario"] = scenario
+    st.session_state["channels_list"] = channel_list
+    st.session_state["optimization_channels"] = {
+        channel_name: False for channel_name in channel_list
+    }
+    st.session_state["rcs"] = response_curves
+
+    # orig_rcs_path = os.path.join(
+    #     st.session_state["project_path"], f"orig_rcs_{target_col}_{panel_col}.json"
+    # )
+    # if Path(orig_rcs_path).exists():
+    #     with open(orig_rcs_path, "r") as f:
+    #         st.session_state["orig_rcs"] = json.load(f)
+    # else:
+    #     st.session_state["orig_rcs"] = response_curves.copy()
+    #     with open(orig_rcs_path, "w") as f:
+    #         json.dump(st.session_state["orig_rcs"], f)
+
+    st.session_state["powers"] = powers
+    st.session_state["actual_contribution_df"] = pd.DataFrame(actual_output_dic)
+    st.session_state["actual_input_df"] = pd.DataFrame(actual_input_dic)
+
+    for channel in channels.values():
+        st.session_state[channel.name] = numerize(
+            channel.actual_total_spends * channel.conversion_rate, 1
+        )
+
+    # st.session_state["xlsx_buffer"] = io.BytesIO()
+    #
+    # if Path("../saved_scenarios.pkl").exists():
+    #     with open("../saved_scenarios.pkl", "rb") as f:
+    #         st.session_state["saved_scenarios"] = pickle.load(f)
+    # else:
+    #     st.session_state["saved_scenarios"] = OrderedDict()
+    #
+    # st.session_state["total_spends_change"] = 0
+    # st.session_state["optimization_channels"] = {
+    #     channel_name: False for channel_name in channel_list
+    # }
+    # st.session_state["disable_download_button"] = True
+
+
+# def initialize_data():
+#     # fetch data from excel
+#     output = pd.read_excel('data.xlsx',sheet_name=None)
+#     raw_df = output['RAW DATA MMM']
+#     contribution_df = output['CONTRIBUTION MMM']
+#     Revenue_df = output['Revenue']
+
+#     ## channels to be shows
+#     channel_list = []
+#     for col in raw_df.columns:
+#         if 'click' in col.lower() or 'spend' in col.lower() or 'imp' in col.lower():
+#
+#             channel_list.append(col)
+#         else:
+#             pass
+
+#     ## NOTE : Considered only Desktop spends for all calculations
+#     acutal_df = raw_df[raw_df.Region == 'Desktop'].copy()
+#     ## NOTE : Considered one year of data
+#     acutal_df = acutal_df[acutal_df.Date>'2020-12-31']
+#     actual_df = acutal_df.drop('Region',axis=1).sort_values(by='Date')[[*channel_list,'Date']]
+
+#     ##load response curves
+#     with open('./grammarly_response_curves.json','r') as f:
+#         response_curves = json.load(f)
+
+#     ## create channel dict for scenario creation
+#     dates = actual_df.Date.values
+#     channels = {}
+#     rcs = {}
+#     constant = 0.
+#     for i,info_dict in enumerate(response_curves):
+#         name = info_dict.get('name')
+#         response_curve_type = info_dict.get('response_curve')
+#         response_curve_params = info_dict.get('params')
+#         rcs[name] = response_curve_params
+#         if name != 'constant':
+#             spends = actual_df[name].values
+#             channel = Channel(name=name,dates=dates,
+#                             spends=spends,
+#                             response_curve_type=response_curve_type,
+#                             response_curve_params=response_curve_params,
+#                             bounds=np.array([-30,30]))
+
+#             channels[name] = channel
+#         else:
+#             constant = info_dict.get('value',0.) * len(dates)
+
+#     ## create scenario
+#     scenario = Scenario(name='default', channels=channels, constant=constant)
+#     default_scenario_dict = class_to_dict(scenario)
+
+
+#     ## setting session variables
+#     st.session_state['initialized'] = True
+#     st.session_state['actual_df'] = actual_df
+#     st.session_state['raw_df'] = raw_df
+#     st.session_state['default_scenario_dict'] = default_scenario_dict
+#     st.session_state['scenario'] = scenario
+#     st.session_state['channels_list'] = channel_list
+#     st.session_state['optimization_channels'] = {channel_name : False for channel_name in channel_list}
+#     st.session_state['rcs'] = rcs
+#     for channel in channels.values():
+#         if channel.name not in st.session_state:
+#             st.session_state[channel.name] = float(channel.actual_total_spends)
+
+#     if 'xlsx_buffer' not in st.session_state:
+#         st.session_state['xlsx_buffer'] = io.BytesIO()
+
+#     ## for saving scenarios
+#     if 'saved_scenarios' not in st.session_state:
+#         if Path('../saved_scenarios.pkl').exists():
+#             with open('../saved_scenarios.pkl','rb') as f:
+#                 st.session_state['saved_scenarios'] = pickle.load(f)
+
+#         else:
+#             st.session_state['saved_scenarios'] = OrderedDict()
+
+#     if 'total_spends_change' not in st.session_state:
+#         st.session_state['total_spends_change'] = 0
+
+#     if 'optimization_channels' not in st.session_state:
+#         st.session_state['optimization_channels'] = {channel_name : False for channel_name in channel_list}
+
+
+#     if 'disable_download_button' not in st.session_state:
+#         st.session_state['disable_download_button'] = True
+def create_channel_summary(scenario, target_column):
+    def round_off(x, round_off_decimal=0):
+        # round off
+        try:
+            x = float(x)
+            if x < 1 and x > 0:
+                round_off_decimal = int(np.floor(np.abs(np.log10(x)))) + max(
+                    round_off_decimal, 1
+                )
+                x = np.round(x, round_off_decimal)
+            elif x < 0 and x > -1:
+                round_off_decimal = int(np.floor(np.abs(np.log10(np.abs(x))))) + max(
+                    round_off_decimal, 1
+                )
+                x = -np.round(x, round_off_decimal)
+            else:
+                x = np.round(x, round_off_decimal)
+            return x
+        except:
+            return x
+
+    summary_columns = []
+
+    actual_spends_rows = []
+
+    actual_sales_rows = []
+
+    actual_roi_rows = []
+
+    for channel in scenario.channels.values():
+
+        name_mod = channel.name.replace("_", " ")
+
+        if name_mod.lower().endswith(" imp"):
+            name_mod = name_mod.replace("Imp", " Impressions")
+
+        summary_columns.append(name_mod)
+
+        actual_spends_rows.append(
+            format_numbers(float(channel.actual_total_spends * channel.conversion_rate))
+        )
+
+        actual_sales_rows.append(format_numbers((float(channel.actual_total_sales))))
+
+        roi = (channel.actual_total_sales) / (
+            channel.actual_total_spends * channel.conversion_rate
+        )
+        if roi < 0.0001:
+            roi = 0
+
+        actual_roi_rows.append(
+            decimal_formater(
+                str(round_off(roi, 2)),
+                n_decimals=2,
+            )
+        )
+
+    actual_summary_df = pd.DataFrame(
+        [
+            summary_columns,
+            actual_spends_rows,
+            actual_sales_rows,
+            actual_roi_rows,
+        ]
+    ).T
+
+    actual_summary_df.columns = ["Channel", "Spends", target_column, "ROI"]
+
+    actual_summary_df[target_column] = actual_summary_df[target_column].map(
+        lambda x: str(x)[1:]
+    )
+
+    return actual_summary_df
+
+
+# def create_channel_summary(scenario):
+#
+#     # Provided data
+#     data = {
+#         'Channel': ['Paid Search', 'Ga will cid baixo risco', 'Digital tactic others', 'Fb la tier 1', 'Fb la tier 2', 'Paid social others', 'Programmatic', 'Kwai', 'Indicacao', 'Infleux', 'Influencer'],
+#         'Spends': ['$ 11.3K', '$ 155.2K', '$ 50.7K', '$ 125.4K', '$ 125.2K', '$ 105K', '$ 3.3M', '$ 47.5K', '$ 55.9K', '$ 632.3K', '$ 48.3K'],
+#         'Revenue': ['558.0K', '3.5M', '5.2M', '3.1M', '3.1M', '2.1M', '20.8M', '1.6M', '728.4K', '22.9M', '4.8M']
+#     }
+#
+#     # Create DataFrame
+#     df = pd.DataFrame(data)
+#
+#     # Convert currency strings to numeric values
+#     df['Spends'] = df['Spends'].replace({'\$': '', 'K': '*1e3', 'M': '*1e6'}, regex=True).map(pd.eval).astype(int)
+#     df['Revenue'] = df['Revenue'].replace({'\$': '', 'K': '*1e3', 'M': '*1e6'}, regex=True).map(pd.eval).astype(int)
+#
+#     # Calculate ROI
+#     df['ROI'] = ((df['Revenue'] - df['Spends']) / df['Spends'])
+#
+#     # Format columns
+#     format_currency = lambda x: f"${x:,.1f}"
+#     format_roi = lambda x: f"{x:.1f}"
+#
+#     df['Spends'] = ['$ 11.3K', '$ 155.2K', '$ 50.7K', '$ 125.4K', '$ 125.2K', '$ 105K', '$ 3.3M', '$ 47.5K', '$ 55.9K', '$ 632.3K', '$ 48.3K']
+#     df['Revenue'] =  ['$ 536.3K', '$ 3.4M', '$ 5M', '$ 3M', '$ 3M', '$ 2M', '$ 20M', '$ 1.5M', '$ 7.1M', '$ 22M', '$ 4.6M']
+#     df['ROI'] = df['ROI'].apply(format_roi)
+#
+#     return df
+
+
+# @st.cache_resource()
+# def create_contribution_pie(_scenario):
+#     colors_map = {
+#         col: color
+#         for col, color in zip(
+#             st.session_state["channels_list"],
+#             plotly.colors.n_colors(
+#                 plotly.colors.hex_to_rgb("#BE6468"),
+#                 plotly.colors.hex_to_rgb("#E7B8B7"),
+#                 20,
+#             ),
+#         )
+#     }
+#     total_contribution_fig = make_subplots(
+#         rows=1,
+#         cols=2,
+#         subplot_titles=["Spends", "Revenue"],
+#         specs=[[{"type": "pie"}, {"type": "pie"}]],
+#     )
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[
+#                 channel_name_formating(channel_name)
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + ["Non Media"],
+#             values=[
+#                 round(
+#                     _scenario.channels[channel_name].actual_total_spends
+#                     * _scenario.channels[channel_name].conversion_rate,
+#                     1,
+#                 )
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + [0],
+#             marker=dict(
+#                 colors=[
+#                     plotly.colors.label_rgb(colors_map[channel_name])
+#                     for channel_name in st.session_state["channels_list"]
+#                 ]
+#                 + ["#F0F0F0"]
+#             ),
+#             hole=0.3,
+#         ),
+#         row=1,
+#         col=1,
+#     )
+
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[
+#                 channel_name_formating(channel_name)
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + ["Non Media"],
+#             values=[
+#                 _scenario.channels[channel_name].actual_total_sales
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + [_scenario.correction.sum() + _scenario.constant.sum()],
+#             hole=0.3,
+#         ),
+#         row=1,
+#         col=2,
+#     )
+
+#     total_contribution_fig.update_traces(
+#         textposition="inside", texttemplate="%{percent:.1%}"
+#     )
+#     total_contribution_fig.update_layout(
+#         uniformtext_minsize=12,
+#         title="Channel contribution",
+#         uniformtext_mode="hide",
+#     )
+#     return total_contribution_fig
+
+
+# @st.cache_resource()
+# def create_contribution_pie(_scenario):
+#     colors = plotly.colors.qualitative.Plotly  # A diverse color palette
+#     colors_map = {
+#         col: colors[i % len(colors)]
+#         for i, col in enumerate(st.session_state["channels_list"])
+#     }
+
+#     total_contribution_fig = make_subplots(
+#         rows=1,
+#         cols=2,
+#         subplot_titles=["Spends", "Revenue"],
+#         specs=[[{"type": "pie"}, {"type": "pie"}]],
+#     )
+
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[
+#                 channel_name_formating(channel_name)
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + ["Non Media"],
+#             values=[
+#                 round(
+#                     _scenario.channels[channel_name].actual_total_spends
+#                     * _scenario.channels[channel_name].conversion_rate,
+#                     1,
+#                 )
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + [0],
+#             marker=dict(
+#                 colors=[
+#                     colors_map[channel_name]
+#                     for channel_name in st.session_state["channels_list"]
+#                 ]
+#                 + ["#F0F0F0"]
+#             ),
+#             hole=0.3,
+#         ),
+#         row=1,
+#         col=1,
+#     )
+
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[
+#                 channel_name_formating(channel_name)
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + ["Non Media"],
+#             values=[
+#                 _scenario.channels[channel_name].actual_total_sales
+#                 for channel_name in st.session_state["channels_list"]
+#             ]
+#             + [_scenario.correction.sum() + _scenario.constant.sum()],
+#             marker=dict(
+#                 colors=[
+#                     colors_map[channel_name]
+#                     for channel_name in st.session_state["channels_list"]
+#                 ]
+#                 + ["#F0F0F0"]
+#             ),
+#             hole=0.3,
+#         ),
+#         row=1,
+#         col=2,
+#     )
+
+#     total_contribution_fig.update_traces(
+#         textposition="inside", texttemplate="%{percent:.1%}"
+#     )
+#     total_contribution_fig.update_layout(
+#         uniformtext_minsize=12,
+#         title="Channel contribution",
+#         uniformtext_mode="hide",
+#     )
+#     return total_contribution_fig
+
+
+# @st.cache_resource()
+def create_contribution_pie(_scenario, target_col):
+    colors = plotly.colors.qualitative.Plotly  # A diverse color palette
+    colors_map = {
+        col: colors[i % len(colors)]
+        for i, col in enumerate(st.session_state["channels_list"])
+    }
+
+    spends_values = [
+        round(
+            _scenario.channels[channel_name].actual_total_spends
+            * _scenario.channels[channel_name].conversion_rate,
+            1,
+        )
+        for channel_name in st.session_state["channels_list"]
+    ]
+    spends_values.append(0)  # Adding Non Media value
+
+    revenue_values = [
+        _scenario.channels[channel_name].actual_total_sales
+        for channel_name in st.session_state["channels_list"]
+    ]
+    revenue_values.append(
+        _scenario.correction.sum() + _scenario.constant.sum()
+    )  # Adding Non Media value
+
+    total_contribution_fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=["Spend", target_col],
+        specs=[[{"type": "pie"}, {"type": "pie"}]],
+    )
+
+    total_contribution_fig.add_trace(
+        go.Pie(
+            labels=[
+                channel_name_formating(channel_name)
+                for channel_name in st.session_state["channels_list"]
+            ]
+            + ["Non Media"],
+            values=spends_values,
+            marker=dict(
+                colors=[
+                    colors_map[channel_name]
+                    for channel_name in st.session_state["channels_list"]
+                ]
+                + ["#F0F0F0"]
+            ),
+            hole=0.3,
+        ),
+        row=1,
+        col=1,
+    )
+
+    total_contribution_fig.add_trace(
+        go.Pie(
+            labels=[
+                channel_name_formating(channel_name)
+                for channel_name in st.session_state["channels_list"]
+            ]
+            + ["Non Media"],
+            values=revenue_values,
+            marker=dict(
+                colors=[
+                    colors_map[channel_name]
+                    for channel_name in st.session_state["channels_list"]
+                ]
+                + ["#F0F0F0"]
+            ),
+            hole=0.3,
+        ),
+        row=1,
+        col=2,
+    )
+
+    total_contribution_fig.update_traces(
+        textposition="inside", texttemplate="%{percent:.1%}"
+    )
+    total_contribution_fig.update_layout(
+        uniformtext_minsize=12,
+        title="Channel contribution",
+        uniformtext_mode="hide",
+    )
+    return total_contribution_fig
+
+
+# def create_contribuion_stacked_plot(scenario):
+#     weekly_contribution_fig = make_subplots(rows=1, cols=2,subplot_titles=['Spends','Revenue'],specs=[[{"type": "bar"}, {"type": "bar"}]])
+#     raw_df = st.session_state['raw_df']
+#     df = raw_df.sort_values(by='Date')
+#     x = df.Date
+#     weekly_spends_data = []
+#     weekly_sales_data = []
+#     for channel_name in st.session_state['channels_list']:
+#         weekly_spends_data.append((go.Bar(x=x,
+#                                           y=scenario.channels[channel_name].actual_spends * scenario.channels[channel_name].conversion_rate,
+#                                           name=channel_name_formating(channel_name),
+#                                           hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+#                                           legendgroup=channel_name)))
+#         weekly_sales_data.append((go.Bar(x=x,
+#                                          y=scenario.channels[channel_name].actual_sales,
+#                                          name=channel_name_formating(channel_name),
+#                                          hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+#                                          legendgroup=channel_name, showlegend=False)))
+#     for _d in weekly_spends_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=1)
+#     for _d in weekly_sales_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=2)
+#     weekly_contribution_fig.add_trace(go.Bar(x=x,
+#                                          y=scenario.constant + scenario.correction,
+#                                          name='Non Media',
+#                                          hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), row=1, col=2)
+#     weekly_contribution_fig.update_layout(barmode='stack', title='Channel contribuion by week', xaxis_title='Date')
+#     weekly_contribution_fig.update_xaxes(showgrid=False)
+#     weekly_contribution_fig.update_yaxes(showgrid=False)
+#     return weekly_contribution_fig
+
+# @st.cache_resource()
+# def create_channel_spends_sales_plot(channel):
+#     if channel is not None:
+#         x = channel.dates
+#         _spends = channel.actual_spends * channel.conversion_rate
+#         _sales = channel.actual_sales
+#         channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+#         channel_sales_spends_fig.add_trace(go.Bar(x=x, y=_sales,marker_color='#c1f7dc',name='Revenue', hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), secondary_y = False)
+#         channel_sales_spends_fig.add_trace(go.Scatter(x=x, y=_spends,line=dict(color='#005b96'),name='Spends',hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}"), secondary_y = True)
+#         channel_sales_spends_fig.update_layout(xaxis_title='Date',yaxis_title='Revenue',yaxis2_title='Spends ($)',title='Channel spends and Revenue week wise')
+#         channel_sales_spends_fig.update_xaxes(showgrid=False)
+#         channel_sales_spends_fig.update_yaxes(showgrid=False)
+#     else:
+#         raw_df = st.session_state['raw_df']
+#         df = raw_df.sort_values(by='Date')
+#         x = df.Date
+#         scenario = class_from_dict(st.session_state['default_scenario_dict'])
+#         _sales = scenario.constant + scenario.correction
+#         channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+#         channel_sales_spends_fig.add_trace(go.Bar(x=x, y=_sales,marker_color='#c1f7dc',name='Revenue', hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}"), secondary_y = False)
+#         # channel_sales_spends_fig.add_trace(go.Scatter(x=x, y=_spends,line=dict(color='#15C39A'),name='Spends',hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}"), secondary_y = True)
+#         channel_sales_spends_fig.update_layout(xaxis_title='Date',yaxis_title='Revenue',yaxis2_title='Spends ($)',title='Channel spends and Revenue week wise')
+#         channel_sales_spends_fig.update_xaxes(showgrid=False)
+#         channel_sales_spends_fig.update_yaxes(showgrid=False)
+#     return channel_sales_spends_fig
+
+
+# Define a shared color palette
+
+
+# def create_contribution_pie():
+#     color_palette = ['#F3F3F0', '#5E7D7E', '#2FA1FF', '#00EDED', '#00EAE4', '#304550', '#EDEBEB', '#7FBEFD', '#003059', '#A2F3F3', '#E1D6E2', '#B6B6B6']
+#     total_contribution_fig = make_subplots(rows=1, cols=2, subplot_titles=['Spends', 'Revenue'], specs=[[{"type": "pie"}, {"type": "pie"}]])
+#
+#     channels_list = ['Paid Search', 'Ga will cid baixo risco', 'Digital tactic others', 'Fb la tier 1', 'Fb la tier 2', 'Paid social others', 'Programmatic', 'Kwai', 'Indicacao', 'Infleux', 'Influencer', 'Non Media']
+#
+#     # Assign colors from the limited palette to channels
+#     colors_map = {col: color_palette[i % len(color_palette)] for i, col in enumerate(channels_list)}
+#     colors_map['Non Media'] = color_palette[5]  # Assign fixed green color for 'Non Media'
+#
+#     # Hardcoded values for Spends and Revenue
+#     spends_values = [0.5, 3.36, 1.1, 2.7, 2.7, 2.27, 70.6, 1, 1, 13.7, 1, 0]
+#     revenue_values = [1, 4, 5, 3, 3, 2, 50.8, 1.5, 0.7, 13, 0, 16]
+#
+#     # Add trace for Spends pie chart
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[channel_name for channel_name in channels_list],
+#             values=spends_values,
+#             marker=dict(colors=[colors_map[channel_name] for channel_name in channels_list]),
+#             hole=0.3
+#         ),
+#         row=1, col=1
+#     )
+#
+#     # Add trace for Revenue pie chart
+#     total_contribution_fig.add_trace(
+#         go.Pie(
+#             labels=[channel_name for channel_name in channels_list],
+#             values=revenue_values,
+#             marker=dict(colors=[colors_map[channel_name] for channel_name in channels_list]),
+#             hole=0.3
+#         ),
+#         row=1, col=2
+#     )
+#
+#     total_contribution_fig.update_traces(textposition='inside', texttemplate='%{percent:.1%}')
+#     total_contribution_fig.update_layout(uniformtext_minsize=12, title='Channel contribution', uniformtext_mode='hide')
+#     return total_contribution_fig
+
+# @st.cache_resource()
+# def create_contribuion_stacked_plot(_scenario):
+#     weekly_contribution_fig = make_subplots(
+#         rows=1,
+#         cols=2,
+#         subplot_titles=["Spends", "Revenue"],
+#         specs=[[{"type": "bar"}, {"type": "bar"}]],
+#     )
+#     raw_df = st.session_state["raw_df"]
+#     df = raw_df.sort_values(by="Date")
+#     x = df.Date
+#     weekly_spends_data = []
+#     weekly_sales_data = []
+
+#     for i, channel_name in enumerate(st.session_state["channels_list"]):
+#         color = color_palette[i % len(color_palette)]
+
+#         weekly_spends_data.append(
+#             go.Bar(
+#                 x=x,
+#                 y=_scenario.channels[channel_name].actual_spends
+#                 * _scenario.channels[channel_name].conversion_rate,
+#                 name=channel_name_formating(channel_name),
+#                 hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+#                 legendgroup=channel_name,
+#                 marker_color=color,
+#             )
+#         )
+
+#         weekly_sales_data.append(
+#             go.Bar(
+#                 x=x,
+#                 y=_scenario.channels[channel_name].actual_sales,
+#                 name=channel_name_formating(channel_name),
+#                 hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+#                 legendgroup=channel_name,
+#                 showlegend=False,
+#                 marker_color=color,
+#             )
+#         )
+
+#     for _d in weekly_spends_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=1)
+#     for _d in weekly_sales_data:
+#         weekly_contribution_fig.add_trace(_d, row=1, col=2)
+
+#     weekly_contribution_fig.add_trace(
+#         go.Bar(
+#             x=x,
+#             y=_scenario.constant + _scenario.correction,
+#             name="Non Media",
+#             hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+#             marker_color=color_palette[-1],
+#         ),
+#         row=1,
+#         col=2,
+#     )
+
+#     weekly_contribution_fig.update_layout(
+#         barmode="stack",
+#         title="Channel contribution by week",
+#         xaxis_title="Date",
+#     )
+#     weekly_contribution_fig.update_xaxes(showgrid=False)
+#     weekly_contribution_fig.update_yaxes(showgrid=False)
+#     return weekly_contribution_fig
+
+
+# @st.cache_resource()
+def create_contribuion_stacked_plot(_scenario, target_col):
+    color_palette = plotly.colors.qualitative.Plotly  # A diverse color palette
+
+    weekly_contribution_fig = make_subplots(
+        rows=1,
+        cols=2,
+        subplot_titles=["Spend", target_col],
+        specs=[[{"type": "bar"}, {"type": "bar"}]],
+    )
+
+    raw_df = st.session_state["raw_df"]
+    df = raw_df.sort_values(by="Date")
+    x = df.Date
+    weekly_spends_data = []
+    weekly_sales_data = []
+
+    for i, channel_name in enumerate(st.session_state["channels_list"]):
+        color = color_palette[i % len(color_palette)]
+
+        weekly_spends_data.append(
+            go.Bar(
+                x=x,
+                y=_scenario.channels[channel_name].actual_spends
+                * _scenario.channels[channel_name].conversion_rate,
+                name=channel_name_formating(channel_name),
+                hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+                legendgroup=channel_name,
+                marker_color=color,
+            )
+        )
+
+        weekly_sales_data.append(
+            go.Bar(
+                x=x,
+                y=_scenario.channels[channel_name].actual_sales,
+                name=channel_name_formating(channel_name),
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+                legendgroup=channel_name,
+                showlegend=False,
+                marker_color=color,
+            )
+        )
+
+    for _d in weekly_spends_data:
+        weekly_contribution_fig.add_trace(_d, row=1, col=1)
+    for _d in weekly_sales_data:
+        weekly_contribution_fig.add_trace(_d, row=1, col=2)
+
+    weekly_contribution_fig.add_trace(
+        go.Bar(
+            x=x,
+            y=_scenario.constant + _scenario.correction,
+            name="Non Media",
+            hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            marker_color=color_palette[-1],
+        ),
+        row=1,
+        col=2,
+    )
+
+    weekly_contribution_fig.update_layout(
+        barmode="stack",
+        title="Channel contribution by week",
+        xaxis_title="Date",
+    )
+    weekly_contribution_fig.update_xaxes(showgrid=False)
+    weekly_contribution_fig.update_yaxes(showgrid=False)
+    return weekly_contribution_fig
+
+
+def create_channel_spends_sales_plot(channel, target_col):
+    if channel is not None:
+        x = channel.dates
+        _spends = channel.actual_spends * channel.conversion_rate
+        _sales = channel.actual_sales
+        channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+        channel_sales_spends_fig.add_trace(
+            go.Bar(
+                x=x,
+                y=_sales,
+                marker_color=color_palette[
+                    3
+                ],  # You can choose a color from the palette
+                name=target_col,
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            ),
+            secondary_y=False,
+        )
+
+        channel_sales_spends_fig.add_trace(
+            go.Scatter(
+                x=x,
+                y=_spends,
+                line=dict(
+                    color=color_palette[2]
+                ),  # You can choose another color from the palette
+                name="Spends",
+                hovertemplate="Date:%{x}<br>Spend:%{y:$.2s}",
+            ),
+            secondary_y=True,
+        )
+
+        channel_sales_spends_fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title=target_col,
+            yaxis2_title="Spend ($)",
+            title="Weekly Channel Spends and " + target_col,
+        )
+        channel_sales_spends_fig.update_xaxes(showgrid=False)
+        channel_sales_spends_fig.update_yaxes(showgrid=False)
+    else:
+        raw_df = st.session_state["raw_df"]
+        df = raw_df.sort_values(by="Date")
+        x = df.Date
+        scenario = class_from_dict(st.session_state["default_scenario_dict"])
+        _sales = scenario.constant + scenario.correction
+        channel_sales_spends_fig = make_subplots(specs=[[{"secondary_y": True}]])
+        channel_sales_spends_fig.add_trace(
+            go.Bar(
+                x=x,
+                y=_sales,
+                marker_color=color_palette[
+                    0
+                ],  # You can choose a color from the palette
+                name="Revenue",
+                hovertemplate="Date:%{x}<br>Revenue:%{y:$.2s}",
+            ),
+            secondary_y=False,
+        )
+
+        channel_sales_spends_fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title="Revenue",
+            yaxis2_title="Spend ($)",
+            title="Channel spends and Revenue week-wise",
+        )
+        channel_sales_spends_fig.update_xaxes(showgrid=False)
+        channel_sales_spends_fig.update_yaxes(showgrid=False)
+
+    return channel_sales_spends_fig
+
+
+def format_numbers(value, n_decimals=1, include_indicator=True):
+    if include_indicator:
+        return f"{CURRENCY_INDICATOR} {numerize(value,n_decimals)}"
+    else:
+        return f"{numerize(value,n_decimals)}"
+
+
+def decimal_formater(num_string, n_decimals=1):
+    parts = num_string.split(".")
+    if len(parts) == 1:
+        return num_string + "." + "0" * n_decimals
+    else:
+        to_be_padded = n_decimals - len(parts[-1])
+        if to_be_padded > 0:
+            return num_string + "0" * to_be_padded
+        else:
+            return num_string
+
+
+def channel_name_formating(channel_name):
+    name_mod = channel_name.replace("_", " ")
+    if name_mod.lower().endswith(" imp"):
+        name_mod = name_mod.replace("Imp", "Spend")
+    elif name_mod.lower().endswith(" clicks"):
+        name_mod = name_mod.replace("Clicks", "Spend")
+    return name_mod
+
+
+def send_email(email, message):
+    s = smtplib.SMTP("smtp.gmail.com", 587)
+    s.starttls()
+    s.login("[email protected]", "jgydhpfusuremcol")
+    s.sendmail("[email protected]", email, message)
+    s.quit()
+
+
+# if __name__ == "__main__":
+#     initialize_data()