add training script

synthetic_data_generation.ipynb

@@ -0,0 +1,193 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "id": "eaa22304",
+      "metadata": {
+        "id": "eaa22304"
+      },
+      "source": [
+        "### Kernel Density Estimation  \n",
+        "Given n data points, X$\\in R^{n\\times m}$, estimate the probability density function of the data i.e. Prob(x).\n",
+        "\n",
+        "In KDE, the pdf is given by $P(x) = \\frac{1}{nh}\\sum_{i=1}^{N}K(\\frac{X_i-x}{h})$,\n",
+        "where K is the kernel function, h is smoothing bandwidth (small h undersmoothing, large h oversmoothing)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "e139aff4",
+      "metadata": {
+        "id": "e139aff4"
+      },
+      "outputs": [],
+      "source": [
+        "import sklearn\n",
+        "import fnmatch\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import seaborn as sns\n",
+        "import statsmodels.api as sm\n",
+        "import matplotlib.pyplot as plt\n",
+        "from sklearn.decomposition import PCA\n",
+        "from sklearn.neighbors import KernelDensity\n",
+        "from sklearn.model_selection import GridSearchCV"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "14a48fb0",
+      "metadata": {
+        "id": "14a48fb0"
+      },
+      "source": [
+        "#### Load the real data and select samples for a specific race and sex"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "c784a28b",
+      "metadata": {
+        "id": "c784a28b"
+      },
+      "outputs": [],
+      "source": [
+        "df = pd.read_csv('istaging_all.csv') # load istaging data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "0159223c",
+      "metadata": {
+        "id": "0159223c"
+      },
+      "outputs": [],
+      "source": [
+        "# select black females\n",
+        "df = df[((df.Race == 'Black') & (df.Sex == 'F'))].reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "0758d447",
+      "metadata": {
+        "id": "0758d447"
+      },
+      "outputs": [],
+      "source": [
+        "# select baseline data for each subject\n",
+        "df.Date = pd.to_datetime(df.Date)\n",
+        "df_tp1 = df.loc[df.groupby('PTID')['Date'].idxmin()].reset_index(drop=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "9611e735",
+      "metadata": {
+        "scrolled": true,
+        "id": "9611e735"
+      },
+      "outputs": [],
+      "source": [
+        "# split the data to train and test set, train set will be used to learn the probablity distribtuion of the real data\n",
+        "df_train, df_test = sklearn.model_selection.train_test_split(df_tp1, test_size=0.3, random_state=40)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "0800ceda",
+      "metadata": {
+        "id": "0800ceda"
+      },
+      "source": [
+        "#### Fit a KDE model to estimate the joint probability density of Age and ROI volumes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "b37250f8",
+      "metadata": {
+        "id": "b37250f8"
+      },
+      "outputs": [],
+      "source": [
+        "## standardized ROI grid search\n",
+        "# use grid search to select the bandwidth\n",
+        "cols = ['Age']\n",
+        "roi_cols = [] #fill in with the roi column names\n",
+        "cols.extend(fnmatch.filter(df_train.columns, roi_cols)) # select the ROI volumes\n",
+        "data = df_train.loc[:, cols].to_numpy()\n",
+        "data_standard = pd.DataFrame()\n",
+        "# standardize the data\n",
+        "data_standard['Age'] = (df_train['Age'] - df_train.loc[:, 'Age'].mean()) / df_train.loc[:, 'Age'].std()\n",
+        "data_standard[cols[1:]] =  ((df_train.loc[:, cols[1:]] - df_train.loc[:, cols[1:]].mean()) / df_train.loc[:, cols[1:]].std())\n",
+        "data_standard = data_standard.to_numpy()\n",
+        "\n",
+        "# Use a Gaussian kernel\n",
+        "kde = GridSearchCV(KernelDensity(kernel='gaussian'),{'bandwidth': np.linspace(0, 3, 100)}, cv=5)\n",
+        "kde.fit(data_standard)\n",
+        "kde = kde.best_estimator_\n",
+        "print(f'optimal bandwidth of kernel estimated via grid search is {kde.bandwidth_} ')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "32c78445",
+      "metadata": {
+        "id": "32c78445"
+      },
+      "source": [
+        "#### Generate synthetic data using a KDE model for the specified category of race and sex"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "e06523c2",
+      "metadata": {
+        "id": "e06523c2"
+      },
+      "outputs": [],
+      "source": [
+        "# sample 3000 data points\n",
+        "sample = kde.sample(3000, random_state=0)\n",
+        "sample[:, :] = np.multiply(sample[:, :], df_train.loc[:, cols[:]].std().tolist()) + df_train.loc[:, cols[:]].mean().tolist()\n",
+        "cov_list = np.array([[f'Synth_{i+1}', 'F', 'Black'] for i in range(3000)])\n",
+        "synthetic_data = np.concatenate([cov_list, sample], axis=1)\n",
+        "cols=['PTID', 'Sex', 'Race', 'Age']\n",
+        "cols.extend(roi_cols)\n",
+        "df_kde_synth = pd.DataFrame(synthetic_data, columns=cols)"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.8"
+    },
+    "colab": {
+      "provenance": []
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}