"]},"metadata":{},"output_type":"display_data"}],"source":["for col in df.columns[5:]:\n"," print(\"*\" * 10, col, \"*\" * 10)\n"," print(df[col].value_counts())\n"," plot_value_counts(df, col)"]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n","\n","\n","def calc_metrics_for_col(df, col):\n"," y_true = df[\"label\"]\n"," y_pred = df[col]\n","\n"," accuracy = accuracy_score(y_true, y_pred)\n"," precision = precision_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n"," recall = recall_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n"," f1 = f1_score(y_true, y_pred, average=\"weighted\", labels=np.unique(y_pred))\n","\n"," return accuracy, float(precision), float(recall), float(f1)"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stderr","output_type":"stream","text":["/var/folders/7x/56svhln929zdh2xhr3mwqg4r0000gn/T/ipykernel_76471/4138294296.py:9: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n"," perf_df = pd.concat([perf_df, pd.DataFrame([new_model_metrics])], ignore_index=True)\n"]},{"data":{"text/html":["
\n","\n","
\n"," \n","
\n","
\n","
run
\n","
accuracy
\n","
precision
\n","
recall
\n","
f1
\n","
\n"," \n"," \n","
\n","
0
\n","
checkpoint-88
\n","
0.783667
\n","
0.809455
\n","
0.783667
\n","
0.794048
\n","
\n","
\n","
1
\n","
checkpoint-88_lf
\n","
0.775000
\n","
0.814224
\n","
0.775000
\n","
0.790234
\n","
\n"," \n","
\n","
"],"text/plain":[" run accuracy precision recall f1\n","0 checkpoint-88 0.783667 0.809455 0.783667 0.794048\n","1 checkpoint-88_lf 0.775000 0.814224 0.775000 0.790234"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["import pandas as pd\n","\n","perf_df = pd.DataFrame(columns=[\"run\", \"accuracy\", \"precision\", \"recall\", \"f1\"])\n","for i, col in enumerate(df.columns[5:]):\n"," accuracy, precision, recall, f1 = calc_metrics_for_col(df, col)\n"," new_model_metrics = {\"run\": col.split(\"/\")[-1], \"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}\n","\n"," # Convert the dictionary to a DataFrame and concatenate it with the existing DataFrame\n"," perf_df = pd.concat([perf_df, pd.DataFrame([new_model_metrics])], ignore_index=True)\n","\n","perf_df"]},{"cell_type":"code","execution_count":11,"metadata":{},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABE0AAAHACAYAAABXiZaAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABKAklEQVR4nO3de1hVZf738c/mDCqYqYAG4inTQlAZDK2RMZLSx8nG+WVWSpQ2peSBDkqKWB6wUsSMkXJkdH6TSc1o9XvssfqhNKNiJkplIh5SsUZAKyExwdjr+cPLXTuWh43ABn2/rmtdF3ute631vdleLvZn3+teFsMwDAEAAAAAAMCOi7MLAAAAAAAAaIoITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAATbs4u4HJYrVb95z//UatWrWSxWJxdDgAAAADgKmcYhn744Qd16NBBLi6MN7hWNYvQ5D//+Y+CgoKcXQYAAAAA4Bpz9OhR3XDDDc4uA07SLEKTVq1aSTr3j9XX19fJ1QAAAAAArnYVFRUKCgqyfR7FtalZhCbnb8nx9fUlNAEAAAAANBqmiLi2cWMWAAAAAACACUITAAAAAAAAE4QmAAAAaPYyMjIUEhIiLy8v9e/fX9u3b79o+/T0dPXo0UPe3t4KCgrS1KlTdebMGdv2f/3rXxo+fLg6dOggi8Wid95556LHe/zxx2WxWJSenl4PvQEANBWEJgAAAGjWsrOzlZiYqJSUFO3cuVNhYWGKjY1VWVmZafvVq1dr+vTpSklJUWFhoVasWKHs7Gw999xztjaVlZUKCwtTRkbGJc+/bt06bdu2TR06dKi3PgEAmgZCE6AJqe9vyS7nmAcPHtS9996rdu3aydfXV/fdd59KS0vrvW8AADSUtLQ0jR8/XvHx8erVq5cyMzPl4+OjrKws0/Zbt27VwIED9cADDygkJERDhgzR6NGj7a6Rd999t+bOnat77733ouf+5ptv9OSTT+qNN96Qu7t7vfYLAOB8hCZAE9EQ35Jd6piVlZUaMmSILBaLNm7cqC1btqi6ulrDhw+X1WptlH4DAHAlqqurlZ+fr5iYGNs6FxcXxcTEKC8vz3SfAQMGKD8/3xaSfPXVV3r//fc1dOhQh85ttVo1ZswYPfPMM7r55pvr3gkAQJNFaAI0EQ3xLdmljrllyxYdPnxYK1euVGhoqEJDQ7Vq1Srt2LFDGzdubJR+AwBwJU6cOKGamhr5+/vbrff391dJSYnpPg888IBeeOEF3XbbbXJ3d1fXrl0VHR1t98XD5XjxxRfl5uamSZMm1bl+AEDTRmgCNAEN8S3Z5RyzqqpKFotFnp6etjZeXl5ycXHR5s2b672fAAA0Bbm5uZo/f77+/Oc/a+fOnVq7dq3Wr1+vOXPmXPYx8vPztWTJEq1cuVIWi6UBqwUAOBOhCdAENMS3ZJdzzFtvvVUtWrTQtGnTdPr0aVVWVurpp59WTU2Njh071gA9BQCgfrVt21aurq615uMqLS1VQECA6T7JyckaM2aMxo0bp9DQUN17772aP3++UlNTL/v21H//+98qKytTcHCw3Nzc5ObmpiNHjuipp55SSEjIlXYLANBEEJoAzVR9fEvWrl07vf322/qf//kftWzZUn5+fjp58qT69u0rFxf+ewAANH0eHh7q16+fcnJybOusVqtycnIUFRVlus/p06drXedcXV0lSYZhXNZ5x4wZo88//1wFBQW2pUOHDnrmmWf0wQcf1LE3AICmxs3ZBQC48m/JJCk0NFSVlZV67LHHNGPGjMs+5pAhQ3Tw4EGdOHFCbm5uat26tQICAtSlS5d67iUAAA0jMTFRcXFxioiIUGRkpNLT01VZWan4+HhJ0tixY9WxY0elpqZKkoYPH660tDT16dNH/fv314EDB5ScnKzhw4fbwpNTp07pwIEDtnMcOnRIBQUFatOmjYKDg3X99dfr+uuvt6vD3d1dAQEB6tGjRyP1HADQ0PgqGWgCGuJbMkeP2bZtW7Vu3VobN25UWVmZfv/739dH1655jjxGOjo6WhaLpdYybNgwW5vS0lI9/PDD6tChg3x8fHTXXXdp//79psczDEN33323LBaL3nnnnfruGgA0GaNGjdLChQs1a9YshYeHq6CgQBs2bLDdolpcXGx32+nMmTP11FNPaebMmerVq5ceffRRxcbG6rXXXrO12bFjh/r06aM+ffpIOhfM9OnTR7NmzWrczgEAnIqRJkAT0RDfkl3qmJL017/+VT179lS7du2Ul5enyZMna+rUqXxLVg/OP/I5MzNT/fv3V3p6umJjY1VUVKT27dvXar927VpVV1fbXn/77bcKCwvTf/3Xf0k6F4KMGDFC7u7uevfdd+Xr66u0tDTFxMRoz549atGihd3x0tPTmZwQwDUjISFBCQkJpttyc3PtXru5uSklJUUpKSkXPF50dPRl36pz3uHDhx1qDwBoBoxmoLy83JBklJeXO7sUoEEtXbrUCA4ONjw8PIzIyEhj27Zttm2DBg0y4uLibK/Pnj1rzJ492+jatavh5eVlBAUFGRMmTDC+//77yz6mYRjGtGnTDH9/f8Pd3d3o3r27sWjRIsNqtTZkN68ZkZGRxsSJE22va2pqjA4dOhipqamXtf/ixYuNVq1aGadOnTIMwzCKiooMScbu3bvtjtmuXTtj+fLldvvu2rXL6Nixo3Hs2DFDkrFu3bor7xAAAI3o1VdfNTp16mR4enoakZGRxieffHLBtoMGDTIk1VqGDh1qa1NSUmLExcUZgYGBhre3txEbG2vs27fP7jg//vijMWHCBKNNmzZGixYtjD/84Q9GSUlJg/URTRufQ2EYhmExDAcjdCeoqKiQn5+fysvL5evr6+xyAOCSqqur5ePjo3/84x8aMWKEbX1cXJxOnjypd99995LHCA0NVVRUlF5//XVJ0hdffKHevXvrwIED6tq1q61dUFCQ7rjjDq1cuVLSuVu3IiIilJqaqnvuuUcWi0Xr1q2zqwMAgKYsOztbY8eOtRut+fbbb19wtOZ3331nOlrzL3/5ix5++GEZhqEBAwbI3d1dixYtso3W3LBhg91ozSeeeELr16/XypUr5efnp4SEBLm4uGjLli2N1nc0HXwOhcTtOQDQIC72yOe9e/decv/t27dr9+7dWrFihW3dTTfdpODgYCUlJem1115TixYttHjxYn399dd29+pPnTpVAwYM0D333FN/HQKAxjbbz9kVNI7Z5c6uoElKS0vT+PHjbbcUZ2Zmav369crKytL06dNrtW/Tpo3d6zVr1sjHx8d2i+v+/fu1bds27d69WzfffLMkadmyZQoICNCbb76pcePGqby8XCtWrNDq1as1ePBgST/fxrxt2zbdeuutDdllAE0UE8GiWajvyTRPnTqlhIQE3XDDDfL29lavXr2UmZlpejyDyTTrHe/npa1YsUKhoaGKjIy0rXN3d9fatWu1b98+tWnTRj4+Ptq0aZPuvvtu26TA7733njZu3Kj09HQnVQ4AwJWprq5Wfn6+YmJibOtcXFwUExOjvLy8yzrGihUrdP/999tGkFRVVUmSvLy87I7p6empzZs3S5Ly8/N19uxZu/Oe/8Lics8L4OpTp5EmGRkZevnll1VSUqKwsDAtXbrU7g/7X0tPT9eyZctUXFystm3b6o9//KNSU1Pt/tMCLqS+J9OUzk2QunHjRv39739XSEiIPvzwQ02YMEEdOnSo9dSYK5pMk2/JamnW76cD6vIY6fMqKyu1Zs0avfDCC7W29evXTwUFBSovL1d1dbXatWun/v37KyIiQpK0ceNGHTx4UK1bt7bbb+TIkbr99ttrTYYIAEBT46zRmiUlJfLw8Kh1DfX391dJScmVdwxAs+TwSJPzH3hSUlK0c+dOhYWFKTY2VmVlZabtV69erenTpyslJUWFhYVasWKFsrOz9dxzz11x8bg2/HJ45vkRBD4+PsrKyjJt36ZNGwUEBNiWjz76yG54piRt3bpVcXFxio6OVkhIiB577DGFhYXVGvFQUFCgRYsWXfBccNy18n7W5THS57399tuqqqrSQw89dME2fn5+ateunfbv368dO3bYbsWZPn26Pv/8cxUUFNgWSVq8eLH++te/XnnHAABo4uo6WhMAzDj8P4SjH3i2bt2qgQMH6oEHHlBISIiGDBmi0aNHX3Q4PnBeQwzPlKQBAwbovffe0zfffCPDMLRp0ybt27dPQ4YMsbU5ffq0HnjgAWVkZFxyZAAuz7X2fiYmJmr58uVatWqVCgsL9cQTT9R6jHRSUlKt/VasWKERI0bo+uuvr7Xt7bffVm5urr766iu9++67uvPOOzVixAhbXwMCAnTLLbfYLZIUHByszp07N2BvAQCoH/UxWvPRRx+tte38aM2TJ0/q2LFj2rBhg7799lt16dJF0rlraHV1tU6ePOnweQFcvRwKTerygWfAgAHKz8+3hSRfffWV3n//fQ0dOvSC56mqqlJFRYXdgmvTxYZnXs4wyfPDM8eNG2e3funSperVq5duuOEGeXh46K677lJGRoZ++9vf2towmWb9u9bez1GjRmnhwoWaNWuWwsPDVVBQoA0bNtj6X1xcbDeBqyQVFRVp8+bNpn/sSdKxY8c0ZswY3XTTTZo0aZLGjBmjN998s8H7AqD5qu95pMy2WywWvfzyy7Y2ISEhtbYvWLCgQft5rbgW3k9njdbs16+f3N3d7c5bVFSk4uLiS54XwNXLoTlN6nJ/4QMPPKATJ07otttuk2EY+umnn/T4449f9Pac1NRUPf/8846UBpgyG54pnfuQvW3bNr333nvq1KmT/vWvf2nixInq0KGDYmJibJNp7tq1y0mVw0xzfD8TEhKUkJBgus1sfpEePXroYk+CnzRpkiZNmuRQDc3gyfIAGkhDzCP167D3//2//6dHH31UI0eOtFv/wgsvaPz48bbXrVq1qq9uXbOupfczMTFRcXFxioiIUGRkpNLT02uN1uzYsaNSU1Pt9rvUaM127dopODhYX3zxhSZPnmw3WtPPz0+PPvqoEhMT1aZNG/n6+urJJ59UVFQUT84BrmEN/sjh3NxczZ8/X3/+85/Vv39/HThwQJMnT9acOXOUnJxsuk9SUpISExNtrysqKhQUFNTQpaIJaojJNH/88Uc999xzWrdune2blt69e6ugoEALFy5UTEwMk2k2EN5PAGhc9f3YVkm1/r9+99139bvf/c52i8N5rVq14paGenYtvZ+jRo3S8ePHNWvWLJWUlCg8PLzWaM1fz0VyfrTmhx9+aHrMY8eOKTExUaWlpQoMDNTYsWNrfR5ZvHixXFxcNHLkSFVVVSk2NlZ//vOfG6aTAJoFh27PqcsHnuTkZI0ZM0bjxo1TaGio7r33Xs2fP1+pqamyWq2m+3h6esrX19duwbWpIYZnnj17VmfPnq11oXV1dbX9m2QyzYbB+wkAjaeh5pH6pdLSUq1fv970lsIFCxbo+uuvV58+ffTyyy/rp59+qltHIOnafD8TEhJ05MgRVVVV6ZNPPlH//v1t23Jzc7Vy5Uq79udHa955552mx5s0aZKOHj2q6upqHTlyRHPmzJGHh4ddGy8vL2VkZOi7775TZWWl1q5dS/gHXOMcGmnyyw88I0aMkPTzB54LDT8/ffq06YcZiSHjuDz1PTzT19dXgwYN0jPPPCNvb2916tRJH3/8sf72t78pLS1NkmxPavk1JtO8cs35/Sy8qaej3W2Weu4tdHYJAOpBQzy29ddWrVqlVq1a6Q9/+IPd+kmTJqlv375q06aNtm7dqqSkJB07dsz2/zIcx/sJAM7h8O05jn7gGT58uNLS0tSnTx/b7TnJyckaPny4LTwBLqYhhmeuWbNGSUlJevDBB/Xdd9+pU6dOmjdvnh5//PEG78+1jvcTAJqHC80j9UtZWVl68MEH5eXlZbf+l7dZ9+7dWx4eHvrTn/6k1NRUeXp6NljNuDDeTwCoG4dDE0c/8MycOVMWi0UzZ87UN998o3bt2mn48OGaN29e/fUCV736nkwzICDA4dsyGBlVf3g/AaDhNcQ8Ur/073//W0VFRcrOzr5kLf3799dPP/2kw4cPq0ePHpfXAdhp7u8nozUBNFcOzWlyniP3F7q5uSklJUUHDhzQjz/+qOLiYmVkZNSakBEAAAD1p6Ef27pixQr169dPYWFhl6yloKBALi4upk94weXh/QQA52jwp+cAAADAORrisa3SuScbvv3221q0aFGtbXl5efrkk0/0u9/9Tq1atVJeXp6mTp2qhx56SNddd139d/IawvsJAI2P0AROETJ9vbNLaBSHvS7d5moQuirU2SU0irecXQAAOKgh5pGSzs0lZRiGRo8eXWubp6en1qxZo9mzZ6uqqkqdO3fW1KlT7ebFQN3wfgJA47MYzeDG/oqKCvn5+am8vJzHD18lrp3Q5AFnl9AoQjsHO7uERvFW6rXxuEzuxwbQJMz2c3YFjWN2ubMraBTMaYLmiM+hkOo4pwkAAAAAAMDVjtAEAAAAAADABHOaAAAANCPXzi2uzq6gcTAvGAA0bYw0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgIk6hSYZGRkKCQmRl5eX+vfvr+3bt1+wbXR0tCwWS61l2LBhdS4aAAAAAACgoTkcmmRnZysxMVEpKSnauXOnwsLCFBsbq7KyMtP2a9eu1bFjx2zL7t275erqqv/6r/+64uIBAAAAAAAaisOhSVpamsaPH6/4+Hj16tVLmZmZ8vHxUVZWlmn7Nm3aKCAgwLZ89NFH8vHxITQBAAAAAABNmkOhSXV1tfLz8xUTE/PzAVxcFBMTo7y8vMs6xooVK3T//ferRYsWF2xTVVWliooKuwUAAAAAAKAxORSanDhxQjU1NfL397db7+/vr5KSkkvuv337du3evVvjxo27aLvU1FT5+fnZlqCgIEfKBAAAAAAAuGKN+vScFStWKDQ0VJGRkRdtl5SUpPLyctty9OjRRqoQAAAAAADgHDdHGrdt21aurq4qLS21W19aWqqAgICL7ltZWak1a9bohRdeuOR5PD095enp6UhpAAAAAAAA9cqhkSYeHh7q16+fcnJybOusVqtycnIUFRV10X3ffvttVVVV6aGHHqpbpQAAAAAAAI3IoZEmkpSYmKi4uDhFREQoMjJS6enpqqysVHx8vCRp7Nix6tixo1JTU+32W7FihUaMGKHrr7++fioHAAAAAABoQA6HJqNGjdLx48c1a9YslZSUKDw8XBs2bLBNDltcXCwXF/sBLEVFRdq8ebM+/PDD+qkaAAAAAACggTkcmkhSQkKCEhISTLfl5ubWWtejRw8ZhlGXUwEAAAAAADhFoz49BwAAAAAAoLkgNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwUafQJCMjQyEhIfLy8lL//v21ffv2i7Y/efKkJk6cqMDAQHl6eurGG2/U+++/X6eCAQAAAAAAGoOboztkZ2crMTFRmZmZ6t+/v9LT0xUbG6uioiK1b9++Vvvq6mrdeeedat++vf7xj3+oY8eOOnLkiFq3bl0f9QMAAAAAADQIh0OTtLQ0jR8/XvHx8ZKkzMxMrV+/XllZWZo+fXqt9llZWfruu++0detWubu7S5JCQkKurGoAAAAAAIAG5tDtOdXV1crPz1dMTMzPB3BxUUxMjPLy8kz3ee+99xQVFaWJEyfK399ft9xyi+bPn6+ampoLnqeqqkoVFRV2CwAAAAAAQGNyKDQ5ceKEampq5O/vb7fe399fJSUlpvt89dVX+sc//qGamhq9//77Sk5O1qJFizR37twLnic1NVV+fn62JSgoyJEyAQAAAAAArliDPz3HarWqffv2ev3119WvXz+NGjVKM2bMUGZm5gX3SUpKUnl5uW05evRoQ5cJAAAAAABgx6E5Tdq2bStXV1eVlpbarS8tLVVAQIDpPoGBgXJ3d5erq6ttXc+ePVVSUqLq6mp5eHjU2sfT01Oenp6OlAYAAAAAAFCvHBpp4uHhoX79+iknJ8e2zmq1KicnR1FRUab7DBw4UAcOHJDVarWt27dvnwIDA00DEwAAAAAAgKbA4dtzEhMTtXz5cq1atUqFhYV64oknVFlZaXuaztixY5WUlGRr/8QTT+i7777T5MmTtW/fPq1fv17z58/XxIkT668XAAAAAAAA9czhRw6PGjVKx48f16xZs1RSUqLw8HBt2LDBNjlscXGxXFx+zmKCgoL0wQcfaOrUqerdu7c6duyoyZMna9q0afXXCwAAAAAAgHrmcGgiSQkJCUpISDDdlpubW2tdVFSUtm3bVpdTAQAAAAAAOEWDPz0HAAAAAACgOSI0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMOHm7AIAAAAAAGiOrFarqqurnV0GHOTu7i5XV9fLaktoAgAAAACAg6qrq3Xo0CFZrVZnl4I6aN26tQICAmSxWC7ajtAEAAAAAAAHGIahY8eOydXVVUFBQXJxYeaL5sIwDJ0+fVplZWWSpMDAwIu2JzQBAAAAAMABP/30k06fPq0OHTrIx8fH2eXAQd7e3pKksrIytW/f/qK36hCHAQAAAADggJqaGkmSh4eHkytBXZ0Pu86ePXvRdoQmAAAAAADUwaXmw0DTdbnvHaEJAAAAAACACUITAAAAAAAAE3WaCDYjI0Mvv/yySkpKFBYWpqVLlyoyMtK07cqVKxUfH2+3ztPTU2fOnKnLqQEAAAAAaJJCpq9v1PMdXjCsUc93LXJ4pEl2drYSExOVkpKinTt3KiwsTLGxsbbH9Zjx9fXVsWPHbMuRI0euqGgAAAAAAND8XWoiVmdzODRJS0vT+PHjFR8fr169eikzM1M+Pj7Kysq64D4Wi0UBAQG2xd/f/4qKBgAAAAAAjtuwYYNuu+02tW7dWtdff73+z//5Pzp48KBt+9dff63Ro0erTZs2atGihSIiIvTJJ5/Ytv/P//yPfvOb38jLy0tt27bVvffea9tmsVj0zjvv2J2vdevWWrlypSTp8OHDslgsys7O1qBBg+Tl5aU33nhD3377rUaPHq2OHTvKx8dHoaGhevPNN+2OY7Va9dJLL6lbt27y9PRUcHCw5s2bJ0kaPHiwEhIS7NofP35cHh4eysnJuaLfl0OhSXV1tfLz8xUTE/PzAVxcFBMTo7y8vAvud+rUKXXq1ElBQUG655579OWXX170PFVVVaqoqLBbAAAAAADAlamsrFRiYqJ27NihnJwcubi46N5775XVatWpU6c0aNAgffPNN3rvvff02Wef6dlnn5XVapUkrV+/Xvfee6+GDh2qXbt2KScn54JTdVzM9OnTNXnyZBUWFio2NlZnzpxRv379tH79eu3evVuPPfaYxowZo+3bt9v2SUpK0oIFC5ScnKw9e/Zo9erVtgEZ48aN0+rVq1VVVWVr//e//10dO3bU4MGDr+j35dCcJidOnFBNTU2tkSL+/v7au3ev6T49evRQVlaWevfurfLyci1cuFADBgzQl19+qRtuuMF0n9TUVD3//POOlAYAAAAAAC5h5MiRdq+zsrLUrl077dmzR1u3btXx48f16aefqk2bNpKkbt262drOmzdP999/v93n9bCwMIdrmDJliv7whz/YrXv66adtPz/55JP64IMP9NZbbykyMlI//PCDlixZoldffVVxcXGSpK5du+q2226TJP3hD39QQkKC3n33Xd13332Szs2v+vDDD1/xY6Eb/Ok5UVFRGjt2rMLDwzVo0CCtXbtW7dq102uvvXbBfZKSklReXm5bjh492tBlAgAAAABw1du/f79Gjx6tLl26yNfXVyEhIZKk4uJiFRQUqE+fPrbA5NcKCgp0xx13XHENERERdq9ramo0Z84chYaGqk2bNmrZsqU++OADFRcXS5IKCwtVVVV1wXN7eXlpzJgxtmlDdu7cqd27d+vhhx++4lodGmnStm1bubq6qrS01G59aWmpAgICLusY7u7u6tOnjw4cOHDBNp6envL09HSkNAAAAAAAcAnDhw9Xp06dtHz5cnXo0EFWq1W33HKLqqur5e3tfdF9L7XdYrHIMAy7dWYTvbZo0cLu9csvv6wlS5YoPT1doaGhatGihaZMmaLq6urLOq907had8PBwff311/rrX/+qwYMHq1OnTpfc71IcGmni4eGhfv362U2kYrValZOTo6ioqMs6Rk1Njb744gsFBgY6VikAAAAAAKizb7/9VkVFRZo5c6buuOMO9ezZU99//71te+/evVVQUKDvvvvOdP/evXtfdGLVdu3a6dixY7bX+/fv1+nTpy9Z15YtW3TPPffooYceUlhYmLp06aJ9+/bZtnfv3l3e3t4XPXdoaKgiIiK0fPlyrV69Wo888sglz3s5HL49JzExUcuXL9eqVatUWFioJ554QpWVlYqPj5ckjR07VklJSbb2L7zwgj788EN99dVX2rlzpx566CEdOXJE48aNq5cOAAAAAACAS7vuuut0/fXX6/XXX9eBAwe0ceNGJSYm2raPHj1aAQEBGjFihLZs2aKvvvpK//znP20PfklJSdGbb76plJQUFRYW6osvvtCLL75o23/w4MF69dVXtWvXLu3YsUOPP/643N3dL1lX9+7d9dFHH2nr1q0qLCzUn/70J7s7XLy8vDRt2jQ9++yz+tvf/qaDBw9q27ZtWrFihd1xxo0bpwULFsgwDLun+lwJh0OTUaNGaeHChZo1a5bCw8NVUFCgDRs22CaHLS4utkuWvv/+e40fP149e/bU0KFDVVFRoa1bt6pXr1710gEAAAAAAHBpLi4uWrNmjfLz83XLLbdo6tSpevnll23bPTw89OGHH6p9+/YaOnSoQkNDtWDBArm6ukqSoqOj9fbbb+u9995TeHi4Bg8ebPeEm0WLFikoKEi33367HnjgAT399NPy8fG5ZF0zZ85U3759FRsbq+joaFtw80vJycl66qmnNGvWLPXs2VOjRo1SWVmZXZvRo0fLzc1No0ePlpeX1xX8pn5mMX59w1ETVFFRIT8/P5WXl8vX19fZ5aAehExf7+wSGsVhrwecXUKjCO0c7OwSGsVbqT85u4RG0XNvobNLAHARXEOvLlxDry5cQ68uF/sceubMGR06dEidO3eutw/nuHKHDx9W165d9emnn6pv374XbXu576FDE8ECAAAAAAA0JWfPntW3336rmTNn6tZbb71kYOKIBn/kMAAAAAAAQEPZsmWLAgMD9emnnyozM7Nej81IEwAAAAAA0GxFR0fXetRxfWGkCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAABpEbm6uLBaLTp48Wa9tG4ubswsAAAAAAOCqMNuvkc9X3rjnq4MBAwbo2LFj8vO79O/GkbaNhZEmAAAAAACglurq6is+hoeHhwICAmSxWOq1bWMhNAEAAAAA4BoQHR2thIQEJSQkyM/PT23btlVycrIMw5AkhYSEaM6cORo7dqx8fX312GOPSZI2b96s22+/Xd7e3goKCtKkSZNUWVlpO25VVZWmTZumoKAgeXp6qlu3blqxYoWk2rfcHDlyRMOHD9d1112nFi1a6Oabb9b7779v2laS/vnPf+rmm2+Wp6enQkJCtGjRIrs+hYSEaP78+XrkkUfUqlUrBQcH6/XXX6+33xmhCQAAAAAA14hVq1bJzc1N27dv15IlS5SWlqa//OUvtu0LFy5UWFiYdu3apeTkZB08eFB33XWXRo4cqc8//1zZ2dnavHmzEhISbPuMHTtWb775pl555RUVFhbqtddeU8uWLU3PP3HiRFVVVelf//qXvvjiC7344osXbJufn6/77rtP999/v7744gvNnj1bycnJWrlypV27RYsWKSIiQrt27dKECRP0xBNPqKio6Mp/WWJOEwAAAAAArhlBQUFavHixLBaLevTooS+++EKLFy/W+PHjJUmDBw/WU089ZWs/btw4Pfjgg5oyZYokqXv37nrllVc0aNAgLVu2TMXFxXrrrbf00UcfKSYmRpLUpUuXC56/uLhYI0eOVGho6CXbpqWl6Y477lBycrIk6cYbb9SePXv08ssv6+GHH7a1Gzp0qCZMmCBJmjZtmhYvXqxNmzapR48ejv+CfoWRJgAAAAAAXCNuvfVWuzlDoqKitH//ftXU1EiSIiIi7Np/9tlnWrlypVq2bGlbYmNjZbVadejQIRUUFMjV1VWDBg26rPNPmjRJc+fO1cCBA5WSkqLPP//8gm0LCws1cOBAu3UDBw60q1eSevfubfvZYrEoICBAZWVll1XPpRCaAAAAAAAASVKLFi3sXp86dUp/+tOfVFBQYFs+++wz7d+/X127dpW3t7dDxx83bpy++uorjRkzRl988YUiIiK0dOnSK6rZ3d3d7rXFYpHVar2iY55HaAIAAAAAwDXik08+sXu9bds2de/eXa6urqbt+/btqz179qhbt261Fg8PD4WGhspqterjjz++7BqCgoL0+OOPa+3atXrqqae0fPly03Y9e/bUli1b7NZt2bJFN9544wXrrW+EJgAAAAAAXCOKi4uVmJiooqIivfnmm1q6dKkmT558wfbTpk3T1q1blZCQoIKCAu3fv1/vvvuubSLYkJAQxcXF6ZFHHtE777yjQ4cOKTc3V2+99Zbp8aZMmaIPPvhAhw4d0s6dO7Vp0yb17NnTtO1TTz2lnJwczZkzR/v27dOqVav06quv6umnn77yX8RlYiJYAAAAAACuEWPHjtWPP/6oyMhIubq6avLkybZHC5vp3bu3Pv74Y82YMUO33367DMNQ165dNWrUKFubZcuW6bnnntOECRP07bffKjg4WM8995zp8WpqajRx4kR9/fXX8vX11V133aXFixebtu3bt6/eeustzZo1S3PmzFFgYKBeeOEFu0lgG5rFOP9A5iasoqJCfn5+Ki8vl6+vr7PLQT0Imb7e2SU0isNeDzi7hEYR2jnY2SU0irdSf3J2CY2i595CZ5cA4CK4hl5duIZeXbiGXl0u9jn0zJkzOnTokDp37iwvLy8nVei46OhohYeHKz093dmlON3lvofcngMAAAAAAGCC0AQAAAAAAMAEc5oAAAAAAHANyM3NdXYJzQ4jTQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAANAgZs+erfDwcNvrhx9+WCNGjHBaPY5yc3YBAAAAAABcDUJXhTbq+b6I+6JRz3ctYqQJAAAAAADXoOrqameX0OQRmgAAAAAAcA2Ijo5WQkKCpkyZorZt2yo2Nla7d+/W3XffrZYtW8rf319jxozRiRMnbPtYrVa99NJL6tatmzw9PRUcHKx58+bZtk+bNk033nijfHx81KVLFyUnJ+vs2bPO6F6DqFNokpGRoZCQEHl5eal///7avn37Ze23Zs0aWSyWZnX/EgAAAAAAV4tVq1bJw8NDW7Zs0YIFCzR48GD16dNHO3bs0IYNG1RaWqr77rvP1j4pKUkLFixQcnKy9uzZo9WrV8vf39+2vVWrVlq5cqX27NmjJUuWaPny5Vq8eLEzutYgHJ7TJDs7W4mJicrMzFT//v2Vnp6u2NhYFRUVqX379hfc7/Dhw3r66ad1++23X1HBAAAAAACgbrp3766XXnpJkjR37lz16dNH8+fPt23PyspSUFCQ9u3bp8DAQC1ZskSvvvqq4uLiJEldu3bVbbfdZms/c+ZM288hISF6+umntWbNGj377LON1KOG5fBIk7S0NI0fP17x8fHq1auXMjMz5ePjo6ysrAvuU1NTowcffFDPP/+8unTpckUFAwAAAACAuunXr5/t588++0ybNm1Sy5YtbctNN90kSTp48KAKCwtVVVWlO+6444LHy87O1sCBAxUQEKCWLVtq5syZKi4ubvB+NBaHQpPq6mrl5+crJibm5wO4uCgmJkZ5eXkX3O+FF15Q+/bt9eijj17WeaqqqlRRUWG3AAAAAACAK9OiRQvbz6dOndLw4cNVUFBgt+zfv1+//e1v5e3tfdFj5eXl6cEHH9TQoUP1f//v/9WuXbs0Y8aMq2qCWYduzzlx4oRqamrs7l+SJH9/f+3du9d0n82bN2vFihUqKCi47POkpqbq+eefd6Q0AAAAAADggL59++qf//ynQkJC5OZWOx7o3r27vL29lZOTo3HjxtXavnXrVnXq1EkzZsywrTty5EiD1tzYGvTpOT/88IPGjBmj5cuXq23btpe9X1JSksrLy23L0aNHG7BKAAAAAACuPRMnTtR3332n0aNH69NPP9XBgwf1wQcfKD4+XjU1NfLy8tK0adP07LPP6m9/+5sOHjyobdu2acWKFZLOhSrFxcVas2aNDh48qFdeeUXr1q1zcq/ql0MjTdq2bStXV1eVlpbarS8tLVVAQECt9gcPHtThw4c1fPhw2zqr1XruxG5uKioqUteuXWvt5+npKU9PT0dKAwAAAAAADujQoYO2bNmiadOmaciQIaqqqlKnTp101113ycXl3BiL5ORkubm5adasWfrPf/6jwMBAPf7445Kk3//+95o6daoSEhJUVVWlYcOGKTk5WbNnz3Zir+qXxTAMw5Ed+vfvr8jISC1dulTSuRAkODhYCQkJmj59ul3bM2fO6MCBA3brZs6cqR9++EFLlizRjTfeKA8Pj0ues6KiQn5+fiovL5evr68j5aKJCpm+3tklNIrDXg84u4RGEdo52NklNIq3Un9ydgmNoufeQmeXAOAiuIZeXbiGXl24hl5dLvY59MyZMzp06JA6d+4sLy8vJ1WIK3G576HDjxxOTExUXFycIiIiFBkZqfT0dFVWVio+Pl6SNHbsWHXs2FGpqany8vLSLbfcYrd/69atJanWegAAAAAAgKbE4dBk1KhROn78uGbNmqWSkhKFh4drw4YNtslhi4uLbcN4AAAAAAAAmiuHQxNJSkhIUEJCgum23Nzci+67cuXKupwSAAAAAACgUTEkBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAA4BpgGIYee+wxtWnTRhaLRQUFBc4uqclzc3YBAAAAAABcDQpv6tmo5+u5t9Ch9hs2bNDKlSuVm5urLl26aN++fRo+fLjy8/N17NgxrVu3TiNGjGiYYpspRpoAAAAAAHANOHjwoAIDAzVgwAAFBASosrJSYWFhysjIcHZpTRYjTQAAAAAAuMo9/PDDWrVqlSTJYrGoU6dOOnz4sO6++24nV9a0EZoAAAAAAHCVW7Jkibp27arXX39dn376qVxdXZ1dUrNAaAIAAAAAwFXOz89PrVq1kqurqwICApxdTrPBnCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYYE4TAAAAAACuQadOndKBAwdsrw8dOqSCggK1adNGwcHBTqys6SA0AQAAAACgHvTcW+jsEhyyY8cO/e53v7O9TkxMlCTFxcVp5cqVTqqqaSE0AQAAAADgGjBlyhRNmTLF9jo6OlqGYTivoGaAOU0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAIA6YBLV5uty3ztCEwAAAAAAHODq6ipJqq6udnIlqKvTp09Lktzd3S/ajkcOAwAAAADgADc3N/n4+Oj48eNyd3eXiwvjEZoLwzB0+vRplZWVqXXr1rYA7EIITQAAAAAAcIDFYlFgYKAOHTqkI0eOOLsc1EHr1q0VEBBwyXaEJgAAAAAAOMjDw0Pdu3fnFp1myN3d/ZIjTM4jNAEAAAAAoA5cXFzk5eXl7DLQgLjxCgAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJioU2iSkZGhkJAQeXl5qX///tq+ffsF265du1YRERFq3bq1WrRoofDwcP33f/93nQsGAAAAAABoDA6HJtnZ2UpMTFRKSop27typsLAwxcbGqqyszLR9mzZtNGPGDOXl5enzzz9XfHy84uPj9cEHH1xx8QAAAAAAAA3F4dAkLS1N48ePV3x8vHr16qXMzEz5+PgoKyvLtH10dLTuvfde9ezZU127dtXkyZPVu3dvbd68+YqLBwAAAAAAaCgOhSbV1dXKz89XTEzMzwdwcVFMTIzy8vIuub9hGMrJyVFRUZF++9vfXrBdVVWVKioq7BYAAAAAAIDG5FBocuLECdXU1Mjf399uvb+/v0pKSi64X3l5uVq2bCkPDw8NGzZMS5cu1Z133nnB9qmpqfLz87MtQUFBjpQJAAAAAABwxRrl6TmtWrVSQUGBPv30U82bN0+JiYnKzc29YPukpCSVl5fblqNHjzZGmQAAAAAAADZujjRu27atXF1dVVpaare+tLRUAQEBF9zPxcVF3bp1kySFh4ersLBQqampio6ONm3v6ekpT09PR0oDAAAAAACoVw6NNPHw8FC/fv2Uk5NjW2e1WpWTk6OoqKjLPo7ValVVVZUjpwYAAAAAAGhUDo00kaTExETFxcUpIiJCkZGRSk9PV2VlpeLj4yVJY8eOVceOHZWamirp3PwkERER6tq1q6qqqvT+++/rv//7v7Vs2bL67QkAAAAAAEA9cjg0GTVqlI4fP65Zs2appKRE4eHh2rBhg21y2OLiYrm4/DyApbKyUhMmTNDXX38tb29v3XTTTfr73/+uUaNG1V8vAAAAAAAA6pnDoYkkJSQkKCEhwXTbryd4nTt3rubOnVuX0wAAAAAAADhNozw9BwAAAAAAoLkhNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwUafQJCMjQyEhIfLy8lL//v21ffv2C7Zdvny5br/9dl133XW67rrrFBMTc9H2AAAAAAAATYHDoUl2drYSExOVkpKinTt3KiwsTLGxsSorKzNtn5ubq9GjR2vTpk3Ky8tTUFCQhgwZom+++eaKiwcAAAAAAGgoDocmaWlpGj9+vOLj49WrVy9lZmbKx8dHWVlZpu3feOMNTZgwQeHh4brpppv0l7/8RVarVTk5OVdcPAAAAAAAQENxKDSprq5Wfn6+YmJifj6Ai4tiYmKUl5d3Wcc4ffq0zp49qzZt2jhWKQAAAAAAQCNyc6TxiRMnVFNTI39/f7v1/v7+2rt372UdY9q0aerQoYNd8PJrVVVVqqqqsr2uqKhwpEwAAAAAAIAr1qhPz1mwYIHWrFmjdevWycvL64LtUlNT5efnZ1uCgoIasUoAAAAAAAAHQ5O2bdvK1dVVpaWldutLS0sVEBBw0X0XLlyoBQsW6MMPP1Tv3r0v2jYpKUnl5eW25ejRo46UCQAAAAAAcMUcCk08PDzUr18/u0lcz0/qGhUVdcH9XnrpJc2ZM0cbNmxQRETEJc/j6ekpX19fuwUAAAAAAKAxOTSniSQlJiYqLi5OERERioyMVHp6uiorKxUfHy9JGjt2rDp27KjU1FRJ0osvvqhZs2Zp9erVCgkJUUlJiSSpZcuWatmyZT12BQAAAAAAoP44HJqMGjVKx48f16xZs1RSUqLw8HBt2LDBNjlscXGxXFx+HsCybNkyVVdX649//KPdcVJSUjR79uwrqx4AAAAAAKCBOByaSFJCQoISEhJMt+Xm5tq9Pnz4cF1OAQAAAAAA4FSN+vQcAAAAAACA5oLQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMBEnUKTjIwMhYSEyMvLS/3799f27dsv2PbLL7/UyJEjFRISIovFovT09LrWCgAAAAAA0GgcDk2ys7OVmJiolJQU7dy5U2FhYYqNjVVZWZlp+9OnT6tLly5asGCBAgICrrhgAAAAAACAxuBwaJKWlqbx48crPj5evXr1UmZmpnx8fJSVlWXa/je/+Y1efvll3X///fL09LziggEAAAAAABqDQ6FJdXW18vPzFRMT8/MBXFwUExOjvLy8ei8OAAAAAADAWdwcaXzixAnV1NTI39/fbr2/v7/27t1bb0VVVVWpqqrK9rqioqLejg0AAAAAAHA5muTTc1JTU+Xn52dbgoKCnF0SAAAAAAC4xjgUmrRt21aurq4qLS21W19aWlqvk7wmJSWpvLzcthw9erTejg0AAAAAAHA5HApNPDw81K9fP+Xk5NjWWa1W5eTkKCoqqt6K8vT0lK+vr90CAAAAAADQmBya00SSEhMTFRcXp4iICEVGRio9PV2VlZWKj4+XJI0dO1YdO3ZUamqqpHOTx+7Zs8f28zfffKOCggK1bNlS3bp1q8euAAAAAAAA1B+HQ5NRo0bp+PHjmjVrlkpKShQeHq4NGzbYJoctLi6Wi8vPA1j+85//qE+fPrbXCxcu1MKFCzVo0CDl5uZeeQ8AAAAAAAAagMOhiSQlJCQoISHBdNuvg5CQkBAZhlGX0wAAAAAAADhNk3x6DgAAAAAAgLMRmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAAAAAAAAE4QmAAAAAAAAJghNAAAAAAAATBCaAAAAAAAAmCA0AQAAAAAAMEFoAgAAAAAAYILQBAAAAAAAwAShCQAAAAAAgAlCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABMEJoAAAAAAACYIDQBAAAAAAAwUafQJCMjQyEhIfLy8lL//v21ffv2i7Z/++23ddNNN8nLy0uhoaF6//3361QsAAAAAABAY3E4NMnOzlZiYqJSUlK0c+dOhYWFKTY2VmVlZabtt27dqtGjR+vRRx/Vrl27NGLECI0YMUK7d+++4uIBAAAAAAAaisOhSVpamsaPH6/4+Hj16tVLmZmZ8vHxUVZWlmn7JUuW6K677tIzzzyjnj17as6cOerbt69effXVKy4eAAAAAACgobg50ri6ulr5+flKSkqyrXNxcVFMTIzy8vJM98nLy1NiYqLdutjYWL3zzjsXPE9VVZWqqqpsr8vLyyVJFRUVjpSLJsxaddrZJTSKCovh7BIaRc2PNc4uoVGcqrk2+sn/tUDTxjX06sI19OrCNfTqcv79NIxr4/8jmHMoNDlx4oRqamrk7+9vt97f31979+413aekpMS0fUlJyQXPk5qaqueff77W+qCgIEfKBZzOz9kFNJpCZxfQKCKdXUBj8bt2/uUCaLqunf+JuIZeVbiGXpV++OEH+fHeXrMcCk0aS1JSkt3oFKvVqu+++07XX3+9LBaLEysD8GsVFRUKCgrS0aNH5evr6+xyAABoNriGAk2bYRj64Ycf1KFDB2eXAidyKDRp27atXF1dVVpaare+tLRUAQEBpvsEBAQ41F6SPD095enpabeudevWjpQKoJH5+vryBx8AAHXANRRouhhhAocmgvXw8FC/fv2Uk5NjW2e1WpWTk6OoqCjTfaKiouzaS9JHH310wfYAAAAAAABNgcO35yQmJiouLk4RERGKjIxUenq6KisrFR8fL0kaO3asOnbsqNTUVEnS5MmTNWjQIC1atEjDhg3TmjVrtGPHDr3++uv12xMAAAAAAIB65HBoMmrUKB0/flyzZs1SSUmJwsPDtWHDBttkr8XFxXJx+XkAy4ABA7R69WrNnDlTzz33nLp376533nlHt9xyS/31AoDTeHp6KiUlpdYtdQAA4OK4hgJA02cxeH4SAAAAAABALQ7NaQIAAAAAAHCtIDQBAAAAAAAwQWgCAAAAAABggtAEaOIOHz4si8WigoKCBjtHbm6uLBaLTp482WDnkKTo6GhNmTKlQc8BALh2cI1sOiwWi9555x3b67179+rWW2+Vl5eXwsPDnVYXAFwpQhMAjWbt2rWaM2fOZbd35I/hmpoaJScnq3PnzvL29lbXrl01Z84c/XKu61OnTikhIUE33HCDvL291atXL2VmZtalKwAA1Kur7RqZkpKiFi1aqKioSDk5OXU+DgA4m8OPHAaAumrTpk2DHfvFF1/UsmXLtGrVKt18883asWOH4uPj5efnp0mTJkmSEhMTtXHjRv39739XSEiIPvzwQ02YMEEdOnTQ73//+warDQCAS7narpEHDx7UsGHD1KlTp/ruDgA0KkaaAE2E1WrVSy+9pG7dusnT01PBwcGaN2+ebftXX32l3/3ud/Lx8VFYWJjy8vLs9t+8ebNuv/12eXt7KygoSJMmTVJlZaVte1VVlaZNm6agoCB5enqqW7duWrFihWktp0+f1t13362BAwfq5MmTtm+z1qxZowEDBsjLy0u33HKLPv74Y7v9Pv74Y0VGRsrT01OBgYGaPn26fvrpJ9v2Xw89DgkJ0fz58/XII4+oVatWCg4O1uuvv27b3rlzZ0lSnz59ZLFYFB0dfcHf39atW3XPPfdo2LBhCgkJ0R//+EcNGTJE27dvt2sTFxen6OhohYSE6LHHHlNYWJhdGwBA08M1snldIy0Wi/Lz8/XCCy/IYrFo9uzZDh8DAJoKQhOgiUhKStKCBQuUnJysPXv2aPXq1fL397dtnzFjhp5++mkVFBToxhtv1OjRo21/bB08eFB33XWXRo4cqc8//1zZ2dnavHmzEhISbPuPHTtWb775pl555RUVFhbqtddeU8uWLWvVcfLkSd15552yWq366KOP1Lp1a9u2Z555Rk899ZR27dqlqKgoDR8+XN9++60k6ZtvvtHQoUP1m9/8Rp999pmWLVumFStWaO7cuRft96JFixQREaFdu3ZpwoQJeuKJJ1RUVCRJtj/U/vd//1fHjh3T2rVrL3icAQMGKCcnR/v27ZMkffbZZ9q8ebPuvvtuuzbvvfeevvnmGxmGoU2bNmnfvn0aMmTIRWsEADgX18jmdY08duyYbr75Zj311FM6duyYnn76aYePAQBNhgHA6SoqKgxPT09j+fLltbYdOnTIkGT85S9/sa378ssvDUlGYWGhYRiG8eijjxqPPfaY3X7//ve/DRcXF+PHH380ioqKDEnGRx99ZHr+TZs22Y7Xu3dvY+TIkUZVVVWtGhYsWGBbd/bsWeOGG24wXnzxRcMwDOO5554zevToYVitVlubjIwMo2XLlkZNTY1hGIYxaNAgY/LkybbtnTp1Mh566CHba6vVarRv395YtmyZ3Xl37dp10d+fYRhGTU2NMW3aNMNisRhubm6GxWIx5s+fb9fmzJkzxtixYw1Jhpubm+Hh4WGsWrXqkscGADgP18hzmvo1UpKxbt062+uwsDAjJSXlsvcHgKaKkSZAE1BYWKiqqirdcccdF2zTu3dv28+BgYGSpLKyMknnvjFauXKlWrZsaVtiY2NltVp16NAhFRQUyNXVVYMGDbpoHXfeeae6deum7OxseXh41NoeFRVl+9nNzU0REREqLCy09SEqKkoWi8XWZuDAgTp16pS+/vrry+qXxWJRQECArV9m/v3vf9v184033pAkvfXWW3rjjTe0evVq7dy5U6tWrdLChQu1atUq275Lly7Vtm3b9N577yk/P1+LFi3SxIkT9b//+78X/b0AAJyHa+Q5XCMBwDmYCBZoAry9vS/Zxt3d3fbz+T+6rFarpHMz3v/pT3+yTeb2S8HBwTpw4MBl1TFs2DD985//1J49exQaGnpZ+1ypX/ZLOte38/0yExERYfekgPPDs5955hlNnz5d999/vyQpNDRUR44cUWpqquLi4vTjjz/queee07p16zRs2DBJ5/4YLSgo0MKFCxUTE1PPPQMA1AeukT/jGgkAjY+RJkAT0L17d3l7e9f5kXx9+/bVnj171K1bt1qLh4eHQkNDZbVaa01K92sLFixQXFyc7rjjDu3Zs6fW9m3bttl+/umnn5Sfn6+ePXtKknr27Km8vDy7xxdu2bJFrVq10g033FCnfp3/Jq+mpsa2ztvb265/rVq1knRuYj4XF/v/0lxdXW1/XJ49e1Znz569aBsAQNPDNdIc10gAaByMNAGaAC8vL02bNk3PPvusPDw8NHDgQB0/flxffvnlRYcjnzdt2jTdeuutSkhI0Lhx49SiRQvt2bNHH330kV599VWFhIQoLi5OjzzyiF555RWFhYXpyJEjKisr03333Wd3rIULF6qmpkaDBw9Wbm6ubrrpJtu2jIwMde/eXT179tTixYv1/fff65FHHpEkTZgwQenp6XryySeVkJCgoqIipaSkKDExsdYfYZerffv28vb21oYNG3TDDTfIy8tLfn5+pm2HDx+uefPmKTg4WDfffLN27dqltLQ0W32+vr4aNGiQnnnmGXl7e6tTp076+OOP9be//U1paWl1qg8A0PC4RprjGgkAjcTZk6oAOKempsaYO3eu0alTJ8Pd3d0IDg425s+fbzrR2/fff29IMjZt2mRbt337duPOO+80WrZsabRo0cLo3bu3MW/ePNv2H3/80Zg6daoRGBhoeHh4GN26dTOysrIMw/h5krvvv//e1v7JJ580AgMDjaKiIlsNq1evNiIjIw0PDw+jV69exsaNG+36kJuba/zmN78xPDw8jICAAGPatGnG2bNnbdvNJrlbvHix3TF+PXHc8uXLjaCgIMPFxcUYNGjQBX9/FRUVxuTJk43g4GDDy8vL6NKlizFjxgy7yfqOHTtmPPzww0aHDh0MLy8vo0ePHsaiRYvsJuYDADQ9XCPPacrXSDERLICrlMUwfjFOEABMHD58WJ07d9auXbsUHh7u7HIAAGgyuEYCwNWNOU0AAAAAAABMEJoAAAAAuKA33njD7lHGv1xuvvlmZ5cHAA2K23MAAAAAXNAPP/yg0tJS023u7u7q1KlTI1cEAI2H0AQAAAAAAMAEt+cAAAAAAACYIDQBAAAAAAAwQWgCAAAAAABggtAEAAAAAADABKEJAAAAAACACUITAACuAtXV1c4uAQAA4KpDaAIAQDMUHR2thIQETZkyRW3btlVsbKwsFosKCgpsbU6ePCmLxaLc3FxJUm5uriwWi3JychQRESEfHx8NGDBARUVFzukEAABAE0doAgBAM7Vq1Sp5eHhoy5YtyszMvOz9ZsyYoUWLFmnHjh1yc3PTI4880oBVAgAANF9uzi4AAADUTffu3fXSSy9Jkg4fPnzZ+82bN0+DBg2SJE2fPl3Dhg3TmTNn5OXl1RBlAgAANFuMNAEAoJnq169fnfbr3bu37efAwEBJUllZWb3UBAAAcDUhNAEAoJlq0aKF7WcXl3OXdMMwbOvOnj1rup+7u7vtZ4vFIkmyWq0NUSIAAECzRmgCAMBVoF27dpKkY8eO2db9clJYAAAAOI45TQAAuAp4e3vr1ltv1YIFC9S5c2eVlZVp5syZzi4LAACgWWOkCQAAV4msrCz99NNP6tevn6ZMmaK5c+c6uyQAAIBmzWL88uZnAAAAAAAASGKkCQAAAAAAgClCEwAAAAAAABOEJgAAAAAAACYITQAAAAAAAEwQmgAAAAAAAJggNAEAAAAAADBBaAIAAAAAAGCC0AQAAAAAAMAEoQkAAAAAAIAJQhMAAAAAAAAThCYAAAAAAAAmCE0AAAAAAABM/H/BYUG/f2uWCAAAAABJRU5ErkJggg==","text/plain":[""]},"metadata":{},"output_type":"display_data"}],"source":["# plot metrics for each model\n","import matplotlib.pyplot as plt\n","\n","fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n","\n","perf_df.plot(x=\"run\", y=[\"accuracy\", \"precision\", \"recall\", \"f1\"], kind=\"bar\", ax=ax)\n","\n","# add values on top of bars\n","for p in ax.patches:\n"," ax.annotate(\n"," f\"{p.get_height():.3f}\",\n"," (p.get_x() + p.get_width() / 2, p.get_height()),\n"," ha=\"center\",\n"," va=\"bottom\",\n"," fontsize=10,\n"," )\n","\n","# add title and labels\n","# ax.set_title(\"Metrics for different settings\")\n","# ax.set_ylabel(\"Value\")\n","# ax.set_xlabel(\"Epoch (0: base model, 1-4: fine-tuned models)\")\n","# rotate x labels\n","plt.xticks(rotation=0)\n","\n","# set legend at the right to avoid overlapping with bars\n","plt.legend(loc=\"center left\", bbox_to_anchor=(1.0, 0.5))\n","# plt.tight_layout()\n","\n","plt.show()"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"07_MAC_+_Qwen2-7B-Instructi_Unsloth_train","widgets":{}},"colab":{"gpuType":"T4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}