Spaces:

arad1367
/

Q-Learning-Simulation

Running

App Files Files Community

arad1367 commited on 15 days ago

Commit

9ef4105

verified ·

1 Parent(s): edb5f16

Update index.html

Browse files

Files changed (1) hide show

index.html +1684 -18

index.html CHANGED Viewed

@@ -1,19 +1,1685 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

+<!-- Q-Learning Simulation By Pejman Ebrahimi -->
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Q-Learning Simulation by Pejman Ebrahimi</title>
+    <style>
+      :root {
+        --primary: #3f51b5;
+        --primary-light: #757de8;
+        --primary-dark: #002984;
+        --accent: #ff4081;
+        --accent-light: #ff79b0;
+        --accent-dark: #c60055;
+        --success: #4caf50;
+        --danger: #f44336;
+        --warning: #ff9800;
+        --dark: #212121;
+        --light: #fafafa;
+        --grid-size: 65px;
+        --border-radius: 8px;
+        --box-shadow: 0 3px 6px rgba(0, 0, 0, 0.1),
+          0 3px 6px rgba(0, 0, 0, 0.15);
+        --transition: all 0.3s cubic-bezier(0.25, 0.8, 0.25, 1);
+      }
+      * {
+        box-sizing: border-box;
+        margin: 0;
+        padding: 0;
+        font-family: Arial, sans-serif;
+      }
+      body {
+        background-color: var(--light);
+        color: var(--dark);
+        line-height: 1.6;
+        padding: 0;
+        display: flex;
+        flex-direction: column;
+        min-height: 100vh;
+      }
+      header {
+        background: linear-gradient(
+          135deg,
+          var(--primary-dark) 0%,
+          var(--primary) 100%
+        );
+        color: white;
+        padding: 2rem 0;
+        text-align: center;
+        position: relative;
+        overflow: hidden;
+        box-shadow: var(--box-shadow);
+      }
+      .container {
+        width: 90%;
+        max-width: 1400px;
+        margin: 0 auto;
+        padding: 2rem 0;
+      }
+      .header-content {
+        position: relative;
+        z-index: 10;
+      }
+      h1 {
+        font-size: 2.5rem;
+        margin: 0;
+        font-weight: 300;
+      }
+      .subtitle {
+        font-weight: 300;
+        margin-top: 0.5rem;
+        opacity: 0.9;
+      }
+      .attribution {
+        display: inline-block;
+        margin-top: 1rem;
+        padding: 0.5rem 1rem;
+        background-color: rgba(255, 255, 255, 0.1);
+        border-radius: 50px;
+        font-weight: 400;
+        letter-spacing: 0.5px;
+        position: relative;
+        z-index: 10;
+      }
+      #stars-container {
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 100%;
+        overflow: hidden;
+        z-index: 1;
+      }
+      .star {
+        position: absolute;
+        background-color: #fff;
+        width: 3px;
+        height: 3px;
+        border-radius: 50%;
+        opacity: 0;
+        animation: starAnimation 2s linear forwards;
+      }
+      @keyframes starAnimation {
+        0% {
+          transform: translate(0, 0) scale(0);
+          opacity: 1;
+        }
+        100% {
+          transform: translate(var(--tx), var(--ty)) scale(1);
+          opacity: 0;
+        }
+      }
+      main {
+        flex: 1;
+        padding: 2rem 0;
+      }
+      .grid-layout {
+        display: grid;
+        grid-template-columns: 1fr 1fr;
+        grid-gap: 2rem;
+      }
+      h2 {
+        font-weight: 400;
+        color: var(--primary);
+        margin-bottom: 1rem;
+        border-bottom: 1px solid #eee;
+        padding-bottom: 0.5rem;
+      }
+      h3 {
+        font-weight: 400;
+        color: var(--primary-dark);
+        margin-bottom: 0.5rem;
+      }
+      .card {
+        background-color: white;
+        border-radius: var(--border-radius);
+        box-shadow: var(--box-shadow);
+        padding: 1.5rem;
+        margin-bottom: 2rem;
+        transition: var(--transition);
+      }
+      .card:hover {
+        box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1),
+          0 6px 12px rgba(0, 0, 0, 0.15);
+      }
+      .flex-container {
+        display: flex;
+        gap: 1.5rem;
+      }
+      .world-container {
+        flex: 1;
+      }
+      .world-wrapper {
+        display: flex;
+        gap: 2rem;
+        justify-content: space-between;
+        align-items: flex-start;
+      }
+      .grid-container,
+      .q-values-container {
+        display: grid;
+        grid-template-columns: repeat(5, var(--grid-size));
+        grid-template-rows: repeat(5, var(--grid-size));
+        gap: 5px;
+      }
+      .grid-cell {
+        position: relative;
+        background-color: white;
+        border-radius: 4px;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        border: 1px solid #e0e0e0;
+        box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05);
+        transition: var(--transition);
+      }
+      .grid-cell.obstacle {
+        background-color: var(--danger);
+        color: white;
+      }
+      .grid-cell.goal {
+        background-color: var(--success);
+        color: white;
+      }
+      .agent {
+        position: absolute;
+        width: 26px;
+        height: 26px;
+        background-color: var(--primary);
+        border-radius: 50%;
+        z-index: 10;
+        box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+        transition: all 0.5s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+      }
+      .trail {
+        position: absolute;
+        width: 10px;
+        height: 10px;
+        background-color: rgba(63, 81, 181, 0.3);
+        border-radius: 50%;
+        z-index: 5;
+        transform: scale(0);
+        animation: pulseTrail 1.5s ease-out forwards;
+      }
+      @keyframes pulseTrail {
+        0% {
+          transform: scale(0);
+          opacity: 0.6;
+        }
+        100% {
+          transform: scale(1);
+          opacity: 0;
+        }
+      }
+      .q-cell {
+        background-color: white;
+        border-radius: 4px;
+        padding: 5px;
+        font-size: 0.7rem;
+        border: 1px solid #e0e0e0;
+        position: relative;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+      }
+      .q-arrow {
+        position: absolute;
+        width: 0;
+        height: 0;
+        border-style: solid;
+        transition: var(--transition);
+      }
+      .q-arrow.up {
+        top: 2px;
+        left: 50%;
+        transform: translateX(-50%);
+        border-width: 0 6px 8px 6px;
+        border-color: transparent transparent rgba(63, 81, 181, var(--opacity))
+          transparent;
+      }
+      .q-arrow.right {
+        top: 50%;
+        right: 2px;
+        transform: translateY(-50%);
+        border-width: 6px 0 6px 8px;
+        border-color: transparent transparent transparent
+          rgba(63, 81, 181, var(--opacity));
+      }
+      .q-arrow.down {
+        bottom: 2px;
+        left: 50%;
+        transform: translateX(-50%);
+        border-width: 8px 6px 0 6px;
+        border-color: rgba(63, 81, 181, var(--opacity)) transparent transparent
+          transparent;
+      }
+      .q-arrow.left {
+        top: 50%;
+        left: 2px;
+        transform: translateY(-50%);
+        border-width: 6px 8px 6px 0;
+        border-color: transparent rgba(63, 81, 181, var(--opacity)) transparent
+          transparent;
+      }
+      .q-value {
+        position: absolute;
+        font-size: 9px;
+        color: #666;
+        transition: var(--transition);
+      }
+      .q-value.best {
+        color: var(--primary);
+        font-weight: 700;
+      }
+      .reward-display {
+        position: absolute;
+        font-size: 14px;
+        font-weight: bold;
+        z-index: 20;
+        opacity: 0;
+        animation: fadeUp 1.2s cubic-bezier(0.175, 0.885, 0.32, 1.275) forwards;
+      }
+      @keyframes fadeUp {
+        0% {
+          opacity: 1;
+          transform: translateY(0);
+        }
+        100% {
+          opacity: 0;
+          transform: translateY(-30px);
+        }
+      }
+      .positive-reward {
+        color: var(--success);
+      }
+      .negative-reward {
+        color: var(--danger);
+      }
+      .control-section {
+        display: grid;
+        grid-template-columns: 1fr 1fr;
+        gap: 1.5rem;
+      }
+      .control-panel {
+        display: grid;
+        grid-template-columns: 1fr;
+        gap: 1rem;
+      }
+      .slider-container {
+        margin-bottom: 0.5rem;
+      }
+      .slider-label {
+        display: flex;
+        justify-content: space-between;
+        margin-bottom: 0.25rem;
+      }
+      .slider-label .value {
+        font-weight: 500;
+        color: var(--primary);
+      }
+      input[type="range"] {
+        width: 100%;
+        height: 6px;
+        background: #e0e0e0;
+        border-radius: 3px;
+        appearance: none;
+        margin: 0.5rem 0;
+      }
+      input[type="range"]::-webkit-slider-thumb {
+        appearance: none;
+        width: 16px;
+        height: 16px;
+        background: var(--primary);
+        border-radius: 50%;
+        cursor: pointer;
+        box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);
+      }
+      input[type="range"]::-moz-range-thumb {
+        width: 16px;
+        height: 16px;
+        background: var(--primary);
+        border-radius: 50%;
+        border: none;
+        cursor: pointer;
+        box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);
+      }
+      input[type="range"]::-ms-thumb {
+        width: 16px;
+        height: 16px;
+        background: var(--primary);
+        border-radius: 50%;
+        cursor: pointer;
+        box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);
+      }
+      .slider-description {
+        font-size: 0.8rem;
+        color: #666;
+        margin-top: 0.25rem;
+      }
+      .mode-selector {
+        display: flex;
+        gap: 0.5rem;
+        margin: 1rem 0;
+      }
+      .mode-btn {
+        flex: 1;
+        padding: 0.75rem;
+        text-align: center;
+        background-color: #f5f5f5;
+        border-radius: 4px;
+        cursor: pointer;
+        transition: var(--transition);
+        font-weight: 500;
+      }
+      .mode-btn:hover {
+        background-color: #e0e0e0;
+      }
+      .mode-btn.active {
+        background-color: var(--primary);
+        color: white;
+        box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+      }
+      .btn-container {
+        display: flex;
+        gap: 0.5rem;
+        margin-top: 1rem;
+      }
+      button {
+        padding: 0.75rem 1.25rem;
+        background-color: var(--primary);
+        color: white;
+        border: none;
+        border-radius: 4px;
+        cursor: pointer;
+        transition: var(--transition);
+        font-weight: 500;
+        box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+        flex: 1;
+      }
+      button:hover {
+        background-color: var(--primary-light);
+        box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+      }
+      button:active {
+        transform: translateY(1px);
+      }
+      button.secondary {
+        background-color: #e0e0e0;
+        color: #333;
+      }
+      button.secondary:hover {
+        background-color: #d0d0d0;
+      }
+      .stats-card {
+        display: grid;
+        grid-template-columns: repeat(4, 1fr);
+        gap: 1rem;
+        margin-bottom: 1.5rem;
+      }
+      .stat-box {
+        background-color: #f5f5f5;
+        padding: 1rem;
+        border-radius: 8px;
+        text-align: center;
+        transition: var(--transition);
+      }
+      .stat-box:hover {
+        transform: translateY(-2px);
+        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);
+      }
+      .stat-label {
+        font-size: 0.8rem;
+        color: #666;
+      }
+      .stat-value {
+        font-size: 1.5rem;
+        font-weight: 500;
+        color: var(--primary);
+        margin-top: 0.25rem;
+      }
+      .chart-container {
+        height: 250px;
+        margin: 1.5rem 0;
+        background-color: #f5f5f5;
+        border-radius: 8px;
+        padding: 1rem;
+        position: relative;
+      }
+      .simple-chart {
+        width: 100%;
+        height: 100%;
+        position: relative;
+      }
+      .chart-bar {
+        position: absolute;
+        bottom: 0;
+        background-color: var(--primary);
+        border-radius: 4px 4px 0 0;
+        transition: height 0.3s ease;
+      }
+      .chart-line {
+        position: absolute;
+        bottom: 50%;
+        left: 0;
+        width: 100%;
+        height: 1px;
+        background-color: rgba(0, 0, 0, 0.1);
+      }
+      .chart-label {
+        position: absolute;
+        bottom: -20px;
+        font-size: 10px;
+        text-align: center;
+        transform: translateX(-50%);
+        color: #666;
+      }
+      .chart-legend {
+        position: absolute;
+        top: 10px;
+        right: 10px;
+        display: flex;
+        gap: 10px;
+      }
+      .legend-item {
+        display: flex;
+        align-items: center;
+        gap: 5px;
+        font-size: 12px;
+      }
+      .legend-color {
+        width: 12px;
+        height: 12px;
+        border-radius: 2px;
+      }
+      .leaderboard {
+        width: 100%;
+        border-collapse: collapse;
+      }
+      .leaderboard th {
+        text-align: left;
+        padding: 0.75rem;
+        background-color: #f5f5f5;
+        font-weight: 500;
+      }
+      .leaderboard td {
+        padding: 0.75rem;
+        border-bottom: 1px solid #eee;
+      }
+      .leaderboard tr:last-child td {
+        border-bottom: none;
+      }
+      .notification {
+        position: fixed;
+        top: 20px;
+        right: 20px;
+        background-color: var(--primary);
+        color: white;
+        padding: 1rem 1.5rem;
+        border-radius: var(--border-radius);
+        box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
+        z-index: 1000;
+        opacity: 0;
+        transform: translateX(50px);
+        transition: var(--transition);
+        max-width: 300px;
+      }
+      .notification.show {
+        opacity: 1;
+        transform: translateX(0);
+      }
+      .notification-title {
+        font-weight: 500;
+        margin-bottom: 0.5rem;
+        display: flex;
+        align-items: center;
+        gap: 8px;
+      }
+      .notification-body {
+        font-size: 0.9rem;
+        opacity: 0.9;
+      }
+      .legend {
+        display: flex;
+        flex-wrap: wrap;
+        gap: 1rem;
+        margin-top: 1rem;
+        padding: 0.5rem;
+        background-color: #f5f5f5;
+        border-radius: 4px;
+      }
+      .legend-item {
+        display: flex;
+        align-items: center;
+        gap: 0.5rem;
+        font-size: 0.9rem;
+      }
+      .legend-color {
+        width: 12px;
+        height: 12px;
+        border-radius: 3px;
+      }
+      footer {
+        text-align: center;
+        padding: 1.5rem;
+        background-color: var(--primary-dark);
+        color: white;
+        margin-top: 2rem;
+      }
+      @media (max-width: 1200px) {
+        .grid-layout {
+          grid-template-columns: 1fr;
+        }
+        .world-wrapper {
+          flex-direction: column;
+          align-items: center;
+        }
+        .grid-container,
+        .q-values-container {
+          margin: 0 auto;
+        }
+        .control-section {
+          grid-template-columns: 1fr;
+        }
+        .stats-card {
+          grid-template-columns: repeat(2, 1fr);
+        }
+      }
+      @media (max-width: 768px) {
+        :root {
+          --grid-size: 50px;
+        }
+        .container {
+          width: 95%;
+          padding: 1rem 0;
+        }
+        h1 {
+          font-size: 1.8rem;
+        }
+        .control-panel {
+          grid-template-columns: 1fr;
+        }
+      }
+      /* Loading spinner for initial setup */
+      .loader {
+        border: 4px solid rgba(63, 81, 181, 0.1);
+        border-top: 4px solid var(--primary);
+        border-radius: 50%;
+        width: 40px;
+        height: 40px;
+        animation: spin 1s linear infinite;
+        margin: 2rem auto;
+      }
+      @keyframes spin {
+        0% {
+          transform: rotate(0deg);
+        }
+        100% {
+          transform: rotate(360deg);
+        }
+      }
+      /* Exploration mode visual */
+      .explore-indicator {
+        position: absolute;
+        width: 100%;
+        height: 100%;
+        background-color: rgba(255, 152, 0, 0.2);
+        border-radius: 4px;
+        z-index: 5;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+      }
+    </style>
+  </head>
+  <body>
+    <header>
+      <div id="stars-container"></div>
+      <div class="header-content">
+        <h1>Q-Learning Simulation</h1>
+        <p class="subtitle">Reinforcement Learning in Action</p>
+        <div class="attribution">Designed by Pejman Ebrahimi</div>
+      </div>
+    </header>
+    <main>
+      <div class="container">
+        <div class="loader" id="loader"></div>
+        <div class="grid-layout" id="main-content" style="display: none">
+          <div class="left-section">
+            <div class="card">
+              <h2>Interactive Environment</h2>
+              <div class="world-wrapper">
+                <div class="world-container">
+                  <h3>Grid World</h3>
+                  <div class="grid-container" id="grid-container"></div>
+                </div>
+                <div class="world-container">
+                  <h3>Q-Values</h3>
+                  <div class="q-values-container" id="q-values-container"></div>
+                </div>
+              </div>
+              <div class="legend">
+                <div class="legend-item">
+                  <div
+                    class="legend-color"
+                    style="background-color: var(--primary)"
+                  ></div>
+                  <span>Agent</span>
+                </div>
+                <div class="legend-item">
+                  <div
+                    class="legend-color"
+                    style="background-color: var(--success)"
+                  ></div>
+                  <span>Goal (+10)</span>
+                </div>
+                <div class="legend-item">
+                  <div
+                    class="legend-color"
+                    style="background-color: var(--danger)"
+                  ></div>
+                  <span>Obstacle (-5)</span>
+                </div>
+                <div class="legend-item">
+                  <div
+                    class="legend-color"
+                    style="background-color: rgba(63, 81, 181, 0.3)"
+                  ></div>
+                  <span>Path History</span>
+                </div>
+              </div>
+            </div>
+            <div class="card">
+              <h2>Learning Parameters</h2>
+              <div class="control-section">
+                <div class="control-panel">
+                  <div class="slider-container">
+                    <div class="slider-label">
+                      <span>Learning Rate (α)</span>
+                      <span class="value" id="learning-rate-value">0.1</span>
+                    </div>
+                    <input
+                      type="range"
+                      id="learning-rate"
+                      min="0.01"
+                      max="1"
+                      step="0.01"
+                      value="0.1"
+                    />
+                    <div class="slider-description">
+                      How quickly the agent incorporates new information
+                    </div>
+                  </div>
+                  <div class="slider-container">
+                    <div class="slider-label">
+                      <span>Discount Factor (γ)</span>
+                      <span class="value" id="discount-factor-value">0.9</span>
+                    </div>
+                    <input
+                      type="range"
+                      id="discount-factor"
+                      min="0"
+                      max="0.99"
+                      step="0.01"
+                      value="0.9"
+                    />
+                    <div class="slider-description">
+                      How much future rewards matter
+                    </div>
+                  </div>
+                </div>
+                <div class="control-panel">
+                  <div class="slider-container">
+                    <div class="slider-label">
+                      <span>Exploration Rate (ε)</span>
+                      <span class="value" id="exploration-rate-value">0.3</span>
+                    </div>
+                    <input
+                      type="range"
+                      id="exploration-rate"
+                      min="0"
+                      max="1"
+                      step="0.01"
+                      value="0.3"
+                    />
+                    <div class="slider-description">
+                      Chance of taking random actions
+                    </div>
+                  </div>
+                  <div class="slider-container">
+                    <div class="slider-label">
+                      <span>Animation Speed</span>
+                      <span class="value" id="animation-speed-value"
+                        >300ms</span
+                      >
+                    </div>
+                    <input
+                      type="range"
+                      id="animation-speed"
+                      min="50"
+                      max="1000"
+                      step="50"
+                      value="300"
+                    />
+                    <div class="slider-description">
+                      How quickly the simulation runs
+                    </div>
+                  </div>
+                </div>
+              </div>
+              <div class="mode-selector">
+                <div class="mode-btn active" id="learning-mode">
+                  Learning Mode
+                </div>
+                <div class="mode-btn" id="optimal-mode">
+                  Optimal Policy Mode
+                </div>
+              </div>
+              <div class="btn-container">
+                <button id="start-button">Start Learning</button>
+                <button id="step-button">Single Step</button>
+                <button id="reset-button" class="secondary">Reset</button>
+              </div>
+            </div>
+          </div>
+          <div class="right-section">
+            <div class="card">
+              <h2>Performance Metrics</h2>
+              <div class="stats-card">
+                <div class="stat-box">
+                  <div class="stat-label">Episodes</div>
+                  <div class="stat-value" id="episode-count">0</div>
+                </div>
+                <div class="stat-box">
+                  <div class="stat-label">Current Steps</div>
+                  <div class="stat-value" id="step-count">0</div>
+                </div>
+                <div class="stat-box">
+                  <div class="stat-label">Success Rate</div>
+                  <div class="stat-value" id="success-rate">0%</div>
+                </div>
+                <div class="stat-box">
+                  <div class="stat-label">Avg. Steps</div>
+                  <div class="stat-value" id="avg-completion">0</div>
+                </div>
+              </div>
+              <h3>Learning Progress</h3>
+              <div class="chart-container">
+                <div class="simple-chart" id="steps-chart"></div>
+                <div class="chart-legend">
+                  <div class="legend-item">
+                    <div
+                      class="legend-color"
+                      style="background-color: var(--primary)"
+                    ></div>
+                    <span>Steps per Episode</span>
+                  </div>
+                </div>
+              </div>
+            </div>
+            <div class="card">
+              <h2>Leaderboard - Best Paths</h2>
+              <table class="leaderboard">
+                <thead>
+                  <tr>
+                    <th>Rank</th>
+                    <th>Episode</th>
+                    <th>Steps</th>
+                    <th>Reward</th>
+                  </tr>
+                </thead>
+                <tbody id="leaderboard-body">
+                  <tr>
+                    <td colspan="4" style="text-align: center">
+                      No successful episodes yet
+                    </td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </div>
+    </main>
+    <footer>
+      <p>© 2025 Pejman Ebrahimi • Interactive Q-Learning Simulation</p>
+    </footer>
+    <div class="notification" id="notification">
+      <div class="notification-title">
+        <span>🏆</span>
+        <span id="notification-title">New Record!</span>
+      </div>
+      <div class="notification-body" id="notification-message"></div>
+    </div>
+    <script>
+      // Environment setup
+      const gridSize = 5;
+      const numStates = gridSize * gridSize;
+      const numActions = 4; // Up, Right, Down, Left
+      const obstacles = [7, 8, 17]; // Obstacle positions
+      const goal = 24; // Goal position
+      let agentPos = 0; // Start position
+      // Rewards
+      const stepReward = -0.1; // Penalty for each step
+      const obstacleReward = -5; // Penalty for hitting obstacle
+      const goalReward = 10; // Reward for reaching goal
+      // Learning state
+      let qTable = Array(numStates)
+        .fill()
+        .map(() => Array(numActions).fill(0));
+      let isLearning = false;
+      let isSingleStep = false;
+      let episodes = 0;
+      let steps = 0;
+      let episodeSteps = 0;
+      let stepsHistory = [];
+      let rewardsHistory = [];
+      let successCount = 0;
+      let totalCompletionSteps = 0;
+      let bestPathSteps = Infinity;
+      let leaderboard = [];
+      let currentEpisodeReward = 0;
+      let isOptimalMode = false;
+      let lastActionWasExploration = false;
+      // Agent parameters
+      let learningRate = 0.1;
+      let discountFactor = 0.9;
+      let explorationRate = 0.3;
+      let animationSpeed = 300;
+      // DOM Elements
+      const loader = document.getElementById("loader");
+      const mainContent = document.getElementById("main-content");
+      const gridContainer = document.getElementById("grid-container");
+      const qValuesContainer = document.getElementById("q-values-container");
+      const startButton = document.getElementById("start-button");
+      const stepButton = document.getElementById("step-button");
+      const resetButton = document.getElementById("reset-button");
+      const learningModeBtn = document.getElementById("learning-mode");
+      const optimalModeBtn = document.getElementById("optimal-mode");
+      const episodeCountEl = document.getElementById("episode-count");
+      const stepCountEl = document.getElementById("step-count");
+      const successRateEl = document.getElementById("success-rate");
+      const avgCompletionEl = document.getElementById("avg-completion");
+      const leaderboardBody = document.getElementById("leaderboard-body");
+      const notification = document.getElementById("notification");
+      const stepsChart = document.getElementById("steps-chart");
+      // Parameter sliders
+      const learningRateSlider = document.getElementById("learning-rate");
+      const learningRateValue = document.getElementById("learning-rate-value");
+      const discountFactorSlider = document.getElementById("discount-factor");
+      const discountFactorValue = document.getElementById(
+        "discount-factor-value"
+      );
+      const explorationRateSlider = document.getElementById("exploration-rate");
+      const explorationRateValue = document.getElementById(
+        "exploration-rate-value"
+      );
+      const animationSpeedSlider = document.getElementById("animation-speed");
+      const animationSpeedValue = document.getElementById(
+        "animation-speed-value"
+      );
+      // Update parameters from sliders
+      learningRateSlider.addEventListener("input", function () {
+        learningRate = parseFloat(this.value);
+        learningRateValue.textContent = learningRate.toFixed(2);
+      });
+      discountFactorSlider.addEventListener("input", function () {
+        discountFactor = parseFloat(this.value);
+        discountFactorValue.textContent = discountFactor.toFixed(2);
+      });
+      explorationRateSlider.addEventListener("input", function () {
+        explorationRate = parseFloat(this.value);
+        explorationRateValue.textContent = explorationRate.toFixed(2);
+      });
+      animationSpeedSlider.addEventListener("input", function () {
+        animationSpeed = parseInt(this.value);
+        animationSpeedValue.textContent = animationSpeed + "ms";
+      });
+      // Mode selection
+      learningModeBtn.addEventListener("click", function () {
+        if (isOptimalMode) {
+          isOptimalMode = false;
+          learningModeBtn.classList.add("active");
+          optimalModeBtn.classList.remove("active");
+        }
+      });
+      optimalModeBtn.addEventListener("click", function () {
+        if (!isOptimalMode) {
+          isOptimalMode = true;
+          optimalModeBtn.classList.add("active");
+          learningModeBtn.classList.remove("active");
+        }
+      });
+      // Create star animation for header
+      function createStars() {
+        const starsContainer = document.getElementById("stars-container");
+        const numStars = 40;
+        for (let i = 0; i < numStars; i++) {
+          setTimeout(() => {
+            const star = document.createElement("div");
+            star.className = "star";
+            // Random position
+            const x = Math.random() * 100;
+            const y = Math.random() * 100;
+            star.style.left = `${x}%`;
+            star.style.top = `${y}%`;
+            // Random size
+            const size = Math.random() * 4 + 1;
+            star.style.width = `${size}px`;
+            star.style.height = `${size}px`;
+            // Random direction
+            const tx = (Math.random() - 0.5) * 200;
+            const ty = (Math.random() - 0.5) * 200;
+            star.style.setProperty("--tx", `${tx}px`);
+            star.style.setProperty("--ty", `${ty}px`);
+            starsContainer.appendChild(star);
+            // Remove after animation
+            setTimeout(() => {
+              if (star) star.remove();
+            }, 2000);
+          }, i * 50);
+        }
+        // Repeat the animation
+        setTimeout(createStars, 4000);
+      }
+      // Initialize environment
+      function initializeGrid() {
+        gridContainer.innerHTML = "";
+        qValuesContainer.innerHTML = "";
+        // Create grid cells and q-value cells
+        for (let i = 0; i < numStates; i++) {
+          // Grid cell
+          const cell = document.createElement("div");
+          cell.className = "grid-cell";
+          cell.dataset.index = i;
+          if (obstacles.includes(i)) {
+            cell.classList.add("obstacle");
+            cell.textContent = "🚫";
+          } else if (i === goal) {
+            cell.classList.add("goal");
+            cell.textContent = "🏆";
+          }
+          gridContainer.appendChild(cell);
+          // Q-value cell
+          const qCell = document.createElement("div");
+          qCell.className = "q-cell";
+          qCell.dataset.index = i;
+          // Add arrows for each action
+          const directions = ["up", "right", "down", "left"];
+          for (let j = 0; j < numActions; j++) {
+            const arrow = document.createElement("div");
+            arrow.className = `q-arrow ${directions[j]}`;
+            arrow.dataset.action = j;
+            arrow.style.setProperty("--opacity", "0.2");
+            qCell.appendChild(arrow);
+            const value = document.createElement("div");
+            value.className = "q-value";
+            value.dataset.action = j;
+            value.textContent = "0.00";
+            qCell.appendChild(value);
+          }
+          // Position the q-values within the cell
+          const values = qCell.querySelectorAll(".q-value");
+          values[0].style.position = "absolute";
+          values[0].style.top = "5px";
+          values[0].style.left = "50%";
+          values[0].style.transform = "translateX(-50%)";
+          values[1].style.position = "absolute";
+          values[1].style.top = "50%";
+          values[1].style.right = "5px";
+          values[1].style.transform = "translateY(-50%)";
+          values[2].style.position = "absolute";
+          values[2].style.bottom = "5px";
+          values[2].style.left = "50%";
+          values[2].style.transform = "translateX(-50%)";
+          values[3].style.position = "absolute";
+          values[3].style.top = "50%";
+          values[3].style.left = "5px";
+          values[3].style.transform = "translateY(-50%)";
+          qValuesContainer.appendChild(qCell);
+        }
+        // Add agent
+        const startCell = document.querySelector(
+          `.grid-cell[data-index="${agentPos}"]`
+        );
+        const agent = document.createElement("div");
+        agent.className = "agent";
+        agent.id = "agent";
+        startCell.appendChild(agent);
+      }
+      // Create a simple chart without external libraries
+      function updateSimpleChart() {
+        if (stepsHistory.length === 0) return;
+        // Clear chart
+        stepsChart.innerHTML = "";
+        // Find max value for scaling
+        const maxSteps = Math.max(...stepsHistory);
+        const chartWidth = stepsChart.clientWidth;
+        const chartHeight = stepsChart.clientHeight;
+        const barWidth = Math.max(5, chartWidth / stepsHistory.length - 4);
+        // Create bars
+        stepsHistory.forEach((steps, i) => {
+          // Calculate height percentage
+          const heightPercent = steps / maxSteps;
+          const barHeight = heightPercent * chartHeight * 0.8;
+          // Create bar
+          const bar = document.createElement("div");
+          bar.className = "chart-bar";
+          bar.style.height = `${barHeight}px`;
+          bar.style.width = `${barWidth}px`;
+          bar.style.left = `${i * (chartWidth / stepsHistory.length)}px`;
+          // Add tooltip
+          bar.title = `Episode ${i + 1}: ${steps} steps`;
+          // Add to chart
+          stepsChart.appendChild(bar);
+          // Add label every 5 episodes
+          if ((i + 1) % 5 === 0 || i === 0) {
+            const label = document.createElement("div");
+            label.className = "chart-label";
+            label.textContent = i + 1;
+            label.style.left = `${
+              i * (chartWidth / stepsHistory.length) + barWidth / 2
+            }px`;
+            stepsChart.appendChild(label);
+          }
+        });
+        // Add mid line
+        const midLine = document.createElement("div");
+        midLine.className = "chart-line";
+        stepsChart.appendChild(midLine);
+      }
+      // Show notification
+      function showNotification(title, message, duration = 3000) {
+        document.getElementById("notification-title").textContent = title;
+        document.getElementById("notification-message").textContent = message;
+        notification.classList.add("show");
+        setTimeout(() => {
+          notification.classList.remove("show");
+        }, duration);
+      }
+      // Update leaderboard
+      function updateLeaderboard() {
+        // Sort leaderboard by steps (ascending)
+        leaderboard.sort((a, b) => a.steps - b.steps);
+        // Keep only top 5
+        if (leaderboard.length > 5) {
+          leaderboard = leaderboard.slice(0, 5);
+        }
+        // Update display
+        leaderboardBody.innerHTML = "";
+        if (leaderboard.length === 0) {
+          const row = document.createElement("tr");
+          const cell = document.createElement("td");
+          cell.colSpan = 4;
+          cell.style.textAlign = "center";
+          cell.textContent = "No successful episodes yet";
+          row.appendChild(cell);
+          leaderboardBody.appendChild(row);
+        } else {
+          leaderboard.forEach((entry, index) => {
+            const row = document.createElement("tr");
+            const rankCell = document.createElement("td");
+            rankCell.textContent = index + 1;
+            row.appendChild(rankCell);
+            const episodeCell = document.createElement("td");
+            episodeCell.textContent = entry.episode;
+            row.appendChild(episodeCell);
+            const stepsCell = document.createElement("td");
+            stepsCell.textContent = entry.steps;
+            row.appendChild(stepsCell);
+            const rewardCell = document.createElement("td");
+            rewardCell.textContent = entry.reward.toFixed(1);
+            row.appendChild(rewardCell);
+            leaderboardBody.appendChild(row);
+          });
+        }
+      }
+      // Update Q-value visualization
+      function updateQValues() {
+        for (let i = 0; i < numStates; i++) {
+          const qCell = qValuesContainer.querySelector(
+            `.q-cell[data-index="${i}"]`
+          );
+          // Skip obstacles and goal
+          if (obstacles.includes(i) || i === goal) continue;
+          // Find max Q-value for this state
+          const maxQ = Math.max(...qTable[i]);
+          const bestAction = qTable[i].indexOf(maxQ);
+          // Update each action's display
+          for (let j = 0; j < numActions; j++) {
+            const qValue = qTable[i][j];
+            const valueEl = qCell.querySelector(`.q-value[data-action="${j}"]`);
+            const arrowEl = qCell.querySelector(`.q-arrow[data-action="${j}"]`);
+            // Update value text
+            valueEl.textContent = qValue.toFixed(2);
+            // Update styling for best action
+            if (j === bestAction && maxQ > 0) {
+              valueEl.classList.add("best");
+            } else {
+              valueEl.classList.remove("best");
+            }
+            // Update arrow opacity based on value
+            const opacity =
+              qValue <= 0 ? 0.1 : Math.min(0.2 + (qValue / 10) * 0.8, 1);
+            arrowEl.style.setProperty("--opacity", opacity);
+          }
+        }
+      }
+      // Get action based on Q-values and exploration rate
+      function getAction(state) {
+        // In optimal mode, always choose best action
+        if (isOptimalMode) {
+          return qTable[state].indexOf(Math.max(...qTable[state]));
+        }
+        // Exploration (random action)
+        if (Math.random() < explorationRate) {
+          lastActionWasExploration = true;
+          return Math.floor(Math.random() * numActions);
+        }
+        // Exploitation (best action)
+        lastActionWasExploration = false;
+        return qTable[state].indexOf(Math.max(...qTable[state]));
+      }
+      // Get next state based on current state and action
+      function getNextState(state, action) {
+        let row = Math.floor(state / gridSize);
+        let col = state % gridSize;
+        let newRow = row;
+        let newCol = col;
+        // Move according to action (0=Up, 1=Right, 2=Down, 3=Left)
+        switch (action) {
+          case 0:
+            newRow = Math.max(0, row - 1);
+            break;
+          case 1:
+            newCol = Math.min(gridSize - 1, col + 1);
+            break;
+          case 2:
+            newRow = Math.min(gridSize - 1, row + 1);
+            break;
+          case 3:
+            newCol = Math.max(0, col - 1);
+            break;
+        }
+        return newRow * gridSize + newCol;
+      }
+      // Get reward for a given state
+      function getReward(state) {
+        if (state === goal) return goalReward;
+        if (obstacles.includes(state)) return obstacleReward;
+        return stepReward;
+      }
+      // Check if episode is done
+      function isDone(state) {
+        return state === goal; // Only goal state ends episode - obstacles don't terminate episode
+      }
+      // Move agent
+      function moveAgent(newPos) {
+        const agent = document.getElementById("agent");
+        if (agent) agent.remove();
+        const cell = document.querySelector(
+          `.grid-cell[data-index="${newPos}"]`
+        );
+        const newAgent = document.createElement("div");
+        newAgent.className = "agent";
+        newAgent.id = "agent";
+        cell.appendChild(newAgent);
+        // Add trail effect
+        const oldCell = document.querySelector(
+          `.grid-cell[data-index="${agentPos}"]`
+        );
+        if (oldCell && agentPos !== newPos) {
+          const trail = document.createElement("div");
+          trail.className = "trail";
+          oldCell.appendChild(trail);
+          // Remove trail after a delay
+          setTimeout(() => {
+            if (trail) trail.remove();
+          }, 2000);
+        }
+        // Show exploration indicator
+        if (lastActionWasExploration) {
+          const exploreIndicator = document.createElement("div");
+          exploreIndicator.className = "explore-indicator";
+          exploreIndicator.innerHTML = "🔍";
+          cell.appendChild(exploreIndicator);
+          setTimeout(() => {
+            if (exploreIndicator) exploreIndicator.remove();
+          }, animationSpeed * 0.8);
+        }
+        agentPos = newPos;
+      }
+      // Display reward
+      function displayReward(reward, pos) {
+        const cell = document.querySelector(`.grid-cell[data-index="${pos}"]`);
+        const display = document.createElement("div");
+        display.className = "reward-display";
+        if (reward > 0) {
+          display.classList.add("positive-reward");
+          display.textContent = `+${reward}`;
+        } else {
+          display.classList.add("negative-reward");
+          display.textContent = reward;
+        }
+        cell.appendChild(display);
+        // Remove after animation completes
+        setTimeout(() => {
+          if (display) display.remove();
+        }, 1500);
+      }
+      // Update statistics
+      function updateStats() {
+        episodeCountEl.textContent = episodes;
+        stepCountEl.textContent = episodeSteps;
+        // Success rate
+        if (episodes > 0) {
+          const rate = Math.round((successCount / episodes) * 100);
+          successRateEl.textContent = `${rate}%`;
+        }
+        // Average completion steps
+        if (successCount > 0) {
+          const avg = Math.round(totalCompletionSteps / successCount);
+          avgCompletionEl.textContent = avg;
+        }
+      }
+      // Take a step in the environment
+      function step() {
+        if (!isLearning && !isSingleStep) return;
+        // Choose action
+        const action = getAction(agentPos);
+        // Get new state and reward
+        const newState = getNextState(agentPos, action);
+        const reward = getReward(newState);
+        // Update total reward for this episode
+        currentEpisodeReward += reward;
+        // Update Q-value
+        if (!isOptimalMode) {
+          const maxFutureQ = Math.max(...qTable[newState]);
+          qTable[agentPos][action] =
+            qTable[agentPos][action] +
+            learningRate *
+              (reward + discountFactor * maxFutureQ - qTable[agentPos][action]);
+        }
+        // Move agent
+        moveAgent(newState);
+        displayReward(reward, newState);
+        // Update display
+        updateQValues();
+        // Update counters
+        steps++;
+        episodeSteps++;
+        updateStats();
+        // Check if episode is done (only when reaching goal, not obstacles)
+        if (isDone(newState) || episodeSteps > 100) {
+          // If it reached the goal successfully
+          if (newState === goal) {
+            successCount++;
+            totalCompletionSteps += episodeSteps;
+            // Check if it's a new record
+            if (episodeSteps < bestPathSteps) {
+              bestPathSteps = episodeSteps;
+              // Show notification
+              showNotification(
+                "New Record!",
+                `Episode ${
+                  episodes + 1
+                } found the goal in just ${episodeSteps} steps with a total reward of ${currentEpisodeReward.toFixed(
+                  1
+                )}!`
+              );
+            }
+            // Add to leaderboard
+            leaderboard.push({
+              episode: episodes + 1,
+              steps: episodeSteps,
+              reward: currentEpisodeReward,
+            });
+            updateLeaderboard();
+          }
+          // Reset for next episode
+          setTimeout(() => {
+            episodes++;
+            // Store episode data for chart
+            stepsHistory.push(episodeSteps);
+            rewardsHistory.push(currentEpisodeReward);
+            updateSimpleChart();
+            // Reduce exploration rate over time
+            if (!isOptimalMode) {
+              explorationRate = Math.max(0.01, explorationRate * 0.99);
+              explorationRateValue.textContent = explorationRate.toFixed(2);
+              explorationRateSlider.value = explorationRate;
+            }
+            // Reset agent position
+            agentPos = 0;
+            moveAgent(agentPos);
+            episodeSteps = 0;
+            currentEpisodeReward = 0;
+            updateStats();
+            // Continue learning if not single step
+            if (isLearning && !isSingleStep) {
+              setTimeout(step, animationSpeed);
+            } else {
+              isSingleStep = false;
+            }
+          }, animationSpeed);
+        } else {
+          // Continue episode
+          if (isLearning && !isSingleStep) {
+            setTimeout(step, animationSpeed);
+          } else {
+            isSingleStep = false;
+          }
+        }
+      }
+      // Reset environment
+      function resetEnvironment() {
+        if (isLearning) {
+          isLearning = false;
+          startButton.textContent = "Start Learning";
+        }
+        // Clear trails
+        document.querySelectorAll(".trail").forEach((trail) => trail.remove());
+        document
+          .querySelectorAll(".reward-display")
+          .forEach((display) => display.remove());
+        document
+          .querySelectorAll(".explore-indicator")
+          .forEach((indicator) => indicator.remove());
+        // Reset agent position
+        agentPos = 0;
+        moveAgent(agentPos);
+        // Reset learning state
+        qTable = Array(numStates)
+          .fill()
+          .map(() => Array(numActions).fill(0));
+        episodes = 0;
+        steps = 0;
+        episodeSteps = 0;
+        stepsHistory = [];
+        rewardsHistory = [];
+        successCount = 0;
+        totalCompletionSteps = 0;
+        bestPathSteps = Infinity;
+        leaderboard = [];
+        currentEpisodeReward = 0;
+        // Reset parameters to defaults
+        learningRate = 0.1;
+        discountFactor = 0.9;
+        explorationRate = 0.3;
+        // Update sliders
+        learningRateSlider.value = learningRate;
+        learningRateValue.textContent = learningRate.toFixed(2);
+        discountFactorSlider.value = discountFactor;
+        discountFactorValue.textContent = discountFactor.toFixed(2);
+        explorationRateSlider.value = explorationRate;
+        explorationRateValue.textContent = explorationRate.toFixed(2);
+        // Update display
+        updateQValues();
+        updateStats();
+        updateLeaderboard();
+        updateSimpleChart();
+      }
+      // Start/stop learning
+      startButton.addEventListener("click", function () {
+        if (isLearning) {
+          isLearning = false;
+          startButton.textContent = "Start Learning";
+        } else {
+          isLearning = true;
+          startButton.textContent = "Pause Learning";
+          step();
+        }
+      });
+      // Single step
+      stepButton.addEventListener("click", function () {
+        if (!isLearning) {
+          isSingleStep = true;
+          step();
+        }
+      });
+      // Reset environment
+      resetButton.addEventListener("click", resetEnvironment);
+      // Initialize
+      window.onload = function () {
+        // Show loader while initializing
+        setTimeout(() => {
+          initializeGrid();
+          updateQValues();
+          createStars();
+          // Hide loader and show main content
+          loader.style.display = "none";
+          mainContent.style.display = "grid";
+          // Show welcome notification
+          showNotification(
+            "Welcome!",
+            'Click "Start Learning" to begin the Q-Learning simulation.',
+            5000
+          );
+        }, 800);
+      };
+    </script>
+  </body>
 </html>