Merge marimo-team/learn (add __marimo__ to gitignore)

.github/workflows/deploy.yml

@@ -0,0 +1,56 @@
+name: Deploy to GitHub Pages
+
+on:
+  push:
+    branches: ['main']
+  workflow_dispatch:
+
+concurrency:
+  group: 'pages'
+  cancel-in-progress: false
+
+env:
+  UV_SYSTEM_PYTHON: 1
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 🚀 Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: 🐍 Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: 📦 Install dependencies
+        run: |
+          uv pip install marimo
+
+      - name: 🛠️ Export notebooks
+        run: |
+          python scripts/build.py
+
+      - name: 📤 Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site
+
+  deploy:
+    needs: build
+
+    permissions:
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: 🚀 Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/hf_sync.yml

@@ -0,0 +1,45 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config --global user.name "GitHub Action"
+          git config --global user.email "[email protected]"
+
+      - name: Prepend frontmatter to README
+        run: |
+          if [ -f README.md ] && ! grep -q "^---" README.md; then
+            FRONTMATTER="---
+          title: marimo learn
+          emoji: 🧠
+          colorFrom: blue
+          colorTo: indigo
+          sdk: docker
+          sdk_version: \"latest\"
+          app_file: app.py
+          pinned: false
+          ---
+
+          "
+            echo "$FRONTMATTER$(cat README.md)" > README.md
+            git add README.md
+            git commit -m "Add HF frontmatter to README" || echo "No changes to commit"
+          fi
+
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push -f https://mylessss:[email protected]/spaces/marimo-team/marimo-learn main

.github/workflows/typos.yaml

@@ -13,4 +13,3 @@ jobs:
         uses: styfle/[email protected]
       - uses: actions/checkout@v4
       - uses: crate-ci/[email protected]
-name: Tests

.gitignore

@@ -172,3 +172,6 @@ cython_debug/
 
 # Marimo specific
 __marimo__
+
+# Generated site content
+_site/

Dockerfile

@@ -0,0 +1,26 @@
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+WORKDIR /app
+
+# Create a non-root user
+RUN useradd -m appuser
+
+# Copy application files
+COPY _server/main.py _server/main.py
+COPY polars/ polars/
+COPY duckdb/ duckdb/
+
+# Set proper ownership
+RUN chown -R appuser:appuser /app
+
+# Switch to non-root user
+USER appuser
+
+# Create virtual environment and install dependencies
+RUN uv venv
+RUN uv export --script _server/main.py | uv pip install -r -
+
+ENV PORT=7860
+EXPOSE 7860
+
+CMD ["uv", "run", "_server/main.py"]

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <em>A curated collection of educational <a href="https://github.com/marimo-team/marimo">marimo</a> notebooks</em>.
+  <span><em>A curated collection of educational <a href="https://github.com/marimo-team/marimo">marimo</a> notebooks</em> || <a href="https://discord.gg/rT48v2Y9fe">💬 Discord</a></span>
 </p>
 
 # 📚 Learn
@@ -29,6 +29,7 @@ notebooks for educators, students, and practitioners.
 - 📏 Linear algebra
 - ❄️ Polars
 - 🔥 Pytorch
+- 🗄️ Duckdb
 - 📈 Altair
 - 📈 Plotly
 - 📈 matplotlib
@@ -57,12 +58,27 @@ Here's a contribution checklist:
 If you aren't comfortable adding a new notebook or course, you can also request
 what you'd like to see by [filing an issue](https://github.com/marimo-team/learn/issues/new?template=example_request.yaml).
 
+## Building and Previewing
+
+The site is built using a Python script that exports marimo notebooks to HTML and generates an index page.
+
+```bash
+# Build the site
+python scripts/build.py --output-dir _site
+
+# Preview the site (builds first)
+python scripts/preview.py
+
+# Preview without rebuilding
+python scripts/preview.py --no-build
+```
+
 ## Community
 
 We're building a community. Come hang out with us!
 
 - 🌟 [Star us on GitHub](https://github.com/marimo-team/examples)
-- 💬 [Chat with us on Discord](https://marimo.io/discord?ref=readme)
+- 💬 [Chat with us on Discord](https://discord.gg/rT48v2Y9fe)
 - 📧 [Subscribe to our Newsletter](https://marimo.io/newsletter)
 - ☁️ [Join our Cloud Waitlist](https://marimo.io/cloud)
 - ✏️ [Start a GitHub Discussion](https://github.com/marimo-team/marimo/discussions)

_server/README.md

@@ -0,0 +1,24 @@
+# marimo learn server
+
+This folder contains server code for hosting marimo apps.
+
+## Running the server
+
+```bash
+cd _server
+uv run --no-project main.py
+```
+
+## Building a Docker image
+
+From the root directory, run:
+
+```bash
+docker build -t marimo-learn .
+```
+
+## Running the Docker container
+
+```bash
+docker run -p 7860:7860 marimo-learn
+```

_server/main.py

@@ -0,0 +1,90 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "fastapi",
+#     "marimo",
+#     "starlette",
+#     "python-dotenv",
+#     "pydantic",
+#     "duckdb",
+#     "altair==5.5.0",
+#     "beautifulsoup4==4.13.3",
+#     "httpx==0.28.1",
+#     "marimo",
+#     "nest-asyncio==1.6.0",
+#     "numba==0.61.0",
+#     "numpy==2.1.3",
+#     "polars==1.24.0",
+# ]
+# ///
+
+import logging
+import os
+from pathlib import Path
+
+import marimo
+from dotenv import load_dotenv
+from fastapi import FastAPI, Request
+from fastapi.responses import HTMLResponse
+
+# Load environment variables
+load_dotenv()
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Get port from environment variable or use default
+PORT = int(os.environ.get("PORT", 7860))
+
+root_dir = Path(__file__).parent.parent
+
+ROOTS = [
+    root_dir / "polars",
+    root_dir / "duckdb",
+]
+
+
+server = marimo.create_asgi_app(include_code=True)
+app_names: list[str] = []
+
+for root in ROOTS:
+    for filename in root.iterdir():
+        if filename.is_file() and filename.suffix == ".py":
+            app_path = root.stem + "/" + filename.stem
+            server = server.with_app(path=f"/{app_path}", root=str(filename))
+            app_names.append(app_path)
+
+# Create a FastAPI app
+app = FastAPI()
+
+logger.info(f"Mounting {len(app_names)} apps")
+for app_name in app_names:
+    logger.info(f"  /{app_name}")
+
+
+@app.get("/")
+async def home(request: Request):
+    html_content = """
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>marimo learn</title>
+    </head>
+    <body>
+        <h1>Welcome to marimo learn!</h1>
+        <p>This is a collection of interactive tutorials for learning data science libraries with marimo.</p>
+        <p>Check out the <a href="https://github.com/marimo-team/learn">GitHub repository</a> for more information.</p>
+    </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content)
+
+
+app.mount("/", server.build())
+
+# Run the server
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")

duckdb/README.md

@@ -0,0 +1,28 @@
+# Learn DuckDB
+
+_🚧 This collection is a work in progress. Please help us add notebooks!_
+
+This collection of marimo notebooks is designed to teach you the basics of
+DuckDB, a fast in-memory OLAP engine that can interoperate with Dataframes.
+These notebooks also show how marimo gives DuckDB superpowers.
+
+**Help us build this course! ⚒️**
+
+We're seeking contributors to help us build these notebooks. Every contributor
+will be acknowledged as an author in this README and in their contributed
+notebooks. Head over to the [tracking
+issue](https://github.com/marimo-team/learn/issues/48) to sign up for a planned
+notebook or propose your own.
+
+**Running notebooks.** To run a notebook locally, use
+
+```bash
+uvx marimo edit <file_url>
+```
+
+You can also open notebooks in our online playground by appending marimo.app/ to a notebook's URL.
+
+
+**Authors.**
+
+Thanks to all our notebook authors!

functional_programming/05_functors.py

@@ -0,0 +1,1313 @@
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "marimo",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.17"
+app = marimo.App(app_title="Category Theory and Functors")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        # Category Theory and Functors
+
+        In this notebook, you will learn:
+
+        * Why `length` is a *functor* from the category of `list concatenation` to the category of `integer addition`
+        * How to *lift* an ordinary function into a specific *computational context*
+        * How to write an *adapter* between two categories
+
+        In short, a mathematical functor is a **mapping** between two categories in category theory. In practice, a functor represents a type that can be mapped over.
+
+        /// admonition | Intuitions 
+
+        - A simple intuition is that a `Functor` represents a **container** of values, along with the ability to apply a function uniformly to every element in the container.
+        - Another intuition is that a `Functor` represents some sort of **computational context**.
+        - Mathematically, `Functors` generalize the idea of a container or a computational context.
+        ///
+
+        We will start with intuition, introduce the basics of category theory, and then examine functors from a categorical perspective.
+
+        /// details | Notebook metadata
+            type: info
+
+        version: 0.1.1 | last modified: 2025-03-16 | author: [métaboulie](https://github.com/metaboulie)<br/>
+        reviewer: [Haleshot](https://github.com/Haleshot)
+
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        # Functor as a Computational Context
+
+        A [**Functor**](https://wiki.haskell.org/Functor) is an abstraction that represents a computational context with the ability to apply a function to every value inside it without altering the structure of the context itself. This enables transformations while preserving the shape of the data.
+
+        To understand this, let's look at a simple example.
+
+        ## [The One-Way Wrapper Design Pattern](http://blog.sigfpe.com/2007/04/trivial-monad.html)
+
+        Often, we need to wrap data in some kind of context. However, when performing operations on wrapped data, we typically have to:
+
+        1. Unwrap the data.
+        2. Modify the unwrapped data.
+        3. Rewrap the modified data.
+
+        This process is tedious and inefficient. Instead, we want to wrap data **once** and apply functions directly to the wrapped data without unwrapping it.
+
+        /// admonition | Rules for a One-Way Wrapper
+
+        1. We can wrap values, but we cannot unwrap them.
+        2. We should still be able to apply transformations to the wrapped data.
+        3. Any operation that depends on wrapped data should itself return a wrapped result.
+        ///
+
+        Let's define such a `Wrapper` class:
+
+        ```python
+        from dataclasses import dataclass
+        from typing import Callable, Generic, TypeVar
+
+        A = TypeVar("A")
+        B = TypeVar("B")
+
+        @dataclass
+        class Wrapper(Generic[A]):
+            value: A
+        ```
+
+        Now, we can create an instance of wrapped data:
+
+        ```python
+        wrapped = Wrapper(1)
+        ```
+
+        ### Mapping Functions Over Wrapped Data
+
+        To modify wrapped data while keeping it wrapped, we define an `fmap` method:
+
+        ```python
+        @dataclass
+        class Wrapper(Functor, Generic[A]):
+            value: A
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "Wrapper[A]") -> "Wrapper[B]":
+                return Wrapper(f(a.value))
+        ```
+
+        Now, we can apply transformations without unwrapping:
+
+        ```python
+        >>> Wrapper.fmap(lambda x: x + 1, wrapper)
+        Wrapper(value=2)
+
+        >>> Wrapper.fmap(lambda x: [x], wrapper)
+        Wrapper(value=[1])
+        ```
+
+        > Try using the `Wrapper` in the cell below.
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, B, Callable, Functor, Generic, dataclass, pp):
+    @dataclass
+    class Wrapper(Functor, Generic[A]):
+        value: A
+
+        @classmethod
+        def fmap(cls, f: Callable[[A], B], a: "Wrapper[A]") -> "Wrapper[B]":
+            return Wrapper(f(a.value))
+
+
+    wrapper = Wrapper(1)
+
+    pp(Wrapper.fmap(lambda x: x + 1, wrapper))
+    pp(Wrapper.fmap(lambda x: [x], wrapper))
+    return Wrapper, wrapper
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        We can analyze the type signature of `fmap` for `Wrapper`:
+
+        * `f` is of type `Callable[[A], B]`
+        * `a` is of type `Wrapper[A]`
+        * The return value is of type `Wrapper[B]`
+
+        Thus, in Python's type system, we can express the type signature of `fmap` as:
+
+        ```python
+        fmap(f: Callable[[A], B], a: Wrapper[A]) -> Wrapper[B]:
+        ```
+
+        Essentially, `fmap`:
+
+        1. Takes a function `Callable[[A], B]` and a `Wrapper[A]` instance as input.
+        2. Applies the function to the value inside the wrapper.
+        3. Returns a new `Wrapper[B]` instance with the transformed value, leaving the original wrapper and its internal data unmodified.
+
+        Now, let's examine `list` as a similar kind of wrapper.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## The List Wrapper
+
+        We can define a `List` class to represent a wrapped list that supports `fmap`:
+
+        ```python
+        @dataclass
+        class List(Functor, Generic[A]):
+            value: list[A]
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "List[A]") -> "List[B]":
+                return List([f(x) for x in a.value])
+        ```
+
+        Now, we can apply transformations:
+
+        ```python
+        >>> flist = List([1, 2, 3, 4])
+        >>> List.fmap(lambda x: x + 1, flist)
+        List(value=[2, 3, 4, 5])
+        >>> List.fmap(lambda x: [x], flist)
+        List(value=[[1], [2], [3], [4]])
+        ```
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, B, Callable, Functor, Generic, dataclass, pp):
+    @dataclass
+    class List(Functor, Generic[A]):
+        value: list[A]
+
+        @classmethod
+        def fmap(cls, f: Callable[[A], B], a: "List[A]") -> "List[B]":
+            return List([f(x) for x in a.value])
+
+
+    flist = List([1, 2, 3, 4])
+    pp(List.fmap(lambda x: x + 1, flist))
+    pp(List.fmap(lambda x: [x], flist))
+    return List, flist
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ### Extracting the Type of `fmap`
+
+        The type signature of `fmap` for `List` is:
+
+        ```python
+        fmap(f: Callable[[A], B], a: List[A]) -> List[B]
+        ```
+
+        Similarly, for `Wrapper`:
+
+        ```python
+        fmap(f: Callable[[A], B], a: Wrapper[A]) -> Wrapper[B]
+        ```
+
+        Both follow the same pattern, which we can generalize as:
+
+        ```python
+        fmap(f: Callable[[A], B], a: Functor[A]) -> Functor[B]
+        ```
+
+        where `Functor` can be `Wrapper`, `List`, or any other wrapper type that follows the same structure.
+
+        ### Functors in Haskell (optional)
+
+        In Haskell, the type of `fmap` is:
+
+        ```haskell
+        fmap :: Functor f => (a -> b) -> f a -> f b
+        ```
+
+        or equivalently:
+
+        ```haskell
+        fmap :: Functor f => (a -> b) -> (f a -> f b)
+        ```
+
+        This means that `fmap` **lifts** an ordinary function into the **functor world**, allowing it to operate within a computational context.
+
+        Now, let's define an abstract class for `Functor`.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Defining Functor
+
+        Recall that, a **Functor** is an abstraction that allows us to apply a function to values inside a computational context while preserving its structure. 
+
+        To define `Functor` in Python, we use an abstract base class:
+
+        ```python
+        from dataclasses import dataclass
+        from typing import Callable, Generic, TypeVar
+        from abc import ABC, abstractmethod
+
+        A = TypeVar("A")
+        B = TypeVar("B")
+
+        @dataclass
+        class Functor(ABC, Generic[A]):
+            @classmethod
+            @abstractmethod
+            def fmap(f: Callable[[A], B], a: "Functor[A]") -> "Functor[B]":
+                raise NotImplementedError
+        ```
+
+        We can now extend custom wrappers, containers, or computation contexts with this `Functor` base class, implement the `fmap` method, and apply any function.
+
+        Next, let's implement a more complex data structure: [RoseTree](https://en.wikipedia.org/wiki/Rose_tree).
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Case Study: RoseTree
+
+        A **RoseTree** is a tree where:
+
+        - Each node holds a **value**.
+        - Each node has a **list of child nodes** (which are also RoseTrees).
+
+        This structure is useful for representing hierarchical data, such as:
+
+        - Abstract Syntax Trees (ASTs)
+        - File system directories
+        - Recursive computations
+
+        We can implement `RoseTree` by extending the `Functor` class:
+
+        ```python
+        from dataclasses import dataclass
+        from typing import Callable, Generic, TypeVar
+
+        A = TypeVar("A")
+        B = TypeVar("B")
+
+        @dataclass
+        class RoseTree(Functor, Generic[a]):
+    
+            value: A
+            children: list["RoseTree[A]"]
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "RoseTree[A]") -> "RoseTree[B]":
+                return RoseTree(
+                    f(a.value), [cls.fmap(f, child) for child in a.children]
+                )
+
+            def __repr__(self) -> str:
+                return f"Node: {self.value}, Children: {self.children}"
+        ```
+
+        - The function is applied **recursively** to each node's value.
+        - The tree structure **remains unchanged**.
+        - Only the values inside the tree are modified.
+
+        > Try using `RoseTree` in the cell below.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(A, B, Callable, Functor, Generic, dataclass, mo):
+    @dataclass
+    class RoseTree(Functor, Generic[A]):
+        """
+        ### Doc: RoseTree
+
+        A Functor implementation of `RoseTree`, allowing transformation of values while preserving the tree structure.
+
+        **Attributes**
+
+        - `value (A)`: The value stored in the node.
+        - `children (list[RoseTree[A]])`: A list of child nodes forming the tree structure.
+
+        **Methods:**
+
+        - `fmap(f: Callable[[A], B], a: "RoseTree[A]") -> "RoseTree[B]"`
+
+            Applies a function to each value in the tree, producing a new `RoseTree[b]` with transformed values.
+
+        **Implementation logic:**
+
+          - The function `f` is applied to the root node's `value`.
+          - Each child in `children` recursively calls `fmap`, ensuring all values in the tree are mapped.
+          - The overall tree structure remains unchanged.
+        """
+
+        value: A
+        children: list["RoseTree[A]"]
+
+        @classmethod
+        def fmap(cls, f: Callable[[A], B], a: "RoseTree[A]") -> "RoseTree[B]":
+            return RoseTree(
+                f(a.value), [cls.fmap(f, child) for child in a.children]
+            )
+
+        def __repr__(self) -> str:
+            return f"Node: {self.value}, Children: {self.children}"
+
+
+    mo.md(RoseTree.__doc__)
+    return (RoseTree,)
+
+
+@app.cell
+def _(RoseTree, pp):
+    rosetree = RoseTree(1, [RoseTree(2, []), RoseTree(3, [RoseTree(4, [])])])
+
+    pp(rosetree)
+    pp(RoseTree.fmap(lambda x: [x], rosetree))
+    pp(RoseTree.fmap(lambda x: RoseTree(x, []), rosetree))
+    return (rosetree,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Generic Functions that can be Used with Any Functor
+
+        One of the powerful features of functors is that we can write **generic functions** that can work with any functor.
+
+        Remember that in Haskell, the type of `fmap` can be written as:
+
+        ```haskell
+        fmap :: Functor f => (a -> b) -> (f a -> f b)
+        ```
+
+        Translating to Python, we get:
+
+        ```python
+        def fmap(func: Callable[[A], B]) -> Callable[[Functor[A]], Functor[B]]
+        ```
+
+        This means that `fmap`:
+
+        - Takes an **ordinary function** `Callable[[A], B]` as input.
+        - Outputs a function that:
+            - Takes a **functor** of type `Functor[A]` as input.
+            - Outputs a **functor** of type `Functor[B]`.
+
+        We can implement a similar idea in Python:
+
+        ```python
+        fmap = lambda f, functor: functor.__class__.fmap(f, functor)
+        inc = lambda functor: fmap(lambda x: x + 1, functor)
+        ```
+
+        - **`fmap`**: Lifts an ordinary function (`f`) to the functor world, allowing the function to operate on the wrapped value inside the functor.
+        - **`inc`**: A specific instance of `fmap` that operates on any functor. It takes a functor, applies the function `lambda x: x + 1` to every value inside it, and returns a new functor with the updated values.
+
+        Thus, **`fmap`** transforms an ordinary function into a **function that operates on functors**, and **`inc`** is a specific case where it increments the value inside the functor.
+
+        ### Applying the `inc` Function to Various Functors
+
+        You can now apply `inc` to any functor like `Wrapper`, `List`, or `RoseTree`:
+
+        ```python
+        # Applying `inc` to a Wrapper
+        wrapper = Wrapper(5)
+        inc(wrapper)  # Wrapper(value=6)
+
+        # Applying `inc` to a List
+        list_wrapper = List([1, 2, 3])
+        inc(list_wrapper)  # List(value=[2, 3, 4])
+
+        # Applying `inc` to a RoseTree
+        tree = RoseTree(1, [RoseTree(2, []), RoseTree(3, [])])
+        inc(tree)  # RoseTree(value=2, children=[RoseTree(value=3, children=[]), RoseTree(value=4, children=[])])
+        ```
+
+        > Try using `fmap` in the cell below.
+        """
+    )
+    return
+
+
+@app.cell
+def _(flist, pp, rosetree, wrapper):
+    fmap = lambda f, functor: functor.__class__.fmap(f, functor)
+    inc = lambda functor: fmap(lambda x: x + 1, functor)
+
+    pp(inc(wrapper))
+    pp(inc(flist))
+    pp(inc(rosetree))
+    return fmap, inc
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Functor laws
+
+        In addition to providing a function `fmap` of the specified type, functors are also required to satisfy two equational laws:
+
+        ```haskell
+        fmap id = id                    -- fmap preserves identity
+        fmap (g . h) = fmap g . fmap h  -- fmap distributes over composition
+        ```
+
+        1. `fmap` should preserve the **identity function**, in the sense that applying `fmap` to this function returns the same function as the result.
+        2. `fmap` should also preserve **function composition**. Applying two composed functions `g` and `h` to a functor via `fmap` should give the same result as first applying `fmap` to `g` and then applying `fmap` to `h`.
+
+        /// admonition | 
+        - Any `Functor` instance satisfying the first law `(fmap id = id)` will automatically satisfy the [second law](https://github.com/quchen/articles/blob/master/second_functor_law.mo) as well.
+        ///
+
+        ### Functor Law Verification
+
+        We can define `id` and `compose` in `Python` as below:
+
+        ```python
+        id = lambda x: x
+        compose = lambda f, g: lambda x: f(g(x))
+        ```
+
+        We can add a helper function `check_functor_law` to verify that an instance satisfies the functor laws.
+
+        ```Python
+        check_functor_law = lambda functor: repr(fmap(id, functor)) == repr(functor)
+        ```
+
+        We can verify the functor we've defined.
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    id = lambda x: x
+    compose = lambda f, g: lambda x: f(g(x))
+    return compose, id
+
+
+@app.cell
+def _(fmap, id):
+    check_functor_law = lambda functor: repr(fmap(id, functor)) == repr(functor)
+    return (check_functor_law,)
+
+
+@app.cell
+def _(check_functor_law, flist, pp, rosetree, wrapper):
+    for functor in (wrapper, flist, rosetree):
+        pp(check_functor_law(functor))
+    return (functor,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        And here is an `EvilFunctor`. We can verify it's not a valid `Functor`.
+
+        ```python
+        @dataclass
+        class EvilFunctor(Functor, Generic[A]):
+            value: list[A]
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "EvilFunctor[A]") -> "EvilFunctor[B]":
+                return (
+                    cls([a.value[0]] * 2 + list(map(f, a.value[1:])))
+                    if a.value
+                    else []
+                )
+        ```
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, B, Callable, Functor, Generic, check_functor_law, dataclass, pp):
+    @dataclass
+    class EvilFunctor(Functor, Generic[A]):
+        value: list[A]
+
+        @classmethod
+        def fmap(
+            cls, f: Callable[[A], B], a: "EvilFunctor[A]"
+        ) -> "EvilFunctor[B]":
+            return (
+                cls([a.value[0]] * 2 + [f(x) for x in a.value[1:]])
+                if a.value
+                else []
+            )
+
+
+    pp(check_functor_law(EvilFunctor([1, 2, 3, 4])))
+    return (EvilFunctor,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Final definition of Functor
+
+        We can now draft the final definition of `Functor` with some utility functions.
+
+        ```Python
+            @classmethod
+            @abstractmethod
+            def fmap(cls, f: Callable[[A], B], a: "Functor[A]") -> "Functor[B]":
+                return NotImplementedError
+
+            @classmethod
+            def const_fmap(cls, a: "Functor[A]", b: B) -> "Functor[B]":
+                return cls.fmap(lambda _: b, a)
+
+            @classmethod
+            def void(cls, a: "Functor[A]") -> "Functor[None]":
+                return cls.const_fmap(a, None)
+        ```
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(A, ABC, B, Callable, Generic, abstractmethod, dataclass, mo):
+    @dataclass
+    class Functor(ABC, Generic[A]):
+        """
+        ### Doc: Functor
+
+        A generic interface for types that support mapping over their values.
+
+        **Methods:**
+
+        - `fmap(f: Callable[[A], B], a: Functor[A]) -> Functor[B]`
+          Abstract method to apply a function to all values inside a functor.
+
+        - `const_fmap(a: "Functor[A]", b: B) -> Functor[B]`
+          Replaces all values inside a functor with a constant `b`, preserving the original structure.
+
+        - `void(a: "Functor[A]") -> Functor[None]`
+          Equivalent to `const_fmap(a, None)`, transforming all values in a functor into `None`.
+        """
+
+        @classmethod
+        @abstractmethod
+        def fmap(cls, f: Callable[[A], B], a: "Functor[A]") -> "Functor[B]":
+            return NotImplementedError
+
+        @classmethod
+        def const_fmap(cls, a: "Functor[A]", b: B) -> "Functor[B]":
+            return cls.fmap(lambda _: b, a)
+
+        @classmethod
+        def void(cls, a: "Functor[A]") -> "Functor[None]":
+            return cls.const_fmap(a, None)
+
+
+    mo.md(Functor.__doc__)
+    return (Functor,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md("""> Try with utility functions in the cell below""")
+    return
+
+
+@app.cell
+def _(List, RoseTree, flist, pp, rosetree):
+    pp(RoseTree.const_fmap(rosetree, "λ"))
+    pp(RoseTree.void(rosetree))
+    pp(List.const_fmap(flist, "λ"))
+    pp(List.void(flist))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Functors for Non-Iterable Types
+
+        In the previous examples, we implemented functors for **iterables**, like `List` and `RoseTree`, which are inherently **iterable types**. This is a natural fit for functors, as iterables can be mapped over.
+
+        However, **functors are not limited to iterables**. There are cases where we want to apply the concept of functors to types that are not inherently iterable, such as types that represent optional values, computations, or other data structures.
+
+        ### The Maybe Functor
+
+        One example is the **`Maybe`** type from Haskell, which is used to represent computations that can either result in a value or no value (`Nothing`). 
+
+        We can define the `Maybe` functor as below:
+
+        ```python
+        @dataclass
+        class Maybe(Functor, Generic[A]):
+            value: None | A
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "Maybe[A]") -> "Maybe[B]":
+                return (
+                    cls(None) if a.value is None else cls(f(a.value))
+                )
+
+            def __repr__(self):
+                return "Nothing" if self.value is None else repr(self.value)
+        ```
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, B, Callable, Functor, Generic, dataclass):
+    @dataclass
+    class Maybe(Functor, Generic[A]):
+        value: None | A
+
+        @classmethod
+        def fmap(cls, f: Callable[[A], B], a: "Maybe[A]") -> "Maybe[B]":
+            return cls(None) if a.value is None else cls(f(a.value))
+
+        def __repr__(self):
+            return "Nothing" if self.value is None else repr(self.value)
+    return (Maybe,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        **`Maybe`** is a functor that can either hold a value or be `Nothing` (equivalent to `None` in Python). The `fmap` method applies a function to the value inside the functor, if it exists. If the value is `None` (representing `Nothing`), `fmap` simply returns `None`.
+
+        By using `Maybe` as a functor, we gain the ability to apply transformations (`fmap`) to potentially absent values, without having to explicitly handle the `None` case every time.
+
+        > Try using `Maybe` in the cell below.
+        """
+    )
+    return
+
+
+@app.cell
+def _(Maybe, pp):
+    mint = Maybe(1)
+    mnone = Maybe(None)
+
+    pp(Maybe.fmap(lambda x: x + 1, mint))
+    pp(Maybe.fmap(lambda x: x + 1, mnone))
+    return mint, mnone
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Limitations of Functor
+
+        Functors abstract the idea of mapping a function over each element of a structure. Suppose now that we wish to generalise this idea to allow functions with any number of arguments to be mapped, rather than being restricted to functions with a single argument. More precisely, suppose that we wish to define a hierarchy of `fmap` functions with the following types:
+
+        ```haskell
+        fmap0 :: a -> f a
+
+        fmap1 :: (a -> b) -> f a -> f b
+
+        fmap2 :: (a -> b -> c) -> f a -> f b -> f c
+
+        fmap3 :: (a -> b -> c -> d) -> f a -> f b -> f c -> f d
+        ```
+
+        And we have to declare a special version of the functor class for each case.
+
+        We will learn how to resolve this problem in the next notebook on `Applicatives`.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        # Introduction to Categories
+
+        A [category](https://en.wikibooks.org/wiki/Haskell/Category_theory#Introduction_to_categories) is, in essence, a simple collection. It has three components: 
+
+        - A collection of **objects**.
+        - A collection of **morphisms**, each of which ties two objects (a _source object_ and a _target object_) together. If $f$ is a morphism with source object $C$ and target object $B$, we write $f : C → B$.
+        - A notion of **composition** of these morphisms. If $g : A → B$ and $f : B → C$ are two morphisms, they can be composed, resulting in a morphism $f ∘ g : A → C$.
+
+        ## Category laws
+
+        There are three laws that categories need to follow. 
+
+        1. The composition of morphisms needs to be **associative**. Symbolically, $f ∘ (g ∘ h) = (f ∘ g) ∘ h$
+
+            - Morphisms are applied right to left, so with $f ∘ g$ first $g$ is applied, then $f$. 
+
+        2. The category needs to be **closed** under the composition operation. So if $f : B → C$ and $g : A → B$, then there must be some morphism $h : A → C$ in the category such that $h = f ∘ g$. 
+
+        3. Given a category $C$ there needs to be for every object $A$ an **identity** morphism, $id_A : A → A$ that is an identity of composition with other morphisms. Put precisely, for every morphism $g : A → B$: $g ∘ id_A = id_B ∘ g = g$
+
+        /// attention | The definition of a category does not define: 
+
+        - what `∘` is,
+        - what `id` is, or
+        - what `f`, `g`, and `h` might be. 
+
+        Instead, category theory leaves it up to us to discover what they might be.
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## The Python category
+
+        The main category we'll be concerning ourselves with in this part is the Python category, or we can give it a shorter name: `Py`. `Py` treats Python types as objects and Python functions as morphisms. A function `def f(a: A) -> B` for types A and B is a morphism in Python.
+
+        Remember that we defined the `id` and `compose` function above as:
+
+        ```Python
+        def id(x: Generic[A]) -> Generic[A]:
+            return x
+
+        def compose(f: Callable[[B], C], g: Callable[[A], B]) -> Callable[[A], C]:
+            return lambda x: f(g(x))  
+        ```
+
+        We can check second law easily. 
+
+        For the first law, we have:
+
+        ```python
+        # compose(f, g) = lambda x: f(g(x))
+        f ∘ (g ∘ h) 
+        = compose(f, compose(g, h)) 
+        = lambda x: f(compose(g, h)(x))
+        = lambda x: f(lambda y: g(h(y))(x))
+        = lambda x: f(g(h(x)))
+
+        (f ∘ g) ∘ h 
+        = compose(compose(f, g), h)
+        = lambda x: compose(f, g)(h(x))
+        = lambda x: lambda y: f(g(y))(h(x))
+        = lambda x: f(g(h(x)))
+        ```
+
+        For the third law, we have: 
+
+        ```python
+        g ∘ id_A 
+        = compose(g: Callable[[a], b], id: Callable[[a], a]) -> Callable[[a], b]
+        = lambda x: g(id(x))
+        = lambda x: g(x) # id(x) = x
+        = g
+        ```
+        the similar proof can be applied to $id_B ∘ g =g$.
+
+        Thus `Py` is a valid category.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        # Functors, again
+
+        A functor is essentially a transformation between categories, so given categories $C$ and $D$, a functor $F : C → D$:
+
+        - Maps any object $A$ in $C$ to $F ( A )$, in $D$.
+        - Maps morphisms $f : A → B$ in $C$ to $F ( f ) : F ( A ) → F ( B )$ in $D$.
+
+        /// admonition | 
+
+        Endofunctors are functors from a category to itself.
+
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Functors on the category of Python
+
+        Remember that a functor has two parts: it maps objects in one category to objects in another and morphisms in the first category to morphisms in the second. 
+
+        Functors in Python are from `Py` to `func`, where `func` is the subcategory of `Py` defined on just that functor's types. E.g. the RoseTree functor goes from `Py` to `RoseTree`, where `RoseTree` is the category containing only RoseTree types, that is, `RoseTree[T]` for any type `T`. The morphisms in `RoseTree` are functions defined on RoseTree types, that is, functions `Callable[[RoseTree[T]], RoseTree[U]]` for types `T`, `U`.
+
+        Recall the definition of `Functor`:
+
+        ```Python
+        @dataclass
+        class Functor(ABC, Generic[A])
+        ```
+
+        And RoseTree: 
+
+        ```Python
+        @dataclass
+        class RoseTree(Functor, Generic[A])
+        ```
+
+        **Here's the key part:** the _type constructor_ `RoseTree` takes any type `T` to a new type, `RoseTree[T]`. Also, `fmap` restricted to `RoseTree` types takes a function `Callable[[A], B]` to a function `Callable[[RoseTree[A]], RoseTree[B]]`.
+
+        But that's it. We've defined two parts, something that takes objects in `Py` to objects in another category (that of `RoseTree` types and functions defined on `RoseTree` types), and something that takes morphisms in `Py` to morphisms in this category. So `RoseTree` is a functor. 
+
+        To sum up:
+
+        - We work in the category **Py** and its subcategories.  
+        - **Objects** are types (e.g., `int`, `str`, `list`).  
+        - **Morphisms** are functions (`Callable[[A], B]`).  
+        - **Things that take a type and return another type** are type constructors (`RoseTree[T]`).  
+        - **Things that take a function and return another function** are higher-order functions (`Callable[[Callable[[A], B]], Callable[[C], D]]`).  
+        - **Abstract base classes (ABC)** and duck typing provide a way to express polymorphism, capturing the idea that in category theory, structures are often defined over multiple objects at once.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Functor laws, again
+
+        Once again there are a few axioms that functors have to obey. 
+
+        1. Given an identity morphism $id_A$ on an object $A$, $F ( id_A )$ must be the identity morphism on $F ( A )$, i.e.: ${\displaystyle F(\operatorname {id} _{A})=\operatorname {id} _{F(A)}}$
+        2. Functors must distribute over morphism composition, i.e. ${\displaystyle F(f\circ g)=F(f)\circ F(g)}$
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        Remember that we defined the `fmap`, `id` and `compose` as 
+        ```python
+        fmap = lambda f, functor: functor.__class__.fmap(f, functor)  
+        id = lambda x: x
+        compose = lambda f, g: lambda x: f(g(x))
+        ```
+
+        Let's prove that `fmap` is a functor.
+
+        First, let's define a `Category` for a specific `Functor`. We choose to define the `Category` for the `Wrapper` as `WrapperCategory` here for simplicity, but remember that `Wrapper` can be any `Functor`(i.e. `List`, `RoseTree`, `Maybe` and more):
+
+        **Notice that** in this case, we can actually view `fmap` as:
+        ```python
+        fmap = lambda f, functor: functor.fmap(f, functor)  
+        ```
+
+        We define `WrapperCategory` as:
+
+        ```python
+        @dataclass
+        class WrapperCategory:
+            @staticmethod
+            def id(wrapper: Wrapper[A]) -> Wrapper[A]:
+                return Wrapper(wrapper.value)
+
+            @staticmethod
+            def compose(
+                f: Callable[[Wrapper[B]], Wrapper[C]],
+                g: Callable[[Wrapper[A]], Wrapper[B]],
+                wrapper: Wrapper[A]
+            ) -> Callable[[Wrapper[A]], Wrapper[C]]:
+                return f(g(Wrapper(wrapper.value)))
+        ```
+
+        And `Wrapper` is:
+
+        ```Python
+        @dataclass
+        class Wrapper(Functor, Generic[A]):
+            value: A
+
+            @classmethod
+            def fmap(cls, f: Callable[[A], B], a: "Wrapper[A]") -> "Wrapper[B]":
+                return Wrapper(f(a.value))
+        ```
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        We can prove that:
+
+        ```python
+        fmap(id, wrapper)
+        = Wrapper.fmap(id, wrapper)
+        = Wrapper(id(wrapper.value))
+        = Wrapper(wrapper.value)
+        = WrapperCategory.id(wrapper)
+        ```
+        and:
+        ```python
+        fmap(compose(f, g), wrapper)
+        = Wrapper.fmap(compose(f, g), wrapper)
+        = Wrapper(compose(f, g)(wrapper.value))
+        = Wrapper(f(g(wrapper.value)))
+
+        WrapperCategory.compose(fmap(f, wrapper), fmap(g, wrapper), wrapper)
+        = fmap(f, wrapper)(fmap(g, wrapper)(wrapper))
+        = fmap(f, wrapper)(Wrapper.fmap(g, wrapper))
+        = fmap(f, wrapper)(Wrapper(g(wrapper.value)))
+        = Wrapper.fmap(f, Wrapper(g(wrapper.value)))
+        = Wrapper(f(Wrapper(g(wrapper.value)).value))
+        = Wrapper(f(g(wrapper.value)))  # Wrapper(g(wrapper.value)).value = g(wrapper.value)
+        ```
+
+        So our `Wrapper` is a valid `Functor`.
+
+        > Try validating functor laws for `Wrapper` below.
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, B, C, Callable, Wrapper, dataclass):
+    @dataclass
+    class WrapperCategory:
+        @staticmethod
+        def id(wrapper: Wrapper[A]) -> Wrapper[A]:
+            return Wrapper(wrapper.value)
+
+        @staticmethod
+        def compose(
+            f: Callable[[Wrapper[B]], Wrapper[C]],
+            g: Callable[[Wrapper[A]], Wrapper[B]],
+            wrapper: Wrapper[A],
+        ) -> Callable[[Wrapper[A]], Wrapper[C]]:
+            return f(g(Wrapper(wrapper.value)))
+    return (WrapperCategory,)
+
+
+@app.cell
+def _(WrapperCategory, fmap, id, pp, wrapper):
+    pp(fmap(id, wrapper) == WrapperCategory.id(wrapper))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ## Length as a Functor
+
+        Remember that a functor is a transformation between two categories. It is not only limited to a functor from `Py` to `func`, but also includes transformations between other mathematical structures.
+
+        Let’s prove that **`length`** can be viewed as a functor. Specifically, we will demonstrate that `length` is a functor from the **category of list concatenation** to the **category of integer addition**.
+
+        ### Category of List Concatenation
+
+        First, let’s define the category of list concatenation:
+
+        ```python
+        @dataclass
+        class ListConcatenation(Generic[A]):
+            value: list[A]
+
+            @staticmethod
+            def id() -> "ListConcatenation[A]":
+                return ListConcatenation([])
+
+            @staticmethod
+            def compose(
+                this: "ListConcatenation[A]", other: "ListConcatenation[A]"
+            ) -> "ListConcatenation[a]":
+                return ListConcatenation(this.value + other.value)
+        ```
+        """
+    )
+    return
+
+
+@app.cell
+def _(A, Generic, dataclass):
+    @dataclass
+    class ListConcatenation(Generic[A]):
+        value: list[A]
+
+        @staticmethod
+        def id() -> "ListConcatenation[A]":
+            return ListConcatenation([])
+
+        @staticmethod
+        def compose(
+            this: "ListConcatenation[A]", other: "ListConcatenation[A]"
+        ) -> "ListConcatenation[a]":
+            return ListConcatenation(this.value + other.value)
+    return (ListConcatenation,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        - **Identity**: The identity element is an empty list (`ListConcatenation([])`).
+        - **Composition**: The composition of two lists is their concatenation (`this.value + other.value`).
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ### Category of Integer Addition
+
+        Now, let's define the category of integer addition:
+
+        ```python
+        @dataclass
+        class IntAddition:
+            value: int
+
+            @staticmethod
+            def id() -> "IntAddition":
+                return IntAddition(0)
+
+            @staticmethod
+            def compose(this: "IntAddition", other: "IntAddition") -> "IntAddition":
+                return IntAddition(this.value + other.value)
+        ```
+        """
+    )
+    return
+
+
+@app.cell
+def _(dataclass):
+    @dataclass
+    class IntAddition:
+        value: int
+
+        @staticmethod
+        def id() -> "IntAddition":
+            return IntAddition(0)
+
+        @staticmethod
+        def compose(this: "IntAddition", other: "IntAddition") -> "IntAddition":
+            return IntAddition(this.value + other.value)
+    return (IntAddition,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        - **Identity**: The identity element is `IntAddition(0)` (the additive identity).
+        - **Composition**: The composition of two integers is their sum (`this.value + other.value`).
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ### Defining the Length Functor
+
+        We now define the `length` function as a functor, mapping from the category of list concatenation to the category of integer addition:
+
+        ```python
+        length = lambda l: IntAddition(len(l.value))
+        ```
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(IntAddition):
+    length = lambda l: IntAddition(len(l.value))
+    return (length,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md("""This function takes an instance of `ListConcatenation`, computes its length, and returns an `IntAddition` instance with the computed length.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        ### Verifying Functor Laws
+
+        Now, let’s verify that `length` satisfies the two functor laws.
+
+        #### 1. **Identity Law**:
+        The identity law states that applying the functor to the identity element of one category should give the identity element of the other category.
+
+        ```python
+        > length(ListConcatenation.id()) == IntAddition.id()
+        True
+        ```
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md("""This ensures that the length of an empty list (identity in the `ListConcatenation` category) is `0` (identity in the `IntAddition` category).""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        #### 2. **Composition Law**:
+        The composition law states that the functor should preserve composition. Applying the functor to a composed element should be the same as composing the functor applied to the individual elements.
+
+        ```python
+        > lista = ListConcatenation([1, 2])
+        > listb = ListConcatenation([3, 4])
+        > length(ListConcatenation.compose(lista, listb)) == IntAddition.compose(
+        >     length(lista), length(listb)
+        > )
+        True
+        ```
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md("""This ensures that the length of the concatenation of two lists is the same as the sum of the lengths of the individual lists.""")
+    return
+
+
+@app.cell
+def _(IntAddition, ListConcatenation, length, pp):
+    pp(length(ListConcatenation.id()) == IntAddition.id())
+    lista = ListConcatenation([1, 2])
+    listb = ListConcatenation([3, 4])
+    pp(
+        length(ListConcatenation.compose(lista, listb))
+        == IntAddition.compose(length(lista), length(listb))
+    )
+    return lista, listb
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        """
+        # Further reading
+
+        - [The Trivial Monad](http://blog.sigfpe.com/2007/04/trivial-monad.html)
+        - [Haskellwiki. Category Theory](https://en.wikibooks.org/wiki/Haskell/Category_theory)
+        - [Haskellforall. The Category Design Pattern](https://www.haskellforall.com/2012/08/the-category-design-pattern.html)
+        - [Haskellforall. The Functor Design Pattern](https://www.haskellforall.com/2012/09/the-functor-design-pattern.html)
+
+            /// attention | ATTENTION 
+            The functor design pattern doesn't work at all if you aren't using categories in the first place. This is why you should structure your tools using the compositional category design pattern so that you can take advantage of functors to easily mix your tools together. 
+            ///
+
+        - [Haskellwiki. Functor](https://wiki.haskell.org/index.php?title=Functor)
+        - [Haskellwiki. Typeclassopedia#Functor](https://wiki.haskell.org/index.php?title=Typeclassopedia#Functor)
+        - [Haskellwiki. Typeclassopedia#Category](https://wiki.haskell.org/index.php?title=Typeclassopedia#Category)
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    from abc import abstractmethod, ABC
+    return ABC, abstractmethod
+
+
+@app.cell(hide_code=True)
+def _():
+    from dataclasses import dataclass
+    from typing import Callable, Generic, TypeVar
+    from pprint import pp
+    return Callable, Generic, TypeVar, dataclass, pp
+
+
+@app.cell(hide_code=True)
+def _(TypeVar):
+    A = TypeVar("A")
+    B = TypeVar("B")
+    C = TypeVar("C")
+    return A, B, C
+
+
+if __name__ == "__main__":
+    app.run()

functional_programming/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog of the functional-programming course
+
+## 2025-03-11
+
+* Demo version of notebook `05_functors.py`
+
+## 2025-03-13
+
+* `0.1.0` version of notebook `05_functors`
+
+Thank [Akshay](https://github.com/akshayka) and [Haleshot](https://github.com/Haleshot) for reviewing
+
+## 2025-03-16
+
++ Use uppercased letters for `Generic` types, e.g. `A = TypeVar("A")`
++ Refactor the `Functor` class, changing `fmap` and utility methods to `classmethod`
+
+    For example:
+
+    ```python
+    @dataclass
+    class Wrapper(Functor, Generic[A]):
+        value: A
+
+        @classmethod
+        def fmap(cls, f: Callable[[A], B], a: "Wrapper[A]") -> "Wrapper[B]":
+            return Wrapper(f(a.value))
+
+    >>> Wrapper.fmap(lambda x: x + 1, wrapper)
+    Wrapper(value=2)
+    ```
+
++ Move the `check_functor_law` method from `Functor` class to a standard function
+- Rename `ListWrapper` to `List` for simplicity
+- Remove the `Just` class
++ Rewrite proofs

functional_programming/README.md

@@ -0,0 +1,61 @@
+# Learn Functional Programming
+
+_🚧 This collection is a
+[work in progress](https://github.com/marimo-team/learn/issues/51)._
+
+This series of marimo notebooks introduces the powerful paradigm of functional
+programming through Python. Taking inspiration from Haskell and Category Theory,
+we'll build a strong foundation in FP concepts that can transform how you
+approach software development.
+
+## What You'll Learn
+
+**Using only Python's standard library**, we'll construct functional programming
+concepts from first principles.
+
+Topics include:
+
+-   Recursion and higher-order functions
+-   Category theory fundamentals
+-   Functors, applicatives, and monads
+-   Composable abstractions for robust code
+
+## Timeline & Collaboration
+
+I'm currently studying functional programming and Haskell, estimating about 2
+months or even longer to complete this series. The structure may evolve as the
+project develops.
+
+If you're interested in collaborating or have questions, please reach out to me
+on Discord (@eugene.hs).
+
+**Running notebooks.** To run a notebook locally, use
+
+```bash
+uvx marimo edit <URL>
+```
+
+For example, run the `Functor` tutorial with
+
+```bash
+uvx marimo edit https://github.com/marimo-team/learn/blob/main/Functional_programming/05_functors.py
+```
+
+You can also open notebooks in our online playground by appending `marimo.app/`
+to a notebook's URL:
+[marimo.app/github.com/marimo-team/learn/blob/main/functional_programming/05_functors.py](https://marimo.app/https://github.com/marimo-team/learn/blob/main/functional_programming/05_functors.py).
+
+# Description of notebooks
+
+Check [here](https://github.com/marimo-team/learn/issues/51) for current series
+structure.
+
+| Notebook                                                                                                          | Description                                                                                                                                                                                            | References                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| ----------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [05. Category and Functors](https://github.com/marimo-team/learn/blob/main/Functional_programming/05_functors.py) | Learn why `len` is a _Functor_ from `list concatenation` to `integer addition`, how to _lift_ an ordinary function into a _computation context_, and how to write an _adapter_ between two categories. | - [The Trivial Monad](http://blog.sigfpe.com/2007/04/trivial-monad.html) <br> - [Haskellwiki. Category Theory](https://en.wikibooks.org/wiki/Haskell/Category_theory) <br> - [Haskellforall. The Category Design Pattern](https://www.haskellforall.com/2012/08/the-category-design-pattern.html) <br> - [Haskellforall. The Functor Design Pattern](https://www.haskellforall.com/2012/09/the-functor-design-pattern.html) <br> - [Haskellwiki. Functor](https://wiki.haskell.org/index.php?title=Functor) <br> - [Haskellwiki. Typeclassopedia#Functor](https://wiki.haskell.org/index.php?title=Typeclassopedia#Functor) <br> - [Haskellwiki. Typeclassopedia#Category](https://wiki.haskell.org/index.php?title=Typeclassopedia#Category) |
+
+**Authors.**
+
+Thanks to all our notebook authors!
+
+-   [métaboulie](https://github.com/metaboulie)

optimization/05_portfolio_optimization.py

@@ -47,7 +47,7 @@ def _(mo):
         r"""
         ## Asset returns and risk
 
-        We will only model investments held for one period. The initial prices are $p_i > 0$. The end of period prices are $p_i^+ >0$. The asset (fractional) returns are $r_i = (p_i^+-p_i)/p_i$. The porfolio (fractional) return is $R = r^Tw$.
+        We will only model investments held for one period. The initial prices are $p_i > 0$. The end of period prices are $p_i^+ >0$. The asset (fractional) returns are $r_i = (p_i^+-p_i)/p_i$. The portfolio (fractional) return is $R = r^Tw$.
 
         A common model is that $r$ is a random variable with mean ${\bf E}r = \mu$ and covariance ${\bf E{(r-\mu)(r-\mu)^T}} = \Sigma$.
         It follows that $R$ is a random variable with ${\bf E}R = \mu^T w$ and ${\bf var}(R) = w^T\Sigma w$. In real-world applications, $\mu$ and $\Sigma$ are estimated from data and models, and $w$ is chosen using a library like CVXPY.

polars/04_basic_operations.py

@@ -0,0 +1,631 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "marimo",
+#     "polars==1.23.0",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.13"
+app = marimo.App(width="medium")
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Basic operations on data
+        _By [Joram Mutenge](https://www.udemy.com/user/joram-mutenge/)._
+
+        In this notebook, you'll learn how to perform arithmetic operations, comparisons, and conditionals on a Polars dataframe. We'll work with a DataFrame that tracks software usage by year, categorized as either Vintage (old) or Modern (new).
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    import polars as pl
+
+    df = pl.DataFrame(
+        {
+            "software": [
+                "Lotus-123",
+                "WordStar",
+                "dBase III",
+                "VisiCalc",
+                "WinZip",
+                "MS-DOS",
+                "HyperCard",
+                "WordPerfect",
+                "Excel",
+                "Photoshop",
+                "Visual Studio",
+                "Slack",
+                "Zoom",
+                "Notion",
+                "Figma",
+                "Spotify",
+                "VSCode",
+                "Docker",
+            ],
+            "users": [
+                10000,
+                4500,
+                2500,
+                3000,
+                1800,
+                17000,
+                2200,
+                1900,
+                500000,
+                12000000,
+                1500000,
+                3000000,
+                4000000,
+                2000000,
+                2500000,
+                4500000,
+                6000000,
+                3500000,
+            ],
+            "category": ["Vintage"] * 8 + ["Modern"] * 10,
+            "year": [
+                1985,
+                1980,
+                1984,
+                1979,
+                1991,
+                1981,
+                1987,
+                1982,
+                1987,
+                1990,
+                1997,
+                2013,
+                2011,
+                2016,
+                2016,
+                2008,
+                2015,
+                2013,
+            ],
+        }
+    )
+
+    df
+    return df, pl
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Arithmetic
+        ### Addition
+        Let's add 42 users to each piece of software. This means adding 42 to each value under **users**.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users") + 42)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Another way to perform the above operation is using the built-in function.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users").add(42))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Subtraction
+        Let's subtract 42 users to each piece of software.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users") - 42)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Alternatively, you could subtract like this:""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users").sub(42))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Division
+        Suppose the **users** values are inflated, we can reduce them by dividing by 1000. Here's how to do it.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users") / 1000)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Or we could do it with a built-in expression.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users").truediv(1000))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""If we didn't care about the remainder after division (i.e remove numbers after decimal point) we could do it like this.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users").floordiv(1000))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Multiplication
+        Let's pretend the *user* values are deflated and increase them by multiplying by 100.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df.with_columns(pl.col("users") * 100))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Polars also has a built-in function for multiplication.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns(pl.col("users").mul(100))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""So far, we've only modified the values in an existing column. Let's create a column **decade** that will represent the years as decades. Thus 1985 will be 1980 and 2008 will be 2000.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df.with_columns(decade=pl.col("year").floordiv(10).mul(10)))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We could create a new column another way as follows:""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    df.with_columns((pl.col("year").floordiv(10).mul(10)).alias("decade"))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        **Tip**  
+        Polars encounrages you to perform your operations as a chain. This enables you to take advantage of the query optimizer. We'll build upon the above code as a chain.
+
+        ## Comparison
+        ### Equal
+        Let's get all the software categorized as Vintage.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (
+        df.with_columns(decade=pl.col("year").floordiv(10).mul(10))
+        .filter(pl.col("category") == "Vintage")
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We could also do a double comparison. VisiCal is the only software that's vintage and in the decade 1970s. Let's perform this comparison operation.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (
+        df.with_columns(decade=pl.col("year").floordiv(10).mul(10))
+        .filter(pl.col("category") == "Vintage")
+        .filter(pl.col("decade") == 1970)
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        We could also do this comparison in one line, if readability is not a concern
+
+        **Notice** that we must enclose the two expressions between the `&` with parenthesis.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (
+        df.with_columns(decade=pl.col("year").floordiv(10).mul(10))
+        .filter((pl.col("category") == "Vintage") & (pl.col("decade") == 1970))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We can also use the built-in function for equal to comparisons.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(pl.col('category').eq('Vintage'))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Not equal
+        We can also compare if something is `not` equal to something. In this case, category is not vintage.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(pl.col('category') != 'Vintage')
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Or with the built-in function.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(pl.col('category').ne('Vintage'))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Or if you want to be extra clever, you can use the negation symbol `~` used in logic.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(~pl.col('category').eq('Vintage'))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Greater than
+        Let's get the software where the year is greater than 2008 from the above dataframe.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(~pl.col('category').eq('Vintage'))
+     .filter(pl.col('year') > 2008)
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Or if we wanted the year 2008 to be included, we could use great or equal to.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(~pl.col('category').eq('Vintage'))
+     .filter(pl.col('year') >= 2008)
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We could do the previous two operations with built-in functions. Here's with greater than.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(~pl.col('category').eq('Vintage'))
+     .filter(pl.col('year').gt(2008))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""And here's with greater or equal to""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(~pl.col('category').eq('Vintage'))
+     .filter(pl.col('year').ge(2008))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        **Note**: For "less than", and "less or equal to" you can use the operators `<` or `<=`. Alternatively, you can use built-in functions `lt` or `le` respectively.
+
+        ### Is between
+        Polars also allows us to filter between a range of values. Let's get the modern software were the year is between 2013 and 2016. This is inclusive on both ends (i.e. both years are part of the result).
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(pl.col('category').eq('Modern'))
+     .filter(pl.col('year').is_between(2013, 2016))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Or operator
+        If we only want either one of the conditions in the comparison to be met, we could use `|`, which is the `or` operator.
+
+        Let's get software that is either modern or used in the decade 1980s.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter((pl.col('category') == 'Modern') | (pl.col('decade') == 1980))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Conditionals
+        Polars also allows you create new columns based on a condition. Let's create a column *status* that will indicate if the software is "discontinued" or "in use".
+
+        Here's a list of products that are no longer in use.
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    discontinued_list = ['Lotus-123', 'WordStar', 'dBase III', 'VisiCalc', 'MS-DOS', 'HyperCard']
+    return (discontinued_list,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Here's how we can get a dataframe of the products that are discontinued.""")
+    return
+
+
+@app.cell
+def _(df, discontinued_list, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .filter(pl.col('software').is_in(discontinued_list))
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Now, let's create the **status** column.""")
+    return
+
+
+@app.cell
+def _(df, discontinued_list, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .with_columns(pl.when(pl.col('software').is_in(discontinued_list))
+                   .then(pl.lit('Discontinued'))
+                   .otherwise(pl.lit('In use'))
+                   .alias('status')
+                   )
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Unique counts
+        Sometimes you may want to see only the unique values in a column. Let's check the unique decades we have in our DataFrame.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, discontinued_list, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .with_columns(pl.when(pl.col('software').is_in(discontinued_list))
+                   .then(pl.lit('Discontinued'))
+                   .otherwise(pl.lit('In use'))
+                   .alias('status')
+                   )
+     .select('decade').unique()
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Finally, let's find out the number of software used in each decade.""")
+    return
+
+
+@app.cell
+def _(df, discontinued_list, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .with_columns(pl.when(pl.col('software').is_in(discontinued_list))
+                   .then(pl.lit('Discontinued'))
+                   .otherwise(pl.lit('In use'))
+                   .alias('status')
+                   )
+     ['decade'].value_counts()
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We could also rewrite the above code as follows:""")
+    return
+
+
+@app.cell
+def _(df, discontinued_list, pl):
+    (df
+     .with_columns(decade=pl.col('year').floordiv(10).mul(10))
+     .with_columns(pl.when(pl.col('software').is_in(discontinued_list))
+                   .then(pl.lit('Discontinued'))
+                   .otherwise(pl.lit('In use'))
+                   .alias('status')
+                   )
+     .select('decade').to_series().value_counts()
+     )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Hopefully, we've picked your interest to try out Polars the next time you analyze your data.""")
+    return
+
+
+@app.cell
+def _():
+    return
+
+
+if __name__ == "__main__":
+    app.run()

polars/10_strings.py

@@ -0,0 +1,1004 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "altair==5.5.0",
+#     "marimo",
+#     "numpy==2.2.3",
+#     "polars==1.24.0",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.17"
+app = marimo.App(width="medium")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Strings
+
+        _By [Péter Ferenc Gyarmati](http://github.com/peter-gy)_.
+
+        In this chapter we're going to dig into string manipulation. For a fun twist, we'll be mostly playing around with a dataset that every Polars user has bumped into without really thinking about it—the source code of the `polars` module itself. More precisely, we'll use a dataframe that pulls together all the Polars expressions and their docstrings, giving us a cool, hands-on way to explore the expression API in a truly data-driven manner.
+
+        We'll cover parsing, length calculation, case conversion, and much more, with practical examples and visualizations. Finally, we will combine various techniques you learned in prior chapters to build a fully interactive playground in which you can execute the official code examples of Polars expressions.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🛠️  Parsing & Conversion
+
+        Let's warm up with one of the most frequent use cases: parsing raw strings into various formats.
+        We'll take a tiny dataframe with metadata about Python packages represented as raw JSON strings and we'll use Polars string expressions to parse the attributes into their true data types.
+        """
+    )
+    return
+
+
+@app.cell
+def _(pl):
+    pip_metadata_raw_df = pl.DataFrame(
+        [
+            '{"package": "polars", "version": "1.24.0", "released_at": "2025-03-02T20:31:12+0000", "size_mb": "30.9"}',
+            '{"package": "marimo", "version": "0.11.14", "released_at": "2025-03-04T00:28:57+0000", "size_mb": "10.7"}',
+        ],
+        schema={"raw_json": pl.String},
+    )
+    pip_metadata_raw_df
+    return (pip_metadata_raw_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We can use the [`json_decode`](https://docs.pola.rs/api/python/stable/reference/series/api/polars.Series.str.json_decode.html) expression to parse the raw JSON strings into Polars-native structs and we can use the [unnest](https://docs.pola.rs/api/python/stable/reference/dataframe/api/polars.DataFrame.unnest.html) dataframe operation to have a dedicated column per parsed attribute.""")
+    return
+
+
+@app.cell
+def _(pip_metadata_raw_df, pl):
+    pip_metadata_df = pip_metadata_raw_df.select(json=pl.col('raw_json').str.json_decode()).unnest('json')
+    pip_metadata_df
+    return (pip_metadata_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""This is already a much friendlier representation of the data we started out with, but note that since the JSON entries had only string attributes, all values are strings, even the temporal `released_at` and numerical `size_mb` columns.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""As we know that the `size_mb` column should have a decimal representation, we go ahead and use [`to_decimal`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_decimal.html#polars.Expr.str.to_decimal) to perform the conversion.""")
+    return
+
+
+@app.cell
+def _(pip_metadata_df, pl):
+    pip_metadata_df.select(
+        'package',
+        'version',
+        pl.col('size_mb').str.to_decimal(),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Moving on to the `released_at` attribute which indicates the exact time when a given Python package got released, we have a bit more options to consider. We can convert to `Date`, `DateTime`, and `Time` types based on the desired temporal granularity. The [`to_date`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_date.html), [`to_datetime`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_datetime.html), and [`to_time`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_time.html) expressions are here to help us with the conversion, all we need is to provide the desired format string.
+
+        Since Polars uses Rust under the hood to implement all its expressions, we need to consult the [`chrono::format`](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) reference to come up with appropriate format strings.
+
+        Here's a quick reference:
+
+        | Specifier | Meaning            |
+        |-----------|--------------------|
+        | `%Y`      | Year (e.g., 2025) |
+        | `%m`      | Month (01-12)     |
+        | `%d`      | Day (01-31)       |
+        | `%H`      | Hour (00-23)      |
+        | `%z`      | UTC offset        |
+
+        The raw strings we are working with look like `"2025-03-02T20:31:12+0000"`. We can match this using the `"%Y-%m-%dT%H:%M:%S%z"` format string.
+        """
+    )
+    return
+
+
+@app.cell
+def _(pip_metadata_df, pl):
+    pip_metadata_df.select(
+        'package',
+        'version',
+        release_date=pl.col('released_at').str.to_date('%Y-%m-%dT%H:%M:%S%z'),
+        release_datetime=pl.col('released_at').str.to_datetime('%Y-%m-%dT%H:%M:%S%z'),
+        release_time=pl.col('released_at').str.to_time('%Y-%m-%dT%H:%M:%S%z'),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Alternatively, instead of using three different functions to perform the conversion to date, we can use a single one, [`strptime`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strptime.html) which takes the desired temporal data type as its first parameter.""")
+    return
+
+
+@app.cell
+def _(pip_metadata_df, pl):
+    pip_metadata_df.select(
+        'package',
+        'version',
+        release_date=pl.col('released_at').str.strptime(pl.Date, '%Y-%m-%dT%H:%M:%S%z'),
+        release_datetime=pl.col('released_at').str.strptime(pl.Datetime, '%Y-%m-%dT%H:%M:%S%z'),
+        release_time=pl.col('released_at').str.strptime(pl.Time, '%Y-%m-%dT%H:%M:%S%z'),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""And to wrap up this section on parsing and conversion, let's consider a final scenario. What if we don't want to parse the entire raw JSON string, because we only need a subset of its attributes? Well, in this case we can leverage the [`json_path_match`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.json_path_match.html) expression to extract only the desired attributes using standard [JSONPath](https://goessner.net/articles/JsonPath/) syntax.""")
+    return
+
+
+@app.cell
+def _(pip_metadata_raw_df, pl):
+    pip_metadata_raw_df.select(
+        package=pl.col("raw_json").str.json_path_match("$.package"),
+        version=pl.col("raw_json").str.json_path_match("$.version"),
+        release_date=pl.col("raw_json")
+        .str.json_path_match("$.size_mb")
+        .str.to_decimal(),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 📊 Dataset Overview
+
+        Now that we got our hands dirty, let's consider a somewhat wilder dataset for the subsequent sections: a dataframe of metadata about every single expression in your current Polars module.
+
+        At the risk of stating the obvious, in the previous section, when we typed `pl.col('raw_json').str.json_decode()`, we accessed the `json_decode` member of the `str` expression namespace through the `pl.col('raw_json')` expression *instance*. Under the hood, deep inside the Polars source code, there is a corresponding `def json_decode(...)` method with a carefully authored docstring explaining the purpose and signature of the member.
+
+        Since Python makes module introspection simple, we can easily enumerate all Polars expressions and organize their metadata in `expressions_df`, to be used for all the upcoming string manipulation examples.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(pl):
+    def list_members(expr, namespace) -> list[dict]:
+        """Iterates through the attributes of `expr` and returns their metadata"""
+        members = []
+        for attrname in expr.__dir__():
+            is_namespace = attrname in pl.Expr._accessors
+            is_private = attrname.startswith("_")
+            if is_namespace or is_private:
+                continue
+
+            attr = getattr(expr, attrname)
+            members.append(
+                {
+                    "namespace": namespace,
+                    "member": attrname,
+                    "docstring": attr.__doc__,
+                }
+            )
+        return members
+
+
+    def list_expr_meta() -> list[dict]:
+        # Dummy expression instance to 'crawl'
+        expr = pl.lit("")
+        root_members = list_members(expr, "root")
+        namespaced_members: list[list[dict]] = [
+            list_members(getattr(expr, namespace), namespace)
+            for namespace in pl.Expr._accessors
+        ]
+        return sum(namespaced_members, root_members)
+
+
+    expressions_df = pl.from_dicts(list_expr_meta(), infer_schema_length=None).sort('namespace', 'member')
+    expressions_df
+    return expressions_df, list_expr_meta, list_members
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""As the following visualization shows, `str` is one of the richest Polars expression namespaces with multiple dozens of functions in it.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(alt, expressions_df):
+    expressions_df.plot.bar(
+        x=alt.X("count(member):Q", title='Count of Expressions'),
+        y=alt.Y("namespace:N", title='Namespace').sort("-x"),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 📏 Length Calculation
+
+        A common use case is to compute the length of a string. Most people associate string length exclusively with the number of characters the said string consists of; however, in certain scenarios it is useful to also know how much memory is required for storing, so how many bytes are required to represent the textual data.
+
+        The expressions [`len_chars`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.len_chars.html) and [`len_bytes`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.len_bytes.html) are here to help us with these calculations.
+
+        Below, we compute `docstring_len_chars` and `docstring_len_bytes` columns to see how many characters and bytes the documentation of each expression is made up of.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    docstring_length_df = expressions_df.select(
+        'namespace',
+        'member',
+        docstring_len_chars=pl.col("docstring").str.len_chars(),
+        docstring_len_bytes=pl.col("docstring").str.len_bytes(),
+    )
+    docstring_length_df
+    return (docstring_length_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""As the dataframe preview above and the scatterplot below show, the docstring length measured in bytes is almost always bigger than the length expressed in characters. This is due to the fact that the docstrings include characters which require more than a single byte to represent, such as "╞" for displaying dataframe header and body separators.""")
+    return
+
+
+@app.cell
+def _(alt, docstring_length_df):
+    docstring_length_df.plot.point(
+        x=alt.X('docstring_len_chars', title='Docstring Length (Chars)'),
+        y=alt.Y('docstring_len_bytes', title='Docstring Length (Bytes)'),
+        tooltip=['namespace', 'member', 'docstring_len_chars', 'docstring_len_bytes'],
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔠 Case Conversion
+
+        Another frequent string transformation is lowercasing, uppercasing, and titlecasing. We can use [`to_lowercase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_lowercase.html), [`to_uppercase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_lowercase.html) and [`to_titlecase`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.to_titlecase.html) for doing so.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        member_lower=pl.col('member').str.to_lowercase(),
+        member_upper=pl.col('member').str.to_uppercase(),
+        member_title=pl.col('member').str.to_titlecase(),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ➕ Padding
+
+        Sometimes we need to ensure that strings have a fixed-size character length. [`pad_start`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.pad_start.html) and [`pad_end`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.pad_end.html) can be used to fill the "front" or "back" of a string with a supplied character, while [`zfill`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.zfill.html) is a utility for padding the start of a string with `"0"` until it reaches a particular length. In other words, `zfill` is a more specific version of `pad_start`, where the `fill_char` parameter is explicitly set to `"0"`.
+
+        In the example below we take the unique Polars expression namespaces and pad them so that they have a uniform length which you can control via a slider.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    padding = mo.ui.slider(0, 16, step=1, value=8, label="Padding Size")
+    return (padding,)
+
+
+@app.cell
+def _(expressions_df, padding, pl):
+    padded_df = expressions_df.select("namespace").unique().select(
+        "namespace",
+        namespace_front_padded=pl.col("namespace").str.pad_start(padding.value, "_"),
+        namespace_back_padded=pl.col("namespace").str.pad_end(padding.value, "_"),
+        namespace_zfilled=pl.col("namespace").str.zfill(padding.value),
+    )
+    return (padded_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo, padded_df, padding):
+    mo.vstack([
+        padding,
+        padded_df,
+    ])
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔄 Replacing
+
+        Let's say we want to convert from `snake_case` API member names to `kebab-case`, that is, we need to replace the underscore character with a hyphen. For operations like that, we can use [`replace`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace.html) and [`replace_all`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace_all.html).
+
+        As the example below demonstrates, `replace` stops after the first occurrence of the to-be-replaced pattern, while `replace_all` goes all the way through and changes all underscores to hyphens resulting in the `kebab-case` representation we were looking for.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "member",
+        member_kebab_case_partial=pl.col("member").str.replace("_", "-"),
+        member_kebab_case=pl.col("member").str.replace_all("_", "-"),
+    ).sort(pl.col("member").str.len_chars(), descending=True)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        A related expression is [`replace_many`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.replace_many.html), which accepts *many* pairs of to-be-matched patterns and corresponding replacements and uses the [Aho–Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) to carry out the operation with great performance.
+
+        In the example below we replace all instances of `"min"` with `"minimum"` and `"max"` with `"maximum"` using a single expression.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "member",
+        member_modified=pl.col("member").str.replace_many(
+            {
+                "min": "minimum",
+                "max": "maximum",
+            }
+        ),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔍 Searching & Matching
+
+        A common need when working with strings is to determine whether their content satisfies some condition: whether it starts or ends with a particular substring or contains a certain pattern.
+
+        Let's suppose we want to determine whether a member of the Polars expression API is a "converter", such as `to_decimal`, identified by its `"to_"` prefix. We can use [`starts_with`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.starts_with.html) to perform this check.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "namespace",
+        "member",
+        is_converter=pl.col("member").str.starts_with("to_"),
+    ).sort(-pl.col("is_converter").cast(pl.Int8))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Throughout this course as you have gained familiarity with the expression API you might have noticed that some members end with an underscore such as `or_`, since their "body" is a reserved Python keyword.
+
+        Let's use [`ends_with`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.ends_with.html) to find all the members which are named after such keywords.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "namespace",
+        "member",
+        is_escaped_keyword=pl.col("member").str.ends_with("_"),
+    ).sort(-pl.col("is_escaped_keyword").cast(pl.Int8))
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Now let's move on to analyzing the docstrings in a bit more detail. Based on their content we can determine whether a member is deprecated, accepts parameters, comes with examples, or references external URL(s) & related members.
+
+        As demonstrated below, we can compute all these boolean attributes using [`contains`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.contains.html) to check whether the docstring includes a particular substring.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'namespace',
+        'member',
+        is_deprecated=pl.col('docstring').str.contains('.. deprecated', literal=True),
+        has_parameters=pl.col('docstring').str.contains('Parameters'),
+        has_examples=pl.col('docstring').str.contains('Examples'),
+        has_related_members=pl.col('docstring').str.contains('See Also'),
+        has_url=pl.col('docstring').str.contains('https?://'),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""For scenarios where we want to combine multiple substrings to check for, we can use the [`contains`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.contains.html) expression to check for the presence of various patterns.""")
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'namespace',
+        'member',
+        has_reference=pl.col('docstring').str.contains_any(['See Also', 'https://'])
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        From the above analysis we could see that almost all the members come with code examples. It would be interesting to know how many variable assignments are going on within each of these examples, right? That's not as simple as checking for a pre-defined literal string containment though, because variables can have arbitrary names - any valid Python identifier is allowed. While the `contains` function supports checking for regular expressions instead of literal strings too, it would not suffice for this exercise because it only tells us whether there is at least a single occurrence of the sought pattern rather than telling us the exact number of matches.
+
+        Fortunately, we can take advantage of [`count_matches`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.count_matches.html) to achieve exactly what we want. We specify the regular expression `r'[a-zA-Z_][a-zA-Z0-9_]* = '` according to the [`regex` Rust crate](https://docs.rs/regex/latest/regex/) to match Python identifiers and we leave the rest to Polars.
+
+        In `count_matches(r'[a-zA-Z_][a-zA-Z0-9_]* = ')`:
+
+        - `[a-zA-Z_]` matches a letter or underscore (start of a Python identifier).
+        - `[a-zA-Z0-9_]*` matches zero or more letters, digits, or underscores.
+        - ` = ` matches a space, equals sign, and space (indicating assignment).
+
+        This finds variable assignments like `x = ` or `df_result = ` in docstrings.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'namespace',
+        'member',
+        variable_assignment_count=pl.col('docstring').str.count_matches(r'[a-zA-Z_][a-zA-Z0-9_]* = '),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""A related application example is to *find* the first index where a particular pattern is present, so that it can be used for downstream processing such as slicing. Below we use the [`find`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.find.html) expression to determine the index at which a code example starts in the docstring - identified by the Python shell substring `">>>"`.""")
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'namespace',
+        'member',
+        code_example_start=pl.col('docstring').str.find('>>>'),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ✂️ Slicing and Substrings
+
+        Sometimes we are only interested in a particular substring. We can use [`head`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.head.html), [`tail`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.tail.html) and [`slice`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.slice.html) to extract a substring from the start, end, or between arbitrary indices.
+        """
+    )
+    return
+
+
+@app.cell
+def _(mo):
+    slice = mo.ui.slider(1, 50, step=1, value=25, label="Slice Size")
+    return (slice,)
+
+
+@app.cell
+def _(expressions_df, pl, slice):
+    sliced_df = expressions_df.select(
+        # First 25 chars
+        docstring_head=pl.col("docstring").str.head(slice.value),
+        # 50 chars after the first 25 chars
+        docstring_slice=pl.col("docstring").str.slice(slice.value, 2*slice.value),
+        # Last 25 chars
+        docstring_tail=pl.col("docstring").str.tail(slice.value),
+    )
+    return (sliced_df,)
+
+
+@app.cell
+def _(mo, slice, sliced_df):
+    mo.vstack([
+        slice,
+        sliced_df,
+    ])
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ➗ Splitting
+
+        Certain strings follow a well-defined structure and we might be only interested in some parts of them. For example, when dealing with `snake_cased_expression` member names we might be curious to get only the first, second, or $n^{\text{th}}$ word before an underscore. We would need to *split* the string at a particular pattern for downstream processing.
+
+        The [`split`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.split.html), [`split_exact`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.split_exact.html) and [`splitn`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.splitn.html) expressions enable us to achieve this.
+
+        The primary difference between these string splitting utilities is that `split` produces a list of variadic length based on the number of resulting segments, `splitn` returns a struct with at least `0` and at most `n` fields while `split_exact` returns a struct of exactly `n` fields.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'member',
+        member_name_parts=pl.col('member').str.split('_'),
+        member_name_parts_n=pl.col('member').str.splitn('_', n=2),
+        member_name_parts_exact=pl.col('member').str.split_exact('_', n=2),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""As a more practical example, we can use the `split` expression with some aggregation to count the number of times a particular word occurs in member names across all namespaces. This enables us to create a word cloud of the API members' constituents!""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, wordcloud, wordcloud_height, wordcloud_width):
+    mo.vstack([
+        wordcloud_width,
+        wordcloud_height,
+        wordcloud,
+    ])
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    wordcloud_width = mo.ui.slider(0, 64, step=1, value=32, label="Word Cloud Width")
+    wordcloud_height = mo.ui.slider(0, 32, step=1, value=16, label="Word Cloud Height")
+    return wordcloud_height, wordcloud_width
+
+
+@app.cell(hide_code=True)
+def _(alt, expressions_df, pl, random, wordcloud_height, wordcloud_width):
+    wordcloud_df = (
+        expressions_df.select(pl.col("member").str.split("_"))
+        .explode("member")
+        .group_by("member")
+        .agg(pl.len())
+        # Generating random x and y coordinates to distribute the words in the 2D space
+        .with_columns(
+            x=pl.col("member").map_elements(
+                lambda e: random.randint(0, wordcloud_width.value),
+                return_dtype=pl.UInt8,
+            ),
+            y=pl.col("member").map_elements(
+                lambda e: random.randint(0, wordcloud_height.value),
+                return_dtype=pl.UInt8,
+            ),
+        )
+    )
+
+    wordcloud = alt.Chart(wordcloud_df).mark_text(baseline="middle").encode(
+        x=alt.X("x:O", axis=None),
+        y=alt.Y("y:O", axis=None),
+        text="member:N",
+        color=alt.Color("len:Q", scale=alt.Scale(scheme="bluepurple")),
+        size=alt.Size("len:Q", legend=None),
+        tooltip=["member", "len"],
+    ).configure_view(strokeWidth=0)
+    return wordcloud, wordcloud_df
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔗 Concatenation & Joining
+
+        Often we would like to create longer strings from strings we already have. We might want to create a formatted, sentence-like string or join multiple existing strings in our dataframe into a single one.
+
+        The top-level [`concat_str`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.concat_str.html) expression enables us to combine strings *horizontally* in a dataframe. As the example below shows, we can take the `member` and `namespace` column of each row and construct a `description` column in which each row will correspond to the value ``f"- Expression `{member}` belongs to namespace `{namespace}`"``.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    descriptions_df = expressions_df.sample(5).select(
+        description=pl.concat_str(
+            [
+                pl.lit("- Expression "),
+                pl.lit("`"),
+                "member",
+                pl.lit("`"),
+                pl.lit(" belongs to namespace "),
+                pl.lit("`"),
+                "namespace",
+                pl.lit("`"),
+            ],
+        )
+    )
+    descriptions_df
+    return (descriptions_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Now that we have constructed these bullet points through *horizontal* concatenation of strings, we can perform a *vertical* one so that we end up with a single string in which we have a bullet point on each line.
+
+        We will use the [`join`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.join.html) expression to do so.
+        """
+    )
+    return
+
+
+@app.cell
+def _(descriptions_df, pl):
+    descriptions_df.select(pl.col('description').str.join('\n'))
+    return
+
+
+@app.cell(hide_code=True)
+def _(descriptions_df, mo, pl):
+    mo.md(f"""In fact, since the string we constructed dynamically is valid markdown, we can display it dynamically using Marimo's `mo.md` utility!
+
+    ---
+
+    {descriptions_df.select(pl.col('description').str.join('\n')).to_numpy().squeeze().tolist()}
+    """)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔍 Pattern-based Extraction
+
+        In the vast majority of the cases, when dealing with unstructured text data, all we really want is to extract something structured from it. A common use case is to extract URLs from text to get a better understanding of related content.
+
+        In the example below that's exactly what we do. We scan the `docstring` of each API member and extract URLs from them using [`extract`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract.html) and [`extract_all`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract_all.html) using a simple regular expression to match http and https URLs.
+
+        Note that `extract` stops after a first match and returns a scalar result (or `null` if there was no match) while `extract_all` returns a - potentially empty - list of matches.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    url_pattern = r'(https?://[^\s>]+)'
+    expressions_df.select(
+        'namespace',
+        'member',
+        url_match=pl.col('docstring').str.extract(url_pattern),
+        url_matches=pl.col('docstring').str.extract_all(url_pattern),
+    ).filter(pl.col('url_match').is_not_null())
+    return (url_pattern,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Note that in each `docstring` where a code example involving dataframes is present, we will see an output such as "shape: (5, 2)" indicating the number of rows and columns of the dataframe produced by the sample code. Let's say we would like to *capture* this information in a structured way.
+
+        [`extract_groups`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.extract_groups.html) is a really powerful expression allowing us to achieve exactly that.
+
+        Below we define the regular expression `r"shape:\s*\((?<height>\S+),\s*(?<width>\S+)\)"` with two capture groups, named `height` and `width` and pass it as the parameter of `extract_groups`. After execution, for each `docstring`, we end up with fully structured data we can further process downstream!
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        'namespace',
+        'member',
+        example_df_shape=pl.col('docstring').str.extract_groups(r"shape:\s*\((?<height>\S+),\s*(?<width>\S+)\)"),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🧹 Stripping
+
+        Strings might require some cleaning before further processing, such as the removal of some characters from the beginning or end of the text. [`strip_chars_start`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars_start.html), [`strip_chars_end`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars_end.html) and [`strip_chars`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_chars.html) are here to facilitate this.
+
+        All we need to do is to specify a set of characters we would like to get rid of and Polars handles the rest for us.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "member",
+        member_front_stripped=pl.col("member").str.strip_chars_start("a"),
+        member_back_stripped=pl.col("member").str.strip_chars_end("n"),
+        member_fully_stripped=pl.col("member").str.strip_chars("na"),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Note that when using the above expressions, the specified characters do not need to form a sequence; they are handled as a set. However, in certain use cases we only want to strip complete substrings, so we would need our input to be strictly treated as a sequence rather than as a set.
+
+        That's exactly the rationale behind [`strip_prefix`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_prefix.html) and [`strip_suffix`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.strip_suffix.html).
+
+        Below we use these to remove the `"to_"` prefixes and `"_with"` suffixes from each member name.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    expressions_df.select(
+        "member",
+        member_prefix_stripped=pl.col("member").str.strip_prefix("to_"),
+        member_suffix_stripped=pl.col("member").str.strip_suffix("_with"),
+    ).slice(20)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔑 Encoding & Decoding
+
+        Should you find yourself in the need of encoding your strings into [base64](https://en.wikipedia.org/wiki/Base64) or [hexadecimal](https://en.wikipedia.org/wiki/Hexadecimal) format, then Polars has your back with its [`encode`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.encode.html) expression.
+        """
+    )
+    return
+
+
+@app.cell
+def _(expressions_df, pl):
+    encoded_df = expressions_df.select(
+        "member",
+        member_base64=pl.col('member').str.encode('base64'),
+        member_hex=pl.col('member').str.encode('hex'),
+    )
+    encoded_df
+    return (encoded_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""And of course, you can convert back into a human-readable representation using the [`decode`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.str.decode.html) expression.""")
+    return
+
+
+@app.cell
+def _(encoded_df, pl):
+    encoded_df.with_columns(
+        member_base64_decoded=pl.col('member_base64').str.decode('base64').cast(pl.String),
+        member_hex_decoded=pl.col('member_hex').str.decode('hex').cast(pl.String),
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🚀 Application: Dynamic Execution of Polars Examples
+
+        Now that we are familiar with string expressions, we can combine them with other Polars operations to build a fully interactive playground where code examples of Polars expressions can be explored.
+
+        We make use of string expressions to extract the raw Python source code of examples from the docstrings and we leverage the interactive Marimo environment to enable the selection of expressions via a searchable dropdown and a fully functional code editor whose output is rendered with Marimo's rich display utilities.
+
+        In other words, we will use Polars to execute Polars. ❄️ How cool is that?
+
+        ---
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(
+    example_editor,
+    execution_result,
+    expression,
+    expression_description,
+    expression_docs_link,
+    mo,
+):
+    mo.vstack(
+        [
+            mo.md(f'### {expression.value}'),
+            expression,
+            mo.hstack([expression_description, expression_docs_link]),
+            example_editor,
+            execution_result,
+        ]
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, selected_expression_record):
+    expression_description = mo.md(selected_expression_record["description"])
+    expression_docs_link = mo.md(
+        f"🐻‍❄️ [Official Docs](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.{selected_expression_record['expr']}.html)"
+    )
+    return expression_description, expression_docs_link
+
+
+@app.cell(hide_code=True)
+def _(example_editor, execute_code):
+    execution_result = execute_code(example_editor.value)
+    return (execution_result,)
+
+
+@app.cell(hide_code=True)
+def _(code_df, mo):
+    expression = mo.ui.dropdown(code_df.get_column('expr'), value='arr.all', searchable=True)
+    return (expression,)
+
+
+@app.cell(hide_code=True)
+def _(code_df, expression):
+    selected_expression_record = code_df.filter(expr=expression.value).to_dicts()[0]
+    return (selected_expression_record,)
+
+
+@app.cell(hide_code=True)
+def _(mo, selected_expression_record):
+    example_editor = mo.ui.code_editor(value=selected_expression_record["code"])
+    return (example_editor,)
+
+
+@app.cell(hide_code=True)
+def _(expressions_df, pl):
+    code_df = (
+        expressions_df.select(
+            expr=pl.when(pl.col("namespace") == "root")
+            .then("member")
+            .otherwise(pl.concat_str(["namespace", "member"], separator=".")),
+            description=pl.col("docstring")
+            .str.split("\n\n")
+            .list.get(0)
+            .str.slice(9),
+            docstring_lines=pl.col("docstring").str.split("\n"),
+        )
+        .with_row_index()
+        .explode("docstring_lines")
+        .rename({"docstring_lines": "docstring_line"})
+        .with_columns(pl.col("docstring_line").str.strip_chars(" "))
+        .filter(pl.col("docstring_line").str.contains_any([">>> ", "... "]))
+        .with_columns(pl.col("docstring_line").str.slice(4))
+        .group_by(pl.exclude("docstring_line"), maintain_order=True)
+        .agg(code=pl.col("docstring_line").str.join("\n"))
+        .drop("index")
+    )
+    return (code_df,)
+
+
+@app.cell(hide_code=True)
+def _():
+    def execute_code(code: str):
+        import ast
+
+        # Create a new local namespace for execution
+        local_namespace = {}
+
+        # Parse the code into an AST to identify the last expression
+        parsed_code = ast.parse(code)
+
+        # Check if there's at least one statement
+        if not parsed_code.body:
+            return None
+
+        # If the last statement is an expression, we'll need to get its value
+        last_is_expr = isinstance(parsed_code.body[-1], ast.Expr)
+
+        if last_is_expr:
+            # Split the code: everything except the last statement, and the last statement
+            last_expr = ast.Expression(parsed_code.body[-1].value)
+
+            # Remove the last statement from the parsed code
+            parsed_code.body = parsed_code.body[:-1]
+
+            # Execute everything except the last statement
+            if parsed_code.body:
+                exec(
+                    compile(parsed_code, "<string>", "exec"),
+                    globals(),
+                    local_namespace,
+                )
+
+            # Execute the last statement and get its value
+            result = eval(
+                compile(last_expr, "<string>", "eval"), globals(), local_namespace
+            )
+            return result
+        else:
+            # If the last statement is not an expression (e.g., an assignment),
+            # execute the entire code and return None
+            exec(code, globals(), local_namespace)
+            return None
+    return (execute_code,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import polars as pl
+    import marimo as mo
+    import altair as alt
+    import random
+
+    random.seed(42)
+    return alt, mo, pl, random
+
+
+if __name__ == "__main__":
+    app.run()

polars/12_aggregations.py

@@ -0,0 +1,355 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "marimo",
+#     "polars==1.23.0",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.14"
+app = marimo.App(width="medium")
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Aggregations
+        _By [Joram Mutenge](https://www.udemy.com/user/joram-mutenge/)._
+
+        In this notebook, you'll learn how to perform different types of aggregations in Polars, including grouping by categories and time. We'll analyze sales data from a clothing store, focusing on three product categories: hats, socks, and sweaters.
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    import polars as pl
+
+    df = (pl.read_csv('https://raw.githubusercontent.com/jorammutenge/learn-rust/refs/heads/main/sample_sales.csv', try_parse_dates=True)
+          .rename(lambda col: col.replace(' ','_').lower())
+         )
+    df
+    return df, pl
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Grouping by category
+        ### With single category
+        Let's find out how many of each product category we sold.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category')
+     .agg(pl.sum('quantity'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        It looks like we sold more sweaters. Maybe this was a winter season.
+
+        Let's add another aggregate to see how much was spent on the total units for each product.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category')
+     .agg(pl.sum('quantity'),
+          pl.sum('ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We could also write aggregate code for the two columns as a single line.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category')
+     .agg(pl.sum('quantity','ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Actually, the way we've been writing the aggregate lines is syntactic sugar. Here's a longer way of doing it as shown in the [Polars documentation](https://docs.pola.rs/api/python/stable/reference/dataframe/api/polars.dataframe.group_by.GroupBy.agg.html).""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category')
+     .agg(pl.col('quantity').sum(),
+          pl.col('ext_price').sum())
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### With multiple categories
+        We can also group by multiple categories. Let's find out how many items we sold in each product category for each SKU. This more detailed aggregation will produce more rows than the previous DataFrame.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category','sku')
+     .agg(pl.sum('quantity'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Aggregations when grouping data are not limited to sums. You can also use functions like [`max`, `min`, `median`, `first`, and `last`](https://docs.pola.rs/user-guide/expressions/aggregation/#basic-aggregations).  
+
+        Let's find the largest sale quantity for each product category.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .group_by('category')
+     .agg(pl.max('quantity'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Let's make the aggregation more interesting. We'll identify the first customer to purchase each item, along with the quantity they bought and the amount they spent.
+
+        **Note:** To make this work, we'll have to sort the date from earliest to latest.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .sort('date')
+     .group_by('category')
+     .agg(pl.first('account_name','quantity','ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Grouping by time
+        Since `datetime` is a special data type in Polars, we can perform various group-by aggregations on it.  
+
+        Our dataset spans a two-year period. Let's calculate the total dollar sales for each year. We'll do it the naive way first so you can appreciate grouping with time.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(year=pl.col('date').dt.year())
+     .group_by('year')
+     .agg(pl.sum('ext_price').round(2))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        We had more sales in 2014.
+
+        Now let's perform the above operation by groupin with time. This requires sorting the dataframe first.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .sort('date')
+     .group_by_dynamic('date', every='1y')
+     .agg(pl.sum('ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        The beauty of grouping with time is that it allows us to resample the data by selecting whatever time interval we want.
+
+        Let's find out what the quarterly sales were for 2014
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .filter(pl.col('date').dt.year() == 2014)
+     .sort('date')
+     .group_by_dynamic('date', every='1q')
+     .agg(pl.sum('ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Here's an interesting question we can answer that takes advantage of grouping by time.
+
+        Let's find the hour of the day where we had the most sales in dollars.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .sort('date')
+     .group_by_dynamic('date', every='1h')
+     .agg(pl.max('ext_price'))
+     .filter(pl.col('ext_price') == pl.col('ext_price').max())
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Just for fun, let's find the median number of items sold in each SKU and the total dollar amount in each SKU every six days.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .sort('date')
+     .group_by_dynamic('date', every='6d')
+     .agg(pl.first('sku'),
+          pl.median('quantity'),
+          pl.sum('ext_price'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Let's rename the columns to clearly indicate the type of aggregation performed. This will help us identify the aggregation method used on a column without needing to check the code.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .sort('date')
+     .group_by_dynamic('date', every='6d')
+     .agg(pl.first('sku'),
+          pl.median('quantity').alias('median_qty'),
+          pl.sum('ext_price').alias('total_dollars'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Grouping with over
+
+        Sometimes, we may want to perform an aggregation but also keep all the columns and rows of the dataframe.
+
+        Let's assign a value to indicate the number of times each customer visited and bought something.
+        """
+    )
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(buy_freq=pl.col('account_name').len().over('account_name'))
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Finally, let's determine which customers visited the store the most and bought something.""")
+    return
+
+
+@app.cell
+def _(df, pl):
+    (df
+     .with_columns(buy_freq=pl.col('account_name').len().over('account_name'))
+     .filter(pl.col('buy_freq') == pl.col('buy_freq').max())
+     .select('account_name','buy_freq')
+     .unique()
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""There's more you can do with aggregations in Polars such as [sorting with aggregations](https://docs.pola.rs/user-guide/expressions/aggregation/#sorting). We hope that in this notebook, we've armed you with the tools to get started.""")
+    return
+
+
+if __name__ == "__main__":
+    app.run()

polars/14_user_defined_functions.py

@@ -0,0 +1,946 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = [
+#     "altair==5.5.0",
+#     "beautifulsoup4==4.13.3",
+#     "httpx==0.28.1",
+#     "marimo",
+#     "nest-asyncio==1.6.0",
+#     "numba==0.61.0",
+#     "numpy==2.1.3",
+#     "polars==1.24.0",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.17"
+app = marimo.App(width="medium")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # User-Defined Functions
+
+        _By [Péter Ferenc Gyarmati](http://github.com/peter-gy)_.
+
+        Throughout the previous chapters, you've seen how Polars provides a comprehensive set of built-in expressions for flexible data transformation.  But what happens when you need something *more*? Perhaps your project has unique requirements, or you need to integrate functionality from an external Python library. This is where User-Defined Functions (UDFs) come into play, allowing you to extend Polars with your own custom logic.
+
+        In this chapter, we'll weigh the performance trade-offs of UDFs, pinpoint situations where they're truly beneficial, and explore different ways to effectively incorporate them into your Polars workflows. We'll walk through a complete, practical example.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ⚖️ The Cost of UDFs
+
+        > Performance vs. Flexibility
+
+        Polars' built-in expressions are highly optimized for speed and parallel processing. User-defined functions (UDFs), however, introduce a significant performance overhead because they rely on standard Python code, which often runs in a single thread and bypasses Polars' logical optimizations. Therefore, always prioritize native Polars operations *whenever possible*.
+
+        However, UDFs become inevitable when you need to:
+
+        -  **Integrate external libraries:**  Use functionality not directly available in Polars.
+        -  **Implement custom logic:** Handle complex transformations that can't be easily expressed with Polars' built-in functions.
+
+        Let's dive into a real-world project where UDFs were the only way to get the job done, demonstrating a scenario where native Polars expressions simply weren't sufficient.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 📊 Project Overview
+
+        > Scraping and Analyzing Observable Notebook Statistics
+
+        If you're into data visualization, you've probably seen [D3.js](https://d3js.org/) and [Observable Plot](https://observablehq.com/plot/). Both have extensive galleries showcasing amazing visualizations. Each gallery item is a standalone [Observable notebook](https://observablehq.com/documentation/notebooks/), with metrics like stars, comments, and forks – indicators of popularity. But getting and analyzing these statistics directly isn't straightforward. We'll need to scrape the web.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.hstack(
+        [
+            mo.image(
+                "https://minio.peter.gy/static/assets/marimo/learn/polars/14_d3-gallery.png?0",
+                width=600,
+                caption="Screenshot of https://observablehq.com/@d3/gallery",
+            ),
+            mo.image(
+                "https://minio.peter.gy/static/assets/marimo/learn/polars/14_plot-gallery.png?0",
+                width=600,
+                caption="Screenshot of https://observablehq.com/@observablehq/plot-gallery",
+            ),
+        ]
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Our goal is to use Polars UDFs to fetch the HTML content of these gallery pages. Then, we'll use the `BeautifulSoup` Python library to parse the HTML and extract the relevant metadata.  After some data wrangling with native Polars expressions, we'll have a DataFrame listing each visualization notebook. Then, we'll use another UDF to retrieve the number of likes, forks, and comments for each notebook. Finally, we will create our own high-performance UDF to implement a custom notebook ranking scheme. This will involve multiple steps, showcasing different UDF approaches.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.mermaid('''
+    graph LR;
+        url_df --> |"UDF: Fetch HTML"| html_df
+        html_df --> |"UDF: Parse with BeautifulSoup"| parsed_html_df
+        parsed_html_df --> |"Native Polars: Extract Data"| notebooks_df
+        notebooks_df --> |"UDF: Get Notebook Stats"| notebook_stats_df
+        notebook_stats_df --> |"Numba UDF: Compute Popularity"| notebook_popularity_df
+    ''')
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Our starting point, `url_df`, is a simple DataFrame with a single `url` column containing the URLs of the D3 and Observable Plot gallery notebooks.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(pl):
+    url_df = pl.from_dict(
+        {
+            "url": [
+                "https://observablehq.com/@d3/gallery",
+                "https://observablehq.com/@observablehq/plot-gallery",
+            ]
+        }
+    )
+    url_df
+    return (url_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🔂 Element-Wise UDFs
+
+        > Processing Value by Value
+
+        The most common way to use UDFs is to apply them element-wise.  This means our custom function will execute for *each individual row* in a specified column.  Our first task is to fetch the HTML content for each URL in `url_df`.
+
+        We'll define a Python function that takes a `url` (a string) as input, uses the `httpx` library (an HTTP client) to fetch the content, and returns the HTML as a string. We then integrate this function into Polars using the [`map_elements`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.map_elements.html) expression.
+
+        You'll notice we have to explicitly specify the `return_dtype`.  This is *crucial*.  Polars doesn't automatically know what our custom function will return.  We're responsible for defining the function's logic and, therefore, its output type. By providing the `return_dtype`, we help Polars maintain its internal representation of the DataFrame's schema, enabling query optimization. Think of it as giving Polars a "heads-up" about the data type it should expect.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(httpx, pl, url_df):
+    html_df = url_df.with_columns(
+        html=pl.col("url").map_elements(
+            lambda url: httpx.get(url).text,
+            return_dtype=pl.String,
+        )
+    )
+    html_df
+    return (html_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Now, `html_df` holds the HTML for each URL.  We need to parse it. Again, a UDF is the way to go. Parsing HTML with native Polars expressions would be a nightmare! Instead, we'll use the [`beautifulsoup4`](https://pypi.org/project/beautifulsoup4/) library, a standard tool for this.
+
+        These Observable pages are built with [Next.js](https://nextjs.org/), which helpfully serializes page properties as JSON within the HTML. This simplifies our UDF: we'll extract the raw JSON from the `<script id="__NEXT_DATA__" type="application/json">` tag. We'll use [`map_elements`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.map_elements.html) again.  For clarity, we'll define this UDF as a named function, `extract_nextjs_data`, since it's a bit more complex than a simple HTTP request.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(BeautifulSoup):
+    def extract_nextjs_data(html: str) -> str:
+        soup = BeautifulSoup(html, "html.parser")
+        script_tag = soup.find("script", id="__NEXT_DATA__")
+        return script_tag.text
+    return (extract_nextjs_data,)
+
+
+@app.cell(hide_code=True)
+def _(extract_nextjs_data, html_df, pl):
+    parsed_html_df = html_df.select(
+        "url",
+        next_data=pl.col("html").map_elements(
+            extract_nextjs_data,
+            return_dtype=pl.String,
+        ),
+    )
+    parsed_html_df
+    return (parsed_html_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""With some data wrangling of the raw JSON (using *native* Polars expressions!), we get `notebooks_df`, containing the metadata for each notebook.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(parsed_html_df, pl):
+    notebooks_df = (
+        parsed_html_df.select(
+            "url",
+            # We extract the content of every cell present in the gallery notebooks
+            cell=pl.col("next_data")
+            .str.json_path_match("$.props.pageProps.initialNotebook.nodes")
+            .str.json_decode()
+            .list.eval(pl.element().struct.field("value")),
+        )
+        # We want one row per cell
+        .explode("cell")
+        # Only keep categorized notebook listing cells starting with H3
+        .filter(pl.col("cell").str.starts_with("### "))
+        # Split up the cells into [heading, description, config] sections
+        .with_columns(pl.col("cell").str.split("\n\n"))
+        .select(
+            gallery_url="url",
+            # Text after the '### ' heading, ignore '<!--' comments'
+            category=pl.col("cell").list.get(0).str.extract(r"###\s+(.*?)(?:\s+<!--.*?-->|$)"),
+            # Paragraph after heading
+            description=pl.col("cell")
+            .list.get(1)
+            .str.strip_chars(" ")
+            .str.replace_all("](/", "](https://observablehq.com/", literal=True),
+            # Parsed notebook config from ${preview([{...}])}
+            notebooks=pl.col("cell")
+            .list.get(2)
+            .str.strip_prefix("${previews([")
+            .str.strip_suffix("]})}")
+            .str.strip_chars(" \n")
+            .str.split("},")
+            # Simple regex-based attribute extraction from JS/JSON objects like
+            # ```js
+            # {
+            #   path: "@d3/spilhaus-shoreline-map",
+            #   "thumbnail": "66a87355e205d820...",
+            #   title: "Spilhaus shoreline map",
+            #   "author": "D3"
+            # }
+            # ```
+            .list.eval(
+                pl.struct(
+                    *(
+                        pl.element()
+                        .str.extract(f'(?:"{key}"|{key})\s*:\s*"([^"]*)"')
+                        .alias(key)
+                        for key in ["path", "thumbnail", "title"]
+                    )
+                )
+            ),
+        )
+        .explode("notebooks")
+        .unnest("notebooks")
+        .filter(pl.col("path").is_not_null())
+        # Final projection to end up with directly usable values
+        .select(
+            pl.concat_str(
+                [
+                    pl.lit("https://static.observableusercontent.com/thumbnail/"),
+                    "thumbnail",
+                    pl.lit(".jpg"),
+                ],
+            ).alias("notebook_thumbnail_src"),
+            "category",
+            "title",
+            "description",
+            pl.concat_str(
+                [pl.lit("https://observablehq.com"), "path"], separator="/"
+            ).alias("notebook_url"),
+        )
+    )
+    notebooks_df
+    return (notebooks_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 📦 Batch-Wise UDFs
+
+        > Processing Entire Series
+
+        `map_elements` calls the UDF for *each row*. Fine for our tiny, two-rows-tall `url_df`. But `notebooks_df` has almost 400 rows! Individual HTTP requests for each would be painfully slow.
+
+        We want stats for each notebook in `notebooks_df`. To avoid sequential requests, we'll use Polars' [`map_batches`](https://docs.pola.rs/api/python/stable/reference/expressions/api/polars.Expr.map_batches.html). This lets us process an *entire Series* (a column) at once.
+
+        Our UDF, `fetch_html_batch`, will take a *Series* of URLs and use `asyncio` to make concurrent requests – a huge performance boost.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(Iterable, asyncio, httpx, mo):
+    async def _fetch_html_batch(urls: Iterable[str]) -> tuple[str, ...]:
+        async with httpx.AsyncClient(timeout=15) as client:
+            res = await asyncio.gather(*(client.get(url) for url in urls))
+            return tuple((r.text for r in res))
+
+
+    @mo.cache
+    def fetch_html_batch(urls: Iterable[str]) -> tuple[str, ...]:
+        return asyncio.run(_fetch_html_batch(urls))
+    return (fetch_html_batch,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.callout(
+        mo.md("""
+    Since `fetch_html_batch` is a pure Python function and performs multiple network requests, it's a good candidate for caching. We use [`mo.cache`](https://docs.marimo.io/api/caching/#marimo.cache) to avoid redundant requests to the same URL. This is a simple way to improve performance without modifying the core logic.
+    """
+        ),
+        kind="info",
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, notebooks_df):
+    category = mo.ui.dropdown(
+        notebooks_df.sort("category").get_column("category"),
+        value="Maps",
+    )
+    return (category,)
+
+
+@app.cell(hide_code=True)
+def _(category, extract_nextjs_data, fetch_html_batch, notebooks_df, pl):
+    notebook_stats_df = (
+        # Setting filter upstream to limit number of concurrent HTTP requests
+        notebooks_df.filter(category=category.value)
+        .with_columns(
+            notebook_html=pl.col("notebook_url")
+            .map_batches(fetch_html_batch, return_dtype=pl.List(pl.String))
+            .explode()
+        )
+        .with_columns(
+            notebook_data=pl.col("notebook_html")
+            .map_elements(
+                extract_nextjs_data,
+                return_dtype=pl.String,
+            )
+            .str.json_path_match("$.props.pageProps.initialNotebook")
+            .str.json_decode()
+        )
+        .drop("notebook_html")
+        .with_columns(
+            *[
+                pl.col("notebook_data").struct.field(key).alias(key)
+                for key in ["likes", "forks", "comments", "license"]
+            ]
+        )
+        .drop("notebook_data")
+        .with_columns(pl.col("comments").list.len())
+        .select(
+            pl.exclude("description", "notebook_url"),
+            "description",
+            "notebook_url",
+        )
+        .sort("likes", descending=True)
+    )
+    return (notebook_stats_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo, notebook_stats_df):
+    notebooks = mo.ui.table(notebook_stats_df, selection='single', initial_selection=[2], page_size=5)
+    notebook_height = mo.ui.slider(start=400, stop=2000, value=825, step=25, show_value=True, label='Notebook Height')
+    return notebook_height, notebooks
+
+
+@app.cell(hide_code=True)
+def _():
+    def nb_iframe(notebook_url: str, height=825) -> str:
+        embed_url = notebook_url.replace(
+            "https://observablehq.com", "https://observablehq.com/embed"
+        )
+        return f'<iframe width="100%" height="{height}" frameborder="0" src="{embed_url}?cell=*"></iframe>'
+    return (nb_iframe,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Now that we have access to notebook-level statistics, we can rank the visualizations by the number of likes they received & display them interactively.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.callout("💡 Explore the visualizations by paging through the table below and selecting any of its rows.")
+    return
+
+
+@app.cell(hide_code=True)
+def _(category, mo, nb_iframe, notebook_height, notebooks):
+    notebook = notebooks.value.to_dicts()[0]
+    mo.vstack(
+        [
+            mo.hstack([category, notebook_height]),
+            notebooks,
+            mo.md(f"{notebook['description']}"),
+            mo.md('---'),
+            mo.md(nb_iframe(notebook["notebook_url"], notebook_height.value)),
+        ]
+    )
+    return (notebook,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ⚙️ Row-Wise UDFs
+
+        > Accessing All Columns at Once
+
+        Sometimes, you need to work with *all* columns of a row at once.  This is where [`map_rows`](https://docs.pola.rs/api/python/stable/reference/dataframe/api/polars.DataFrame.map_rows.html) comes in. It operates directly on the DataFrame, passing each row to your UDF *as a tuple*.
+
+        Below, `create_notebook_summary` takes a row from `notebook_stats_df` (as a tuple) and returns a formatted Markdown string summarizing the notebook's key stats.  We're essentially reducing the DataFrame to a single column.  While this *could* be done with native Polars expressions, it would be much more cumbersome. This example demonstrates a case where a row-wise UDF simplifies the code, even if the underlying operation isn't inherently complex.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _():
+    def create_notebook_summary(row: tuple) -> str:
+        (
+            thumbnail_src,
+            category,
+            title,
+            likes,
+            forks,
+            comments,
+            license,
+            description,
+            notebook_url,
+        ) = row
+        return (
+            f"""
+    ### [{title}]({notebook_url})
+
+    <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 12px; margin: 12px 0;">
+        <div>⭐ <strong>Likes:</strong> {likes}</div>
+        <div>↗️ <strong>Forks:</strong> {forks}</div>
+        <div>💬 <strong>Comments:</strong> {comments}</div>
+        <div>⚖️ <strong>License:</strong> {license}</div>
+    </div>
+
+    <a href="{notebook_url}" target="_blank">
+        <img src="{thumbnail_src}" style="height: 300px;" />
+    <a/>
+    """.strip('\n')
+        )
+    return (create_notebook_summary,)
+
+
+@app.cell(hide_code=True)
+def _(create_notebook_summary, notebook_stats_df, pl):
+    notebook_summary_df = notebook_stats_df.map_rows(
+        create_notebook_summary,
+        return_dtype=pl.String,
+    ).rename({"map": "summary"})
+    notebook_summary_df.head(1)
+    return (notebook_summary_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.callout("💡 You can explore individual notebook statistics through the carousel. Discover the visualization's source code by clicking the notebook title or the thumbnail.")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, notebook_summary_df):
+    mo.carousel(
+        [
+            mo.lazy(mo.md(summary))
+            for summary in notebook_summary_df.get_column("summary")
+        ]
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🚀 Higher-performance UDFs
+
+        > Leveraging Numba to Make Python Fast
+
+        Python code doesn't *always* mean slow code. While UDFs *often* introduce performance overhead, there are exceptions. NumPy's universal functions ([`ufuncs`](https://numpy.org/doc/stable/reference/ufuncs.html)) and generalized universal functions ([`gufuncs`](https://numpy.org/neps/nep-0005-generalized-ufuncs.html)) provide high-performance operations on NumPy arrays, thanks to low-level implementations.
+
+        But NumPy's built-in functions are predefined. We can't easily use them for *custom* logic. Enter [`numba`](https://numba.pydata.org/).  Numba is a just-in-time (JIT) compiler that translates Python functions into optimized machine code *at runtime*. It provides decorators like [`numba.guvectorize`](https://numba.readthedocs.io/en/stable/user/vectorize.html#the-guvectorize-decorator) that let us create our *own* high-performance `gufuncs` – *without* writing low-level code!
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        Let's create a custom popularity metric to rank notebooks, considering likes, forks, *and* comments (not just likes).  We'll define `weighted_popularity_numba`, decorated with `@numba.guvectorize`.  The decorator arguments specify that we're taking three integer vectors of length `n` and returning a float vector of length `n`.
+
+        The weighted popularity score for each notebook is calculated using the following formula:
+
+        $$
+        \begin{equation}
+        \text{score}_i = w_l \cdot l_i^{f} + w_f \cdot f_i^{f} + w_c \cdot c_i^{f}
+        \end{equation}
+        $$
+
+        with:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, non_linear_factor, weight_comments, weight_forks, weight_likes):
+    mo.md(rf"""
+    | Symbol | Description |
+    |--------|-------------|
+    | $\text{{score}}_i$ | Popularity score for the *i*-th notebook |
+    | $w_l = {weight_likes.value}$ | Weight for likes |
+    | $l_i$ | Number of likes for the *i*-th notebook |
+    | $w_f = {weight_forks.value}$ | Weight for forks |
+    | $f_i$ | Number of forks for the *i*-th notebook |
+    | $w_c = {weight_comments.value}$ | Weight for comments |
+    | $c_i$ | Number of comments for the *i*-th notebook |
+    | $f = {non_linear_factor.value}$ | Non-linear factor (exponent) |
+    """)
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    weight_likes = mo.ui.slider(
+        start=0.1,
+        stop=1,
+        value=0.5,
+        step=0.1,
+        show_value=True,
+        label="⭐ Weight for Likes",
+    )
+    weight_forks = mo.ui.slider(
+        start=0.1,
+        stop=1,
+        value=0.3,
+        step=0.1,
+        show_value=True,
+        label="↗️ Weight for Forks",
+    )
+    weight_comments = mo.ui.slider(
+        start=0.1,
+        stop=1,
+        value=0.5,
+        step=0.1,
+        show_value=True,
+        label="💬 Weight for Comments",
+    )
+    non_linear_factor = mo.ui.slider(
+        start=1,
+        stop=2,
+        value=1.2,
+        step=0.1,
+        show_value=True,
+        label="🎢 Non-Linear Factor",
+    )
+    return non_linear_factor, weight_comments, weight_forks, weight_likes
+
+
+@app.cell(hide_code=True)
+def _(
+    non_linear_factor,
+    np,
+    numba,
+    weight_comments,
+    weight_forks,
+    weight_likes,
+):
+    w_l = weight_likes.value
+    w_f = weight_forks.value
+    w_c = weight_comments.value
+    nlf = non_linear_factor.value
+
+
+    @numba.guvectorize(
+        [(numba.int64[:], numba.int64[:], numba.int64[:], numba.float64[:])],
+        "(n), (n), (n) -> (n)",
+    )
+    def weighted_popularity_numba(
+        likes: np.ndarray,
+        forks: np.ndarray,
+        comments: np.ndarray,
+        out: np.ndarray,
+    ):
+        for i in range(likes.shape[0]):
+            out[i] = (
+                w_l * (likes[i] ** nlf)
+                + w_f * (forks[i] ** nlf)
+                + w_c * (comments[i] ** nlf)
+            )
+    return nlf, w_c, w_f, w_l, weighted_popularity_numba
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We apply our JIT-compiled UDF using `map_batches`, as before.  The key is that we're passing entire columns directly to `weighted_popularity_numba`. Polars and Numba handle the conversion to NumPy arrays behind the scenes. This direct integration is a major benefit of using `guvectorize`.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(notebook_stats_df, pl, weighted_popularity_numba):
+    notebook_popularity_df = (
+        notebook_stats_df.select(
+            pl.col("notebook_thumbnail_src").alias("thumbnail"),
+            "title",
+            "likes",
+            "forks",
+            "comments",
+            popularity=pl.struct(["likes", "forks", "comments"]).map_batches(
+                lambda obj: weighted_popularity_numba(
+                    obj.struct.field("likes"),
+                    obj.struct.field("forks"),
+                    obj.struct.field("comments"),
+                ),
+                return_dtype=pl.Float64,
+            ),
+            url="notebook_url",
+        )
+    )
+    return (notebook_popularity_df,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.callout("💡 Adjust the hyperparameters of the popularity ranking UDF. How do the weights and non-linear factor affect the notebook rankings?")
+    return
+
+
+@app.cell(hide_code=True)
+def _(
+    mo,
+    non_linear_factor,
+    notebook_popularity_df,
+    weight_comments,
+    weight_forks,
+    weight_likes,
+):
+    mo.vstack(
+        [
+            mo.hstack([weight_likes, weight_forks]),
+            mo.hstack([weight_comments, non_linear_factor]),
+            notebook_popularity_df,
+        ]
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""As the slope chart below demonstrates, this new ranking strategy significantly changes the notebook order, as it considers forks and comments, not just likes.""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(alt, notebook_popularity_df, pl):
+    notebook_ranks_df = (
+        notebook_popularity_df.sort("likes", descending=True)
+        .with_row_index("rank_by_likes")
+        .with_columns(pl.col("rank_by_likes") + 1)
+        .sort("popularity", descending=True)
+        .with_row_index("rank_by_popularity")
+        .with_columns(pl.col("rank_by_popularity") + 1)
+        .select("thumbnail", "title", "rank_by_popularity", "rank_by_likes")
+        .unpivot(
+            ["rank_by_popularity", "rank_by_likes"],
+            index="title",
+            variable_name="strategy",
+            value_name="rank",
+        )
+    )
+
+    # Slope chart to visualize rank differences by strategy
+    lines = notebook_ranks_df.plot.line(
+        x="strategy:O",
+        y="rank:Q",
+        color="title:N",
+    )
+    points = notebook_ranks_df.plot.point(
+        x="strategy:O",
+        y="rank:Q",
+        color=alt.Color("title:N", legend=None),
+        fill="title:N",
+    )
+    (points + lines).properties(width=400)
+    return lines, notebook_ranks_df, points
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## ⏱️ Quantifying the Overhead
+
+        > UDF Performance Comparison
+
+        To truly understand the performance implications of using UDFs, let's conduct a benchmark.  We'll create a DataFrame with random numbers and perform the same numerical operation using four different methods:
+
+        1. **Native Polars:** Using Polars' built-in expressions.
+        2. **`map_elements`:**  Applying a Python function element-wise.
+        3. **`map_batches`:** **Applying** a Python function to the entire Series.
+        4. **`map_batches` with Numba:** Applying a JIT-compiled function to batches, similar to a generalized universal function.
+
+        We'll use a simple, but non-trivial, calculation:  `result = (x * 2.5 + 5) / (x + 1)`. This involves multiplication, addition, and division, giving us a realistic representation of a common numerical operation. We'll use the `timeit` module, to accurately measure execution times over multiple trials.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.callout("💡 Tweak the benchmark parameters to explore how execution times change with different sample sizes and trial counts. Do you notice anything surprising as you decrease the number of samples?")
+    return
+
+
+@app.cell(hide_code=True)
+def _(benchmark_plot, mo, num_samples, num_trials):
+    mo.vstack(
+        [
+            mo.hstack([num_samples, num_trials]),
+            mo.md(
+                f"""---
+    Performance comparison over **{num_trials.value:,} trials** with **{num_samples.value:,} samples**.
+
+    > Lower execution times are better.
+    """
+            ),
+            benchmark_plot,
+        ]
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        As anticipated, the `Batch-Wise UDF (Python)` and `Element-Wise UDF` exhibit significantly worse performance, essentially acting as pure-Python for-each loops.  
+
+        However, when Python serves as an interface to lower-level, high-performance libraries, we observe substantial improvements. The `Batch-Wise UDF (NumPy)` lags behind both `Batch-Wise UDF (Numba)` and `Native Polars`, but it still represents a considerable improvement over pure-Python UDFs due to its vectorized computations. 
+
+        Numba's Just-In-Time (JIT) compilation delivers a dramatic performance boost, achieving speeds comparable to native Polars expressions. This demonstrates that UDFs, particularly when combined with tools like Numba, don't inevitably lead to bottlenecks in numerical computations.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    num_samples = mo.ui.slider(
+        start=1_000,
+        stop=1_000_000,
+        value=250_000,
+        step=1000,
+        show_value=True,
+        debounce=True,
+        label="Number of Samples",
+    )
+    num_trials = mo.ui.slider(
+        start=50,
+        stop=1_000,
+        value=100,
+        step=50,
+        show_value=True,
+        debounce=True,
+        label="Number of Trials",
+    )
+    return num_samples, num_trials
+
+
+@app.cell(hide_code=True)
+def _(np, num_samples, pl):
+    rng = np.random.default_rng(42)
+    sample_df = pl.from_dict({"x": rng.random(num_samples.value)})
+    return rng, sample_df
+
+
+@app.cell(hide_code=True)
+def _(np, num_trials, numba, pl, sample_df, timeit):
+    def run_native():
+        sample_df.with_columns(
+            result_native=(pl.col("x") * 2.5 + 5) / (pl.col("x") + 1)
+        )
+
+
+    def _calculate_elementwise(x: float) -> float:
+        return (x * 2.5 + 5) / (x + 1)
+
+
+    def run_map_elements():
+        sample_df.with_columns(
+            result_map_elements=pl.col("x").map_elements(
+                _calculate_elementwise,
+                return_dtype=pl.Float64,
+            )
+        )
+
+
+    def _calculate_batchwise_numpy(x_series: pl.Series) -> pl.Series:
+        x_array = x_series.to_numpy()
+        result_array = (x_array * 2.5 + 5) / (x_array + 1)
+        return pl.Series(result_array)
+
+
+    def run_map_batches_numpy():
+        sample_df.with_columns(
+            result_map_batches_numpy=pl.col("x").map_batches(
+                _calculate_batchwise_numpy,
+                return_dtype=pl.Float64,
+            )
+        )
+
+
+    def _calculate_batchwise_python(x_series: pl.Series) -> pl.Series:
+        x_array = x_series.to_list()
+        result_array = [_calculate_elementwise(x) for x in x_array]
+        return pl.Series(result_array)
+
+
+    def run_map_batches_python():
+        sample_df.with_columns(
+            result_map_batches_python=pl.col("x").map_batches(
+                _calculate_batchwise_python,
+                return_dtype=pl.Float64,
+            )
+        )
+
+
+    @numba.guvectorize([(numba.float64[:], numba.float64[:])], "(n) -> (n)")
+    def _calculate_batchwise_numba(x: np.ndarray, out: np.ndarray):
+        for i in range(x.shape[0]):
+            out[i] = (x[i] * 2.5 + 5) / (x[i] + 1)
+
+
+    def run_map_batches_numba():
+        sample_df.with_columns(
+            result_map_batches_numba=pl.col("x").map_batches(
+                _calculate_batchwise_numba,
+                return_dtype=pl.Float64,
+            )
+        )
+
+
+    def time_method(callable_name: str, number=num_trials.value) -> float:
+        fn = globals()[callable_name]
+        return timeit.timeit(fn, number=number)
+    return (
+        run_map_batches_numba,
+        run_map_batches_numpy,
+        run_map_batches_python,
+        run_map_elements,
+        run_native,
+        time_method,
+    )
+
+
+@app.cell(hide_code=True)
+def _(alt, pl, time_method):
+    benchmark_df = pl.from_dicts(
+        [
+            {
+                "title": "Native Polars",
+                "callable_name": "run_native",
+            },
+            {
+                "title": "Element-Wise UDF",
+                "callable_name": "run_map_elements",
+            },
+            {
+                "title": "Batch-Wise UDF (NumPy)",
+                "callable_name": "run_map_batches_numpy",
+            },
+            {
+                "title": "Batch-Wise UDF (Python)",
+                "callable_name": "run_map_batches_python",
+            },
+            {
+                "title": "Batch-Wise UDF (Numba)",
+                "callable_name": "run_map_batches_numba",
+            },
+        ]
+    ).with_columns(
+        time=pl.col("callable_name").map_elements(
+            time_method, return_dtype=pl.Float64
+        )
+    )
+
+    benchmark_plot = benchmark_df.plot.bar(
+        x=alt.X("title:N", title="Method", sort="-y"),
+        y=alt.Y("time:Q", title="Execution Time (s)", axis=alt.Axis(format=".3f")),
+    ).properties(width=400)
+    return benchmark_df, benchmark_plot
+
+
+@app.cell(hide_code=True)
+def _():
+    import asyncio
+    import timeit
+    from typing import Iterable
+
+    import altair as alt
+    import httpx
+    import marimo as mo
+    import nest_asyncio
+    import numba
+    import numpy as np
+    from bs4 import BeautifulSoup
+
+    import polars as pl
+
+    # Fixes RuntimeError: asyncio.run() cannot be called from a running event loop
+    nest_asyncio.apply()
+    return (
+        BeautifulSoup,
+        Iterable,
+        alt,
+        asyncio,
+        httpx,
+        mo,
+        nest_asyncio,
+        np,
+        numba,
+        pl,
+        timeit,
+    )
+
+
+if __name__ == "__main__":
+    app.run()

polars/README.md

@@ -23,3 +23,4 @@ You can also open notebooks in our online playground by appending marimo.app/ to
 Thanks to all our notebook authors!
 
 * [Koushik Khan](https://github.com/koushikkhan)
+* [Péter Gyarmati](https://github.com/peter-gy)

probability/08_bayes_theorem.py

@@ -307,7 +307,7 @@ def _(mo):
     mo.md(
         r"""
 
-        _This interactive exmaple was made with [marimo](https://github.com/marimo-team/marimo/blob/main/examples/misc/bayes_theorem.py), and is [based on an explanation of Bayes' Theorem by Grant Sanderson](https://www.youtube.com/watch?v=HZGCoVF3YvM&list=PLzq7odmtfKQw2KIbQq0rzWrqgifHKkPG1&index=1&t=3s)_.
+        _This interactive example was made with [marimo](https://github.com/marimo-team/marimo/blob/main/examples/misc/bayes_theorem.py), and is [based on an explanation of Bayes' Theorem by Grant Sanderson](https://www.youtube.com/watch?v=HZGCoVF3YvM&list=PLzq7odmtfKQw2KIbQq0rzWrqgifHKkPG1&index=1&t=3s)_.
 
         Bayes theorem provides a convenient way to calculate the probability
         of a hypothesis event $H$ given evidence $E$:

probability/10_probability_mass_function.py

@@ -0,0 +1,711 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.3",
+#     "scipy==1.15.2",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.17"
+app = marimo.App(width="medium", app_title="Probability Mass Functions")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Probability Mass Functions
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/pmf/), by Stanford professor Chris Piech._
+
+        For a random variable, the most important thing to know is: how likely is each outcome? For a discrete random variable, this information is called the "**Probability Mass Function**". The probability mass function (PMF) provides the "mass" (i.e. amount) of "probability" for each possible assignment of the random variable.
+
+        Formally, the Probability Mass Function is a mapping between the values that the random variable could take on and the probability of the random variable taking on said value. In mathematics, we call these associations functions. There are many different ways of representing functions: you can write an equation, you can make a graph, you can even store many samples in a list.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Properties of a PMF
+
+        For a function $p_X(x)$ to be a valid PMF, it must satisfy:
+
+        1. **Non-negativity**: $p_X(x) \geq 0$ for all $x$
+        2. **Unit total probability**: $\sum_x p_X(x) = 1$
+
+        ### Probabilities Must Sum to 1
+
+        For a variable (call it $X$) to be a proper random variable, it must be the case that if you summed up the values of $P(X=x)$ for all possible values $x$ that $X$ can take on, the result must be 1:
+
+        $$\sum_x P(X=x) = 1$$
+
+        This is because a random variable taking on a value is an event (for example $X=3$). Each of those events is mutually exclusive because a random variable will take on exactly one value. Those mutually exclusive cases define an entire sample space. Why? Because $X$ must take on some value.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## PMFs as Graphs
+
+        Let's start by looking at PMFs as graphs where the $x$-axis is the values that the random variable could take on and the $y$-axis is the probability of the random variable taking on said value.
+
+        In the following example, we show two PMFs:
+
+        - On the left: PMF for the random variable $X$ = the value of a single six-sided die roll
+        - On the right: PMF for the random variable $Y$ = value of the sum of two dice rolls
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(np, plt):
+    # Single die PMF
+    single_die_values = np.arange(1, 7)
+    single_die_probs = np.ones(6) / 6
+
+    # Two dice sum PMF
+    two_dice_values = np.arange(2, 13)
+    two_dice_probs = []
+
+    for dice_sum in two_dice_values:
+        if dice_sum <= 7:
+            dice_prob = (dice_sum-1) / 36
+        else:
+            dice_prob = (13-dice_sum) / 36
+        two_dice_probs.append(dice_prob)
+
+    # Create side-by-side plots
+    dice_fig, (dice_ax1, dice_ax2) = plt.subplots(1, 2, figsize=(12, 4))
+
+    # Single die plot
+    dice_ax1.bar(single_die_values, single_die_probs, width=0.4)
+    dice_ax1.set_xticks(single_die_values)
+    dice_ax1.set_xlabel('Value of die roll (x)')
+    dice_ax1.set_ylabel('Probability: P(X = x)')
+    dice_ax1.set_title('PMF of a Single Die Roll')
+    dice_ax1.grid(alpha=0.3)
+
+    # Two dice sum plot
+    dice_ax2.bar(two_dice_values, two_dice_probs, width=0.4)
+    dice_ax2.set_xticks(two_dice_values)
+    dice_ax2.set_xlabel('Sum of two dice (y)')
+    dice_ax2.set_ylabel('Probability: P(Y = y)')
+    dice_ax2.set_title('PMF of Sum of Two Dice')
+    dice_ax2.grid(alpha=0.3)
+
+    plt.tight_layout()
+    plt.gca()
+    return (
+        dice_ax1,
+        dice_ax2,
+        dice_fig,
+        dice_prob,
+        dice_sum,
+        single_die_probs,
+        single_die_values,
+        two_dice_probs,
+        two_dice_values,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        The information provided in these graphs shows the likelihood of a random variable taking on different values.
+
+        In the graph on the right, the value "6" on the $x$-axis is associated with the probability $\frac{5}{36}$ on the $y$-axis. This $x$-axis refers to the event "the sum of two dice is 6" or $Y = 6$. The $y$-axis tells us that the probability of that event is $\frac{5}{36}$. In full: $P(Y = 6) = \frac{5}{36}$.
+
+        The value "2" is associated with "$\frac{1}{36}$" which tells us that, $P(Y = 2) = \frac{1}{36}$, the probability that two dice sum to 2 is $\frac{1}{36}$. There is no value associated with "1" because the sum of two dice cannot be 1.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## PMFs as Equations
+
+        Here is the exact same information in equation form:
+
+        For a single die roll $X$:
+        $$P(X=x) = \frac{1}{6} \quad \text{ if } 1 \leq x \leq 6$$
+
+        For the sum of two dice $Y$:
+        $$P(Y=y) = \begin{cases}
+        \frac{(y-1)}{36} & \text{ if } 2 \leq y \leq 7\\
+        \frac{(13-y)}{36} & \text{ if } 8 \leq y \leq 12
+        \end{cases}$$
+
+        Let's implement the PMF for $Y$, the sum of two dice, in Python code:
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    def pmf_sum_two_dice(y_val):
+        """Returns the probability that the sum of two dice is y"""
+        if y_val < 2 or y_val > 12:
+            return 0
+        if y_val <= 7:
+            return (y_val-1) / 36
+        else:
+            return (13-y_val) / 36
+
+    # Test the function for a few values
+    test_values = [1, 2, 7, 12, 13]
+    for test_y in test_values:
+        print(f"P(Y = {test_y}) = {pmf_sum_two_dice(test_y)}")
+    return pmf_sum_two_dice, test_values, test_y
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Now, let's verify that our PMF satisfies the property that the sum of all probabilities equals 1:""")
+    return
+
+
+@app.cell
+def _(pmf_sum_two_dice):
+    # Verify that probabilities sum to 1
+    verify_total_prob = sum(pmf_sum_two_dice(y_val) for y_val in range(2, 13))
+    # Round to 10 decimal places to handle floating-point precision
+    verify_total_prob_rounded = round(verify_total_prob, 10)
+    print(f"Sum of all probabilities: {verify_total_prob_rounded}")
+    return verify_total_prob, verify_total_prob_rounded
+
+
+@app.cell(hide_code=True)
+def _(plt, pmf_sum_two_dice):
+    # Create a visual verification
+    verify_y_values = list(range(2, 13))
+    verify_probabilities = [pmf_sum_two_dice(y_val) for y_val in verify_y_values]
+
+    plt.figure(figsize=(10, 4))
+    plt.bar(verify_y_values, verify_probabilities, width=0.4)
+    plt.xticks(verify_y_values)
+    plt.xlabel('Sum of two dice (y)')
+    plt.ylabel('Probability: P(Y = y)')
+    plt.title('PMF of Sum of Two Dice (Total Probability = 1)')
+    plt.grid(alpha=0.3)
+
+    # Add probability values on top of bars
+    for verify_i, verify_prob in enumerate(verify_probabilities):
+        plt.text(verify_y_values[verify_i], verify_prob + 0.001, f'{verify_prob:.3f}', ha='center')
+
+    plt.gca()  # Return the current axes to ensure proper display
+    return verify_i, verify_prob, verify_probabilities, verify_y_values
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Data to Histograms to Probability Mass Functions
+
+        One surprising way to store a likelihood function (recall that a PMF is the name of the likelihood function for discrete random variables) is simply a list of data. Let's simulate summing two dice many times to create an empirical PMF:
+        """
+    )
+    return
+
+
+@app.cell
+def _(np):
+    # Simulate rolling two dice many times
+    sim_num_trials = 10000
+    np.random.seed(42)  # For reproducibility
+
+    # Generate random dice rolls
+    sim_die1 = np.random.randint(1, 7, size=sim_num_trials)
+    sim_die2 = np.random.randint(1, 7, size=sim_num_trials)
+
+    # Calculate the sum
+    sim_dice_sums = sim_die1 + sim_die2
+
+    # Display a small sample of the data
+    print(f"First 20 dice sums: {sim_dice_sums[:20]}")
+    print(f"Total number of trials: {sim_num_trials}")
+    return sim_dice_sums, sim_die1, sim_die2, sim_num_trials
+
+
+@app.cell(hide_code=True)
+def _(collections, np, plt, sim_dice_sums):
+    # Count the frequency of each sum
+    sim_counter = collections.Counter(sim_dice_sums)
+
+    # Sort the values
+    sim_sorted_values = sorted(sim_counter.keys())
+
+    # Calculate the empirical PMF
+    sim_empirical_pmf = [sim_counter[x] / len(sim_dice_sums) for x in sim_sorted_values]
+
+    # Calculate the theoretical PMF
+    sim_theoretical_values = np.arange(2, 13)
+    sim_theoretical_pmf = []
+    for sim_y in sim_theoretical_values:
+        if sim_y <= 7:
+            sim_prob = (sim_y-1) / 36
+        else:
+            sim_prob = (13-sim_y) / 36
+        sim_theoretical_pmf.append(sim_prob)
+
+    # Create a comparison plot
+    sim_fig, (sim_ax1, sim_ax2) = plt.subplots(1, 2, figsize=(12, 4))
+
+    # Empirical PMF (normalized histogram)
+    sim_ax1.bar(sim_sorted_values, sim_empirical_pmf, width=0.4)
+    sim_ax1.set_xticks(sim_sorted_values)
+    sim_ax1.set_xlabel('Sum of two dice')
+    sim_ax1.set_ylabel('Empirical Probability')
+    sim_ax1.set_title(f'Empirical PMF from {len(sim_dice_sums)} Trials')
+    sim_ax1.grid(alpha=0.3)
+
+    # Theoretical PMF
+    sim_ax2.bar(sim_theoretical_values, sim_theoretical_pmf, width=0.4)
+    sim_ax2.set_xticks(sim_theoretical_values)
+    sim_ax2.set_xlabel('Sum of two dice')
+    sim_ax2.set_ylabel('Theoretical Probability')
+    sim_ax2.set_title('Theoretical PMF')
+    sim_ax2.grid(alpha=0.3)
+
+    plt.tight_layout()
+
+    # Let's also look at the raw counts (histogram)
+    plt.figure(figsize=(10, 4))
+    sim_counts = [sim_counter[x] for x in sim_sorted_values]
+    plt.bar(sim_sorted_values, sim_counts, width=0.4)
+    plt.xticks(sim_sorted_values)
+    plt.xlabel('Sum of two dice')
+    plt.ylabel('Frequency')
+    plt.title('Histogram of Dice Sum Frequencies')
+    plt.grid(alpha=0.3)
+
+    # Add count values on top of bars
+    for sim_i, sim_count in enumerate(sim_counts):
+        plt.text(sim_sorted_values[sim_i], sim_count + 19, str(sim_count), ha='center')
+
+    plt.gca()  # Return the current axes to ensure proper display
+    return (
+        sim_ax1,
+        sim_ax2,
+        sim_count,
+        sim_counter,
+        sim_counts,
+        sim_empirical_pmf,
+        sim_fig,
+        sim_i,
+        sim_prob,
+        sim_sorted_values,
+        sim_theoretical_pmf,
+        sim_theoretical_values,
+        sim_y,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        A normalized histogram (where each value is divided by the length of your data list) is an approximation of the PMF. For a dataset of discrete numbers, a histogram shows the count of each value. By the definition of probability, if you divide this count by the number of experiments run, you arrive at an approximation of the probability of the event $P(Y=y)$.
+
+        Let's look at a specific example. If we want to approximate $P(Y=3)$ (the probability that the sum of two dice is 3), we can count the number of times that "3" occurs in our data and divide by the total number of trials:
+        """
+    )
+    return
+
+
+@app.cell
+def _(sim_counter, sim_dice_sums):
+    # Calculate P(Y=3) empirically
+    sim_count_of_3 = sim_counter[3]
+    sim_empirical_prob = sim_count_of_3 / len(sim_dice_sums)
+
+    # Calculate P(Y=3) theoretically
+    sim_theoretical_prob = 2/36  # There are 2 ways to get a sum of 3 out of 36 possible outcomes
+
+    print(f"Count of sum=3: {sim_count_of_3}")
+    print(f"Empirical P(Y=3): {sim_count_of_3}/{len(sim_dice_sums)} = {sim_empirical_prob:.4f}")
+    print(f"Theoretical P(Y=3): 2/36 = {sim_theoretical_prob:.4f}")
+    print(f"Difference: {abs(sim_empirical_prob - sim_theoretical_prob):.4f}")
+    return sim_count_of_3, sim_empirical_prob, sim_theoretical_prob
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        As we can see, with a large number of trials, the empirical PMF becomes a very good approximation of the theoretical PMF. This is an example of the [Law of Large Numbers](https://en.wikipedia.org/wiki/Law_of_large_numbers) in action.
+
+        ## Interactive Example: Exploring PMFs
+
+        Let's create an interactive tool to explore different PMFs:
+        """
+    )
+    return
+
+
+@app.cell
+def _(dist_param1, dist_param2, dist_selection, mo):
+    mo.hstack([dist_selection, dist_param1, dist_param2], justify="space-around")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    dist_selection = mo.ui.dropdown(
+        options=[
+            "bernoulli",
+            "binomial",
+            "geometric",
+            "poisson"
+        ],
+        value="bernoulli",
+        label="Select a distribution"
+    )
+
+    # Parameters for different distributions
+    dist_param1 = mo.ui.slider(
+        start=0.05, 
+        stop=0.95, 
+        step=0.05, 
+        value=0.5, 
+        label="p (success probability)"
+    )
+
+    dist_param2 = mo.ui.slider(
+        start=1, 
+        stop=20, 
+        step=1, 
+        value=10, 
+        label="n (trials) or λ (rate)"
+    )
+    return dist_param1, dist_param2, dist_selection
+
+
+@app.cell(hide_code=True)
+def _(dist_param1, dist_param2, dist_selection, np, plt, stats):
+    # Set up the plot based on the selected distribution
+    if dist_selection.value == "bernoulli":
+        # Bernoulli distribution
+        dist_p = dist_param1.value
+        dist_x_values = np.array([0, 1])
+        dist_pmf_values = [1-dist_p, dist_p]
+        dist_title = f"Bernoulli PMF (p = {dist_p:.2f})"
+        dist_x_label = "Outcome (0 = Failure, 1 = Success)"
+        dist_max_x = 1
+
+    elif dist_selection.value == "binomial":
+        # Binomial distribution
+        dist_n = int(dist_param2.value)
+        dist_p = dist_param1.value
+        dist_x_values = np.arange(0, dist_n+1)
+        dist_pmf_values = stats.binom.pmf(dist_x_values, dist_n, dist_p)
+        dist_title = f"Binomial PMF (n = {dist_n}, p = {dist_p:.2f})"
+        dist_x_label = "Number of Successes"
+        dist_max_x = dist_n
+
+    elif dist_selection.value == "geometric":
+        # Geometric distribution
+        dist_p = dist_param1.value
+        dist_max_x = min(int(5/dist_p), 50)  # Limit the range for visualization
+        dist_x_values = np.arange(1, dist_max_x+1)
+        dist_pmf_values = stats.geom.pmf(dist_x_values, dist_p)
+        dist_title = f"Geometric PMF (p = {dist_p:.2f})"
+        dist_x_label = "Number of Trials Until First Success"
+
+    else:  # Poisson
+        # Poisson distribution
+        dist_lam = dist_param2.value
+        dist_max_x = int(dist_lam*3) + 1  # Reasonable range for visualization
+        dist_x_values = np.arange(0, dist_max_x)
+        dist_pmf_values = stats.poisson.pmf(dist_x_values, dist_lam)
+        dist_title = f"Poisson PMF (λ = {dist_lam})"
+        dist_x_label = "Number of Events"
+
+    # Create the plot
+    plt.figure(figsize=(10, 5))
+
+    # For discrete distributions, use stem plot for clarity
+    dist_markerline, dist_stemlines, dist_baseline = plt.stem(
+        dist_x_values, dist_pmf_values, markerfmt='o', basefmt=' '
+    )
+    plt.setp(dist_markerline, markersize=6)
+    plt.setp(dist_stemlines, linewidth=1.5)
+
+    # Add a bar plot for better visibility
+    plt.bar(dist_x_values, dist_pmf_values, alpha=0.3, width=0.4)
+
+    plt.xlabel(dist_x_label)
+    plt.ylabel("Probability: P(X = x)")
+    plt.title(dist_title)
+    plt.grid(alpha=0.3)
+
+    # Calculate and display expected value and variance
+    if dist_selection.value == "bernoulli":
+        dist_mean = dist_p
+        dist_variance = dist_p * (1-dist_p)
+    elif dist_selection.value == "binomial":
+        dist_mean = dist_n * dist_p
+        dist_variance = dist_n * dist_p * (1-dist_p)
+    elif dist_selection.value == "geometric":
+        dist_mean = 1/dist_p
+        dist_variance = (1-dist_p)/(dist_p**2)
+    else:  # Poisson
+        dist_mean = dist_lam
+        dist_variance = dist_lam
+
+    dist_std_dev = np.sqrt(dist_variance)
+
+    # Add text with distribution properties
+    dist_props_text = (
+        f"Mean: {dist_mean:.3f}\n"
+        f"Variance: {dist_variance:.3f}\n"
+        f"Std Dev: {dist_std_dev:.3f}\n"
+        f"Sum of probabilities: {sum(dist_pmf_values):.6f}"
+    )
+
+    plt.text(0.95, 0.95, dist_props_text,
+             transform=plt.gca().transAxes,
+             verticalalignment='top',
+             horizontalalignment='right',
+             bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))
+
+    plt.gca()  # Return the current axes to ensure proper display
+    return (
+        dist_baseline,
+        dist_lam,
+        dist_markerline,
+        dist_max_x,
+        dist_mean,
+        dist_n,
+        dist_p,
+        dist_pmf_values,
+        dist_props_text,
+        dist_std_dev,
+        dist_stemlines,
+        dist_title,
+        dist_variance,
+        dist_x_label,
+        dist_x_values,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Expected Value from a PMF
+
+        The expected value (or mean) of a discrete random variable is calculated using its PMF:
+
+        $$E[X] = \sum_x x \cdot p_X(x)$$
+
+        This represents the long-run average value of the random variable.
+        """
+    )
+    return
+
+
+@app.cell
+def _(dist_pmf_values, dist_x_values):
+    def calc_expected_value(x_values, pmf_values):
+        """Calculate the expected value of a discrete random variable."""
+        return sum(x * p for x, p in zip(x_values, pmf_values))
+
+    # Calculate expected value for the current distribution
+    ev_dist_mean = calc_expected_value(dist_x_values, dist_pmf_values)
+
+    print(f"Expected value: {ev_dist_mean:.4f}")
+    return calc_expected_value, ev_dist_mean
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Variance from a PMF
+
+        The variance measures the spread or dispersion of a random variable around its mean:
+
+        $$\text{Var}(X) = E[(X - E[X])^2] = \sum_x (x - E[X])^2 \cdot p_X(x)$$
+
+        An alternative formula is:
+
+        $$\text{Var}(X) = E[X^2] - (E[X])^2 = \sum_x x^2 \cdot p_X(x) - \left(\sum_x x \cdot p_X(x)\right)^2$$
+        """
+    )
+    return
+
+
+@app.cell
+def _(dist_pmf_values, dist_x_values, ev_dist_mean, np):
+    def calc_variance(x_values, pmf_values, mean_value):
+        """Calculate the variance of a discrete random variable."""
+        return sum((x - mean_value)**2 * p for x, p in zip(x_values, pmf_values))
+
+    # Calculate variance for the current distribution
+    var_dist_var = calc_variance(dist_x_values, dist_pmf_values, ev_dist_mean)
+    var_dist_std_dev = np.sqrt(var_dist_var)
+
+    print(f"Variance: {var_dist_var:.4f}")
+    print(f"Standard deviation: {var_dist_std_dev:.4f}")
+    return calc_variance, var_dist_std_dev, var_dist_var
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## PMF vs. CDF
+
+        The **Cumulative Distribution Function (CDF)** is related to the PMF but gives the probability that the random variable $X$ is less than or equal to a value $x$:
+
+        $$F_X(x) = P(X \leq x) = \sum_{k \leq x} p_X(k)$$
+
+        While the PMF gives the probability mass at each point, the CDF accumulates these probabilities.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(dist_pmf_values, dist_x_values, np, plt):
+    # Calculate the CDF from the PMF
+    cdf_dist_values = np.cumsum(dist_pmf_values)
+
+    # Create a plot comparing PMF and CDF
+    cdf_fig, (cdf_ax1, cdf_ax2) = plt.subplots(1, 2, figsize=(12, 4))
+
+    # PMF plot
+    cdf_ax1.bar(dist_x_values, dist_pmf_values, width=0.4, alpha=0.7)
+    cdf_ax1.set_xlabel('x')
+    cdf_ax1.set_ylabel('P(X = x)')
+    cdf_ax1.set_title('Probability Mass Function (PMF)')
+    cdf_ax1.grid(alpha=0.3)
+
+    # CDF plot - using step function with 'post' style for proper discrete representation
+    cdf_ax2.step(dist_x_values, cdf_dist_values, where='post', linewidth=2, color='blue')
+    cdf_ax2.scatter(dist_x_values, cdf_dist_values, s=50, color='blue')
+
+    # Set appropriate limits for better visualization
+    if len(dist_x_values) > 0:
+        x_min = min(dist_x_values) - 0.5
+        x_max = max(dist_x_values) + 0.5
+        cdf_ax2.set_xlim(x_min, x_max)
+        cdf_ax2.set_ylim(0, 1.05)  # CDF goes from 0 to 1
+
+    cdf_ax2.set_xlabel('x')
+    cdf_ax2.set_ylabel('P(X ≤ x)')
+    cdf_ax2.set_title('Cumulative Distribution Function (CDF)')
+    cdf_ax2.grid(alpha=0.3)
+
+    plt.tight_layout()
+    plt.gca()  # Return the current axes to ensure proper display
+    return cdf_ax1, cdf_ax2, cdf_dist_values, cdf_fig, x_max, x_min
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        The graphs above illustrate the key difference between PMF and CDF:
+
+        - **PMF (left)**: Shows the probability of the random variable taking each specific value: P(X = x)
+        - **CDF (right)**: Shows the probability of the random variable being less than or equal to each value: P(X ≤ x)
+
+        The CDF at any point is the sum of all PMF values up to and including that point. This is why the CDF is always non-decreasing and eventually reaches 1. For discrete distributions like this one, the CDF forms a step function that jumps at each value in the support of the random variable.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Test Your Understanding
+
+        Choose what you believe are the correct options in the questions below:
+
+        <details>
+        <summary>If X is a discrete random variable with PMF p(x), then p(x) must always be less than 1</summary>
+        ❌ False! While most values in a PMF are typically less than 1, a PMF can have p(x) = 1 for a specific value if the random variable always takes that value (with 100% probability).
+        </details>
+
+        <details>
+        <summary>The sum of all probabilities in a PMF must equal exactly 1</summary>
+        ✅ True! This is a fundamental property of any valid PMF. The total probability across all possible values must be 1, as the random variable must take some value.
+        </details>
+
+        <details>
+        <summary>A PMF can be estimated from data by creating a normalized histogram</summary>
+        ✅ True! Counting the frequency of each value and dividing by the total number of observations gives an empirical PMF.
+        </details>
+
+        <details>
+        <summary>The expected value of a discrete random variable is always one of the possible values of the variable</summary>
+        ❌ False! The expected value is a weighted average and may not be a value the random variable can actually take. For example, the expected value of a fair die roll is 3.5, which is not a possible outcome.
+        </details>
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Practical Applications of PMFs
+
+        PMFs pop up everywhere - network engineers use them to model traffic patterns, reliability teams predict equipment failures, and marketers analyze purchase behavior. In finance, they help price options; in gaming, they're behind every dice roll. Machine learning algorithms like Naive Bayes rely on them, and they're essential for modeling rare events like genetic mutations or system failures.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Key Takeaways
+
+        PMFs give us the probability picture for discrete random variables - they tell us how likely each value is, must be non-negative, and always sum to 1. We can write them as equations, draw them as graphs, or estimate them from data. They're the foundation for calculating expected values and variances, which we'll explore in our next notebook on Expectation, where we'll learn how to summarize random variables with a single, most "expected" value.
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell
+def _():
+    import matplotlib.pyplot as plt
+    import numpy as np
+    from scipy import stats
+    import collections
+    return collections, np, plt, stats
+
+
+if __name__ == "__main__":
+    app.run()

probability/11_expectation.py

@@ -0,0 +1,860 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.3",
+#     "scipy==1.15.2",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.19"
+app = marimo.App(width="medium", app_title="Expectation")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Expectation
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/expectation/), by Stanford professor Chris Piech._
+
+        A random variable is fully represented by its Probability Mass Function (PMF), which describes each value the random variable can take on and the corresponding probabilities. However, a PMF can contain a lot of information. Sometimes it's useful to summarize a random variable with a single value!
+
+        The most common, and arguably the most useful, summary of a random variable is its **Expectation** (also called the expected value or mean).
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Definition of Expectation
+
+        The expectation of a random variable $X$, written $E[X]$, is the average of all the values the random variable can take on, each weighted by the probability that the random variable will take on that value.
+
+        $$E[X] = \sum_x x \cdot P(X=x)$$
+
+        Expectation goes by many other names: Mean, Weighted Average, Center of Mass, 1st Moment. All of these are calculated using the same formula.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Intuition Behind Expectation
+
+        The expected value represents the long-run average value of a random variable over many independent repetitions of an experiment.
+
+        For example, if you roll a fair six-sided die many times and calculate the average of all rolls, that average will approach the expected value of 3.5 as the number of rolls increases.
+
+        Let's visualize this concept:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(np, plt):
+    # Set random seed for reproducibility
+    np.random.seed(42)
+
+    # Simulate rolling a die many times
+    exp_num_rolls = 1000
+    exp_die_rolls = np.random.randint(1, 7, size=exp_num_rolls)
+
+    # Calculate the running average
+    exp_running_avg = np.cumsum(exp_die_rolls) / np.arange(1, exp_num_rolls + 1)
+
+    # Create the plot
+    plt.figure(figsize=(10, 5))
+    plt.plot(range(1, exp_num_rolls + 1), exp_running_avg, label='Running Average')
+    plt.axhline(y=3.5, color='r', linestyle='--', label='Expected Value (3.5)')
+    plt.xlabel('Number of Rolls')
+    plt.ylabel('Average Value')
+    plt.title('Running Average of Die Rolls Approaching Expected Value')
+    plt.legend()
+    plt.grid(alpha=0.3)
+    plt.xscale('log')  # Log scale to better see convergence
+
+    # Add annotations
+    plt.annotate('As the number of rolls increases,\nthe average approaches the expected value',
+                xy=(exp_num_rolls, exp_running_avg[-1]), xytext=(exp_num_rolls/3, 4),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1.5))
+
+    plt.gca()
+    return exp_die_rolls, exp_num_rolls, exp_running_avg
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""## Properties of Expectation""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.accordion(
+        {
+            "1. Linearity of Expectation": mo.md(
+                r"""
+                $$E[aX + b] = a \cdot E[X] + b$$
+
+                Where $a$ and $b$ are constants (not random variables).
+
+                This means that if you multiply a random variable by a constant, the expectation is multiplied by that constant. And if you add a constant to a random variable, the expectation increases by that constant.
+                """
+            ),
+            "2. Expectation of the Sum of Random Variables": mo.md(
+                r"""
+                $$E[X + Y] = E[X] + E[Y]$$
+
+                This is true regardless of the relationship between $X$ and $Y$. They can be dependent, and they can have different distributions. This also applies with more than two random variables:
+
+                $$E\left[\sum_{i=1}^n X_i\right] = \sum_{i=1}^n E[X_i]$$
+                """
+            ),
+            "3. Law of the Unconscious Statistician (LOTUS)": mo.md(
+                r"""
+                $$E[g(X)] = \sum_x g(x) \cdot P(X=x)$$
+
+                This allows us to calculate the expected value of a function $g(X)$ of a random variable $X$ when we know the probability distribution of $X$ but don't explicitly know the distribution of $g(X)$.
+
+                This theorem has the humorous name "Law of the Unconscious Statistician" (LOTUS) because it's so useful that you should be able to employ it unconsciously.
+                """
+            ),
+            "4. Expectation of a Constant": mo.md(
+                r"""
+                $$E[a] = a$$
+
+                Sometimes in proofs, you'll end up with the expectation of a constant (rather than a random variable). Since a constant doesn't change, its expected value is just the constant itself.
+                """
+            ),
+        }
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Calculating Expectation
+
+        Let's calculate the expected value for some common examples:
+
+        ### Example 1: Fair Die Roll
+
+        For a fair six-sided die, the PMF is:
+
+        $$P(X=x) = \frac{1}{6} \text{ for } x \in \{1, 2, 3, 4, 5, 6\}$$
+
+        The expected value is:
+
+        $$E[X] = 1 \cdot \frac{1}{6} + 2 \cdot \frac{1}{6} + 3 \cdot \frac{1}{6} + 4 \cdot \frac{1}{6} + 5 \cdot \frac{1}{6} + 6 \cdot \frac{1}{6} = \frac{21}{6} = 3.5$$
+
+        Let's implement this calculation in Python:
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    def calc_expectation_die():
+        """Calculate the expected value of a fair six-sided die roll."""
+        exp_die_values = range(1, 7)
+        exp_die_probs = [1/6] * 6
+
+        exp_die_expected = sum(x * p for x, p in zip(exp_die_values, exp_die_probs))
+        return exp_die_expected
+
+    exp_die_result = calc_expectation_die()
+    print(f"Expected value of a fair die roll: {exp_die_result}")
+    return calc_expectation_die, exp_die_result
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Example 2: Sum of Two Dice
+
+        Now let's calculate the expected value for the sum of two fair dice. First, we need the PMF:
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    def pmf_sum_two_dice(y_val):
+        """Returns the probability that the sum of two dice is y."""
+        # Count the number of ways to get sum y
+        exp_count = 0
+        for dice1 in range(1, 7):
+            for dice2 in range(1, 7):
+                if dice1 + dice2 == y_val:
+                    exp_count += 1
+        return exp_count / 36  # There are 36 possible outcomes (6×6)
+
+    # Test the function for a few values
+    exp_test_values = [2, 7, 12]
+    for exp_test_y in exp_test_values:
+        print(f"P(Y = {exp_test_y}) = {pmf_sum_two_dice(exp_test_y)}")
+    return exp_test_values, exp_test_y, pmf_sum_two_dice
+
+
+@app.cell
+def _(pmf_sum_two_dice):
+    def calc_expectation_sum_two_dice():
+        """Calculate the expected value of the sum of two dice."""
+        exp_sum_two_dice = 0
+        # Sum of dice can take on the values 2 through 12
+        for exp_x in range(2, 13):
+            exp_pr_x = pmf_sum_two_dice(exp_x)  # PMF gives P(sum is x)
+            exp_sum_two_dice += exp_x * exp_pr_x
+        return exp_sum_two_dice
+
+    exp_sum_result = calc_expectation_sum_two_dice()
+
+    # Round to 2 decimal places for display
+    exp_sum_result_rounded = round(exp_sum_result, 2)
+
+    print(f"Expected value of the sum of two dice: {exp_sum_result_rounded}")
+
+    # Let's also verify this with a direct calculation
+    exp_direct_calc = sum(x * pmf_sum_two_dice(x) for x in range(2, 13))
+    exp_direct_calc_rounded = round(exp_direct_calc, 2)
+
+    print(f"Direct calculation: {exp_direct_calc_rounded}")
+
+    # Verify that this equals 7
+    print(f"Is the expected value exactly 7? {abs(exp_sum_result - 7) < 1e-10}")
+    return (
+        calc_expectation_sum_two_dice,
+        exp_direct_calc,
+        exp_direct_calc_rounded,
+        exp_sum_result,
+        exp_sum_result_rounded,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Visualizing Expectation
+
+        Let's visualize the expectation for the sum of two dice. The expected value is the "center of mass" of the PMF:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(plt, pmf_sum_two_dice):
+    # Create the visualization
+    exp_y_values = list(range(2, 13))
+    exp_probabilities = [pmf_sum_two_dice(y) for y in exp_y_values]
+
+    dice_fig, dice_ax = plt.subplots(figsize=(10, 5))
+    dice_ax.bar(exp_y_values, exp_probabilities, width=0.4)
+    dice_ax.axvline(x=7, color='r', linestyle='--', linewidth=2, label='Expected Value (7)')
+
+    dice_ax.set_xticks(exp_y_values)
+    dice_ax.set_xlabel('Sum of two dice (y)')
+    dice_ax.set_ylabel('Probability: P(Y = y)')
+    dice_ax.set_title('PMF of Sum of Two Dice with Expected Value')
+    dice_ax.grid(alpha=0.3)
+    dice_ax.legend()
+
+    # Add probability values on top of bars
+    for exp_i, exp_prob in enumerate(exp_probabilities):
+        dice_ax.text(exp_y_values[exp_i], exp_prob + 0.001, f'{exp_prob:.3f}', ha='center')
+
+    plt.tight_layout()
+    plt.gca()
+    return dice_ax, dice_fig, exp_i, exp_prob, exp_probabilities, exp_y_values
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Demonstrating the Properties of Expectation
+
+        Let's demonstrate some of these properties with examples:
+        """
+    )
+    return
+
+
+@app.cell
+def _(exp_die_result):
+    # Demonstrate linearity of expectation (1)
+    # E[aX + b] = a*E[X] + b
+
+    # For a die roll X with E[X] = 3.5
+    prop_a = 2
+    prop_b = 10
+
+    # Calculate E[2X + 10] using the property
+    prop_expected_using_property = prop_a * exp_die_result + prop_b
+    prop_expected_using_property_rounded = round(prop_expected_using_property, 2)
+
+    print(f"Using linearity property: E[{prop_a}X + {prop_b}] = {prop_a} * E[X] + {prop_b} = {prop_expected_using_property_rounded}")
+
+    # Calculate E[2X + 10] directly
+    prop_expected_direct = sum((prop_a * x + prop_b) * (1/6) for x in range(1, 7))
+    prop_expected_direct_rounded = round(prop_expected_direct, 2)
+
+    print(f"Direct calculation: E[{prop_a}X + {prop_b}] = {prop_expected_direct_rounded}")
+
+    # Verify they match
+    print(f"Do they match? {abs(prop_expected_using_property - prop_expected_direct) < 1e-10}")
+    return (
+        prop_a,
+        prop_b,
+        prop_expected_direct,
+        prop_expected_direct_rounded,
+        prop_expected_using_property,
+        prop_expected_using_property_rounded,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ### Law of the Unconscious Statistician (LOTUS)
+
+        Let's use LOTUS to calculate $E[X^2]$ for a die roll, which will be useful when we study variance:
+        """
+    )
+    return
+
+
+@app.cell
+def _():
+    # Calculate E[X^2] for a die roll using LOTUS (3)
+    lotus_die_values = range(1, 7)
+    lotus_die_probs = [1/6] * 6
+
+    # Using LOTUS: E[X^2] = sum(x^2 * P(X=x))
+    lotus_expected_x_squared = sum(x**2 * p for x, p in zip(lotus_die_values, lotus_die_probs))
+    lotus_expected_x_squared_rounded = round(lotus_expected_x_squared, 2)
+
+    expected_x_squared = 3.5**2
+    expected_x_squared_rounded = round(expected_x_squared, 2)
+
+    print(f"E[X^2] for a die roll = {lotus_expected_x_squared_rounded}")
+    print(f"(E[X])^2 for a die roll = {expected_x_squared_rounded}")
+    return (
+        expected_x_squared,
+        expected_x_squared_rounded,
+        lotus_die_probs,
+        lotus_die_values,
+        lotus_expected_x_squared,
+        lotus_expected_x_squared_rounded,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// Note
+        Note that E[X^2] != (E[X])^2
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Interactive Example
+
+        Let's explore how the expected value changes as we adjust the parameters of common probability distributions. This interactive visualization focuses specifically on the relationship between distribution parameters and expected values.
+
+        Use the controls below to select a distribution and adjust its parameters. The graph will show how the expected value changes across a range of parameter values.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    # Create UI elements for distribution selection
+    dist_selection = mo.ui.dropdown(
+        options=[
+            "bernoulli",
+            "binomial",
+            "geometric",
+            "poisson"
+        ],
+        value="bernoulli",
+        label="Select a distribution"
+    )
+    return (dist_selection,)
+
+
+@app.cell(hide_code=True)
+def _(dist_selection):
+    dist_selection.center()
+    return
+
+
+@app.cell(hide_code=True)
+def _(dist_description):
+    dist_description
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md("""### Adjust Parameters""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(controls):
+    controls
+    return
+
+
+@app.cell(hide_code=True)
+def _(
+    dist_selection,
+    lambda_range,
+    np,
+    param_lambda,
+    param_n,
+    param_p,
+    param_range,
+    plt,
+):
+    # Calculate expected values based on the selected distribution
+    if dist_selection.value == "bernoulli":
+        # Get parameter range for visualization
+        p_min, p_max = param_range.value
+        param_values = np.linspace(p_min, p_max, 100)
+
+        # E[X] = p for Bernoulli
+        expected_values = param_values
+        current_param = param_p.value
+        current_expected = round(current_param, 2)
+        x_label = "p (probability of success)"
+        title = "Expected Value of Bernoulli Distribution"
+        formula = "E[X] = p"
+
+    elif dist_selection.value == "binomial":
+        p_min, p_max = param_range.value
+        param_values = np.linspace(p_min, p_max, 100)
+
+        # E[X] = np for Binomial
+        n = int(param_n.value)
+        expected_values = [n * p for p in param_values]
+        current_param = param_p.value
+        current_expected = round(n * current_param, 2)
+        x_label = "p (probability of success)"
+        title = f"Expected Value of Binomial Distribution (n={n})"
+        formula = f"E[X] = n × p = {n} × p"
+
+    elif dist_selection.value == "geometric":
+        p_min, p_max = param_range.value
+        # Ensure p is not 0 for geometric distribution
+        p_min = max(0.01, p_min)
+        param_values = np.linspace(p_min, p_max, 100)
+
+        # E[X] = 1/p for Geometric
+        expected_values = [1/p for p in param_values]
+        current_param = param_p.value
+        current_expected = round(1 / current_param, 2)
+        x_label = "p (probability of success)"
+        title = "Expected Value of Geometric Distribution"
+        formula = "E[X] = 1/p"
+
+    else:  # Poisson
+        lambda_min, lambda_max = lambda_range.value
+        param_values = np.linspace(lambda_min, lambda_max, 100)
+
+        # E[X] = lambda for Poisson
+        expected_values = param_values
+        current_param = param_lambda.value
+        current_expected = round(current_param, 2)
+        x_label = "λ (rate parameter)"
+        title = "Expected Value of Poisson Distribution"
+        formula = "E[X] = λ"
+
+    # Create the plot
+    dist_fig, dist_ax = plt.subplots(figsize=(10, 6))
+
+    # Plot the expected value function
+    dist_ax.plot(param_values, expected_values, 'b-', linewidth=2, label="Expected Value Function")
+
+    dist_ax.plot(current_param, current_expected, 'ro', markersize=10, label=f"Current Value: E[X] = {current_expected}")
+
+    dist_ax.hlines(current_expected, param_values[0], current_param, colors='r', linestyles='dashed')
+
+    dist_ax.vlines(current_param, 0, current_expected, colors='r', linestyles='dashed')
+
+    dist_ax.fill_between(param_values, 0, expected_values, alpha=0.2, color='blue')
+
+    dist_ax.set_xlabel(x_label, fontsize=12)
+    dist_ax.set_ylabel("Expected Value: E[X]", fontsize=12)
+    dist_ax.set_title(title, fontsize=14, fontweight='bold')
+    dist_ax.grid(True, alpha=0.3)
+
+    # Move legend to lower right to avoid overlap with formula
+    dist_ax.legend(loc='lower right', fontsize=10)
+
+    # Add formula text box in upper left
+    dist_props = dict(boxstyle='round', facecolor='white', alpha=0.8)
+    dist_ax.text(0.02, 0.95, formula, transform=dist_ax.transAxes, fontsize=12,
+            verticalalignment='top', bbox=dist_props)
+
+    if dist_selection.value == "geometric":
+        max_y = min(50, 2/max(0.01, param_values[0]))
+        dist_ax.set_ylim(0, max_y)
+    elif dist_selection.value == "binomial":
+        dist_ax.set_ylim(0, int(param_n.value) + 1)
+    else:
+        dist_ax.set_ylim(0, max(expected_values) * 1.1)
+
+    annotation_x = current_param + (param_values[-1] - param_values[0]) * 0.05
+    annotation_y = current_expected
+
+    # Adjust annotation position if it would go off the chart
+    if annotation_x > param_values[-1] * 0.9:
+        annotation_x = current_param - (param_values[-1] - param_values[0]) * 0.2
+
+    dist_ax.annotate(
+        f"Parameter: {current_param:.2f}\nE[X] = {current_expected}",
+        xy=(current_param, current_expected),
+        xytext=(annotation_x, annotation_y),
+        arrowprops=dict(facecolor='black', shrink=0.05, width=1.5, alpha=0.7),
+        bbox=dist_props
+    )
+
+    plt.tight_layout()
+    plt.gca()
+    return (
+        annotation_x,
+        annotation_y,
+        current_expected,
+        current_param,
+        dist_ax,
+        dist_fig,
+        dist_props,
+        expected_values,
+        formula,
+        lambda_max,
+        lambda_min,
+        max_y,
+        n,
+        p_max,
+        p_min,
+        param_values,
+        title,
+        x_label,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Expectation vs. Mode
+
+        The expected value (mean) of a random variable is not always the same as its most likely value (mode). Let's explore this with an example:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(np, plt, stats):
+    # Create a skewed distribution
+    skew_n = 10
+    skew_p = 0.25
+
+    # Binomial PMF
+    skew_x_values = np.arange(0, skew_n+1)
+    skew_pmf_values = stats.binom.pmf(skew_x_values, skew_n, skew_p)
+
+    # Find the mode (most likely value)
+    skew_mode = skew_x_values[np.argmax(skew_pmf_values)]
+
+    # Calculate the expected value
+    skew_expected = skew_n * skew_p
+    skew_expected_rounded = round(skew_expected, 2)
+
+    skew_fig, skew_ax = plt.subplots(figsize=(10, 5))
+    skew_ax.bar(skew_x_values, skew_pmf_values, alpha=0.7, width=0.4)
+
+    # Add vertical lines for mode and expected value
+    skew_ax.axvline(x=skew_mode, color='g', linestyle='--', linewidth=2, 
+                label=f'Mode = {skew_mode} (Most likely value)')
+    skew_ax.axvline(x=skew_expected, color='r', linestyle='--', linewidth=2, 
+                label=f'Expected Value = {skew_expected_rounded} (Mean)')
+
+    skew_ax.annotate('Mode', xy=(skew_mode, 0.05), xytext=(skew_mode-2.0, 0.1),
+                arrowprops=dict(facecolor='green', shrink=0.05, width=1.5), color='green')
+    skew_ax.annotate('Expected Value', xy=(skew_expected, 0.05), xytext=(skew_expected+1, 0.15),
+                arrowprops=dict(facecolor='red', shrink=0.05, width=1.5), color='red')
+
+    if skew_mode != int(skew_expected):
+        min_x = min(skew_mode, skew_expected)
+        max_x = max(skew_mode, skew_expected)
+        skew_ax.axvspan(min_x, max_x, alpha=0.2, color='purple')
+
+        # Add text explaining the difference
+        mid_x = (skew_mode + skew_expected) / 2
+        skew_ax.text(mid_x, max(skew_pmf_values) * 0.5, 
+                 f"Difference: {abs(skew_mode - skew_expected_rounded):.2f}",
+                 ha='center', va='center', bbox=dict(facecolor='white', alpha=0.7))
+
+    skew_ax.set_xlabel('Number of Successes')
+    skew_ax.set_ylabel('Probability')
+    skew_ax.set_title(f'Binomial Distribution (n={skew_n}, p={skew_p})')
+    skew_ax.grid(alpha=0.3)
+    skew_ax.legend()
+
+    plt.tight_layout()
+    plt.gca()
+    return (
+        max_x,
+        mid_x,
+        min_x,
+        skew_ax,
+        skew_expected,
+        skew_expected_rounded,
+        skew_fig,
+        skew_mode,
+        skew_n,
+        skew_p,
+        skew_pmf_values,
+        skew_x_values,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// NOTE
+        For the sum of two dice we calculated earlier, we found the expected value to be exactly 7. In that case, 7 also happens to be the mode (most likely outcome) of the distribution. However, this is just a coincidence for this particular example!
+
+        As we can see from the binomial distribution above, the expected value (2.50) and the mode (2) are often different values (this is common in skewed distributions). The expected value represents the "center of mass" of the distribution, while the mode represents the most likely single outcome.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🤔 Test Your Understanding
+
+        Choose what you believe are the correct options in the questions below:
+
+        <details>
+        <summary>The expected value of a random variable is always one of the possible values the random variable can take.</summary>
+        ❌ False! The expected value is a weighted average and may not be a value the random variable can actually take. For example, the expected value of a fair die roll is 3.5, which is not a possible outcome.
+        </details>
+
+        <details>
+        <summary>If X and Y are independent random variables, then E[X·Y] = E[X]·E[Y].</summary>
+        ✅ True! For independent random variables, the expectation of their product equals the product of their expectations.
+        </details>
+
+        <details>
+        <summary>The expected value of a constant random variable (one that always takes the same value) is that constant.</summary>
+        ✅ True! If X = c with probability 1, then E[X] = c.
+        </details>
+
+        <details>
+        <summary>The expected value of the sum of two random variables is always the sum of their expected values, regardless of whether they are independent.</summary>
+        ✅ True! This is the linearity of expectation property: E[X + Y] = E[X] + E[Y], which holds regardless of dependence.
+        </details>
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Practical Applications of Expectation
+
+        Expected values show up everywhere - from investment decisions and insurance pricing to machine learning algorithms and game design. Engineers use them to predict system reliability, data scientists to understand customer behavior, and economists to model market outcomes. They're essential for risk assessment in project management and for optimizing resource allocation in operations research.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Key Takeaways
+
+        Expectation gives us a single value that summarizes a random variable's central tendency - it's the weighted average of all possible outcomes, where the weights are probabilities. The linearity property makes expectations easy to work with, even for complex combinations of random variables. While a PMF gives the complete probability picture, expectation provides an essential summary that helps us make decisions under uncertainty. In our next notebook, we'll explore variance, which measures how spread out a random variable's values are around its expectation.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""#### Appendix (containing helper code)""")
+    return
+
+
+@app.cell(hide_code=True)
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import matplotlib.pyplot as plt
+    import numpy as np
+    from scipy import stats
+    import collections
+    return collections, np, plt, stats
+
+
+@app.cell(hide_code=True)
+def _(dist_selection, mo):
+    # Parameter controls for probability-based distributions
+    param_p = mo.ui.slider(
+        start=0.01, 
+        stop=0.99, 
+        step=0.01, 
+        value=0.5, 
+        label="p (probability of success)",
+        full_width=True
+    )
+
+    # Parameter control for binomial distribution
+    param_n = mo.ui.slider(
+        start=1, 
+        stop=50, 
+        step=1, 
+        value=10, 
+        label="n (number of trials)",
+        full_width=True
+    )
+
+    # Parameter control for Poisson distribution
+    param_lambda = mo.ui.slider(
+        start=0.1, 
+        stop=20, 
+        step=0.1, 
+        value=5, 
+        label="λ (rate parameter)",
+        full_width=True
+    )
+
+    # Parameter range sliders for visualization
+    param_range = mo.ui.range_slider(
+        start=0, 
+        stop=1, 
+        step=0.01, 
+        value=[0, 1], 
+        label="Parameter range to visualize",
+        full_width=True
+    )
+
+    lambda_range = mo.ui.range_slider(
+        start=0, 
+        stop=20, 
+        step=0.1, 
+        value=[0, 20], 
+        label="λ range to visualize",
+        full_width=True
+    )
+
+    # Display appropriate controls based on the selected distribution
+    if dist_selection.value == "bernoulli":
+        controls = mo.hstack([param_p, param_range], justify="space-around")
+    elif dist_selection.value == "binomial":
+        controls = mo.hstack([param_p, param_n, param_range], justify="space-around")
+    elif dist_selection.value == "geometric":
+        controls = mo.hstack([param_p, param_range], justify="space-around")
+    else:  # poisson
+        controls = mo.hstack([param_lambda, lambda_range], justify="space-around")
+    return controls, lambda_range, param_lambda, param_n, param_p, param_range
+
+
+@app.cell(hide_code=True)
+def _(dist_selection, mo):
+    # Create distribution descriptions based on selection
+    if dist_selection.value == "bernoulli":
+        dist_description = mo.md(
+            r"""
+            **Bernoulli Distribution**
+
+            A Bernoulli distribution models a single trial with two possible outcomes: success (1) or failure (0).
+
+            - Parameter: $p$ = probability of success
+            - Expected Value: $E[X] = p$
+            - Example: Flipping a coin once (p = 0.5 for a fair coin)
+            """
+        )
+    elif dist_selection.value == "binomial":
+        dist_description = mo.md(
+            r"""
+            **Binomial Distribution**
+
+            A Binomial distribution models the number of successes in $n$ independent trials.
+
+            - Parameters: $n$ = number of trials, $p$ = probability of success
+            - Expected Value: $E[X] = np$
+            - Example: Number of heads in 10 coin flips
+            """
+        )
+    elif dist_selection.value == "geometric":
+        dist_description = mo.md(
+            r"""
+            **Geometric Distribution**
+
+            A Geometric distribution models the number of trials until the first success.
+
+            - Parameter: $p$ = probability of success
+            - Expected Value: $E[X] = \frac{1}{p}$
+            - Example: Number of coin flips until first heads
+            """
+        )
+    else:  # poisson
+        dist_description = mo.md(
+            r"""
+            **Poisson Distribution**
+
+            A Poisson distribution models the number of events occurring in a fixed interval.
+
+            - Parameter: $\lambda$ = average rate of events
+            - Expected Value: $E[X] = \lambda$
+            - Example: Number of emails received per hour
+            """
+        )
+    return (dist_description,)
+
+
+if __name__ == "__main__":
+    app.run()

probability/12_variance.py

@@ -0,0 +1,631 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.3",
+#     "scipy==1.15.2",
+#     "wigglystuff==0.1.10",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.20"
+app = marimo.App(width="medium", app_title="Variance")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Variance
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/variance/), by Stanford professor Chris Piech._
+
+        In our previous exploration of random variables, we learned about expectation - a measure of central tendency. However, knowing the average value alone doesn't tell us everything about a distribution. Consider these questions:
+
+        - How spread out are the values around the mean?
+        - How reliable is the expectation as a predictor of individual outcomes?
+        - How much do individual samples typically deviate from the average?
+
+        This is where **variance** comes in - it measures the spread or dispersion of a random variable around its expected value.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Definition of Variance
+
+        The variance of a random variable $X$ with expected value $\mu = E[X]$ is defined as:
+
+        $$\text{Var}(X) = E[(X-\mu)^2]$$
+
+        This definition captures the average squared deviation from the mean. There's also an equivalent, often more convenient formula:
+
+        $$\text{Var}(X) = E[X^2] - (E[X])^2$$
+
+        /// tip
+        The second formula is usually easier to compute, as it only requires calculating $E[X^2]$ and $E[X]$, rather than working with deviations from the mean.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Intuition Through Example
+
+        Let's look at a real-world example that illustrates why variance is important. Consider three different groups of graders evaluating assignments in a massive online course. Each grader has their own "grading distribution" - their pattern of assigning scores to work that deserves a 70/100.
+
+        The visualization below shows the probability distributions for three types of graders. Try clicking and dragging the blue numbers to adjust the parameters and see how they affect the variance.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// TIP
+        Try adjusting the blue numbers above to see how:
+
+        - Increasing spread increases variance
+        - The mixture ratio affects how many outliers appear in Grader C's distribution
+        - Changing the true grade shifts all distributions but maintains their relative variances
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(controls):
+    controls
+    return
+
+
+@app.cell(hide_code=True)
+def _(
+    grader_a_spread,
+    grader_b_spread,
+    grader_c_mix,
+    np,
+    plt,
+    stats,
+    true_grade,
+):
+    # Create data for three grader distributions
+    _grader_x = np.linspace(40, 100, 200)
+
+    # Calculate actual variances
+    var_a = grader_a_spread.amount**2
+    var_b = grader_b_spread.amount**2
+    var_c = (1-grader_c_mix.amount) * 3**2 + grader_c_mix.amount * 8**2 + \
+            grader_c_mix.amount * (1-grader_c_mix.amount) * (8-3)**2  # Mixture variance formula
+
+    # Grader A: Wide spread around true grade
+    grader_a = stats.norm.pdf(_grader_x, loc=true_grade.amount, scale=grader_a_spread.amount)
+
+    # Grader B: Narrow spread around true grade
+    grader_b = stats.norm.pdf(_grader_x, loc=true_grade.amount, scale=grader_b_spread.amount)
+
+    # Grader C: Mixture of distributions
+    grader_c = (1-grader_c_mix.amount) * stats.norm.pdf(_grader_x, loc=true_grade.amount, scale=3) + \
+               grader_c_mix.amount * stats.norm.pdf(_grader_x, loc=true_grade.amount, scale=8)
+
+    grader_fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15, 5))
+
+    # Plot each distribution
+    ax1.fill_between(_grader_x, grader_a, alpha=0.3, color='green', label=f'Var ≈ {var_a:.2f}')
+    ax1.axvline(x=true_grade.amount, color='black', linestyle='--', label='True Grade')
+    ax1.set_title('Grader A: High Variance')
+    ax1.set_xlabel('Grade')
+    ax1.set_ylabel('Pr(G = g)')
+    ax1.set_ylim(0, max(grader_a)*1.1)
+
+    ax2.fill_between(_grader_x, grader_b, alpha=0.3, color='blue', label=f'Var ≈ {var_b:.2f}')
+    ax2.axvline(x=true_grade.amount, color='black', linestyle='--')
+    ax2.set_title('Grader B: Low Variance')
+    ax2.set_xlabel('Grade')
+    ax2.set_ylim(0, max(grader_b)*1.1)
+
+    ax3.fill_between(_grader_x, grader_c, alpha=0.3, color='purple', label=f'Var ≈ {var_c:.2f}')
+    ax3.axvline(x=true_grade.amount, color='black', linestyle='--')
+    ax3.set_title('Grader C: Mixed Distribution')
+    ax3.set_xlabel('Grade')
+    ax3.set_ylim(0, max(grader_c)*1.1)
+
+    # Add annotations to explain what's happening
+    ax1.annotate('Wide spread = high variance', 
+                xy=(true_grade.amount, max(grader_a)*0.5),
+                xytext=(true_grade.amount-15, max(grader_a)*0.7),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    ax2.annotate('Narrow spread = low variance', 
+                xy=(true_grade.amount, max(grader_b)*0.5),
+                xytext=(true_grade.amount+8, max(grader_b)*0.7),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    ax3.annotate('Mixture creates outliers', 
+                xy=(true_grade.amount+15, grader_c[np.where(_grader_x >= true_grade.amount+15)[0][0]]),
+                xytext=(true_grade.amount+5, max(grader_c)*0.7),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    # Add legends and adjust layout
+    for _ax in [ax1, ax2, ax3]:
+        _ax.legend()
+        _ax.grid(alpha=0.2)
+
+    plt.tight_layout()
+    plt.gca()
+    return (
+        ax1,
+        ax2,
+        ax3,
+        grader_a,
+        grader_b,
+        grader_c,
+        grader_fig,
+        var_a,
+        var_b,
+        var_c,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// note
+        All three distributions have the same expected value (the true grade), but they differ significantly in their spread:
+
+        - **Grader A** has high variance - grades vary widely from the true value
+        - **Grader B** has low variance - grades consistently stay close to the true value
+        - **Grader C** has a mixture distribution - mostly consistent but with occasional extreme values
+
+        This illustrates why variance is crucial: two distributions can have the same mean but behave very differently in practice.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Computing Variance
+
+        Let's work through some concrete examples to understand how to calculate variance.
+
+        ### Example 1: Fair Die Roll
+
+        Consider rolling a fair six-sided die. We'll calculate its variance step by step:
+        """
+    )
+    return
+
+
+@app.cell
+def _(np):
+    # Define the die values and probabilities
+    die_values = np.array([1, 2, 3, 4, 5, 6])
+    die_probs = np.array([1/6] * 6)
+
+    # Calculate E[X]
+    expected_value = np.sum(die_values * die_probs)
+
+    # Calculate E[X^2]
+    expected_square = np.sum(die_values**2 * die_probs)
+
+    # Calculate Var(X) = E[X^2] - (E[X])^2
+    variance = expected_square - expected_value**2
+
+    # Calculate standard deviation
+    std_dev = np.sqrt(variance)
+
+    print(f"E[X] = {expected_value:.2f}")
+    print(f"E[X^2] = {expected_square:.2f}")
+    print(f"Var(X) = {variance:.2f}")
+    print(f"Standard Deviation = {std_dev:.2f}")
+    return (
+        die_probs,
+        die_values,
+        expected_square,
+        expected_value,
+        std_dev,
+        variance,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// NOTE
+        For a fair die:
+
+        - The expected value (3.50) tells us the average roll
+        - The variance (2.92) tells us how much typical rolls deviate from this average
+        - The standard deviation (1.71) gives us this spread in the original units
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Properties of Variance
+
+        Variance has several important properties that make it useful for analyzing random variables:
+
+        1. **Non-negativity**: $\text{Var}(X) \geq 0$ for any random variable $X$
+        2. **Variance of a constant**: $\text{Var}(c) = 0$ for any constant $c$
+        3. **Scaling**: $\text{Var}(aX) = a^2\text{Var}(X)$ for any constant $a$
+        4. **Translation**: $\text{Var}(X + b) = \text{Var}(X)$ for any constant $b$
+        5. **Independence**: If $X$ and $Y$ are independent, then $\text{Var}(X + Y) = \text{Var}(X) + \text{Var}(Y)$
+
+        Let's verify a property with an example.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Proof of Variance Formula
+
+        The equivalence of the two variance formulas is a fundamental result in probability theory. Here's the proof:
+
+        Starting with the definition $\text{Var}(X) = E[(X-\mu)^2]$ where $\mu = E[X]$:
+
+        \begin{align}
+        \text{Var}(X) &= E[(X-\mu)^2] \\
+        &= \sum_x(x-\mu)^2P(x) && \text{Definition of Expectation}\\
+        &= \sum_x (x^2 -2\mu x + \mu^2)P(x) && \text{Expanding the square}\\
+        &= \sum_x x^2P(x)- 2\mu \sum_x xP(x) + \mu^2 \sum_x P(x) && \text{Distributing the sum}\\
+        &= E[X^2]- 2\mu E[X] + \mu^2 && \text{Definition of expectation}\\
+        &= E[X^2]- 2(E[X])^2 + (E[X])^2 && \text{Since }\mu = E[X]\\
+        &= E[X^2]- (E[X])^2 && \text{Simplifying}
+        \end{align}
+
+        /// tip
+        This proof shows why the formula $\text{Var}(X) = E[X^2] - (E[X])^2$ is so useful - it's much easier to compute $E[X^2]$ and $E[X]$ separately than to work with deviations directly.
+        """
+    )
+    return
+
+
+@app.cell
+def _(die_probs, die_values, np):
+    # Demonstrate scaling property
+    a = 2  # Scale factor
+
+    # Original variance
+    original_var = np.sum(die_values**2 * die_probs) - (np.sum(die_values * die_probs))**2
+
+    # Scaled random variable variance
+    scaled_values = a * die_values
+    scaled_var = np.sum(scaled_values**2 * die_probs) - (np.sum(scaled_values * die_probs))**2
+
+    print(f"Original Variance: {original_var:.2f}")
+    print(f"Scaled Variance (a={a}): {scaled_var:.2f}")
+    print(f"a^2 * Original Variance: {a**2 * original_var:.2f}")
+    print(f"Property holds: {abs(scaled_var - a**2 * original_var) < 1e-10}")
+    return a, original_var, scaled_values, scaled_var
+
+
+@app.cell
+def _():
+    # DIY : Prove more properties as shown above
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Standard Deviation
+
+        While variance is mathematically convenient, it has one practical drawback: its units are squared. For example, if we're measuring grades (0-100), the variance is in "grade points squared." This makes it hard to interpret intuitively.
+
+        The **standard deviation**, denoted by $\sigma$ or $\text{SD}(X)$, is the square root of variance:
+
+        $$\sigma = \sqrt{\text{Var}(X)}$$
+
+        /// tip
+        Standard deviation is often more intuitive because it's in the same units as the original data. For a normal distribution, approximately:
+        - 68% of values fall within 1 standard deviation of the mean
+        - 95% of values fall within 2 standard deviations
+        - 99.7% of values fall within 3 standard deviations
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(controls1):
+    controls1
+    return
+
+
+@app.cell(hide_code=True)
+def _(TangleSlider, mo):
+    normal_mean = mo.ui.anywidget(TangleSlider(
+        amount=0, 
+        min_value=-5, 
+        max_value=5, 
+        step=0.5,
+        digits=1,
+        suffix=" units"
+    ))
+
+    normal_std = mo.ui.anywidget(TangleSlider(
+        amount=1, 
+        min_value=0.1, 
+        max_value=3, 
+        step=0.1,
+        digits=1,
+        suffix=" units"
+    ))
+
+    # Create a grid layout for the controls
+    controls1 = mo.vstack([
+        mo.md("### Interactive Normal Distribution"),
+        mo.hstack([
+            mo.md("Adjust the parameters to see how standard deviation affects the shape of the distribution:"),
+        ]),
+        mo.hstack([
+            mo.md("Mean (μ): "),
+            normal_mean,
+            mo.md("   Standard deviation (σ): "),
+            normal_std
+        ], justify="start"),
+    ])
+    return controls1, normal_mean, normal_std
+
+
+@app.cell(hide_code=True)
+def _(normal_mean, normal_std, np, plt, stats):
+    # data for normal distribution
+    _normal_x = np.linspace(-10, 10, 1000)
+    _normal_y = stats.norm.pdf(_normal_x, loc=normal_mean.amount, scale=normal_std.amount)
+
+    # ranges for standard deviation intervals
+    one_sigma_left = normal_mean.amount - normal_std.amount
+    one_sigma_right = normal_mean.amount + normal_std.amount
+    two_sigma_left = normal_mean.amount - 2 * normal_std.amount
+    two_sigma_right = normal_mean.amount + 2 * normal_std.amount
+    three_sigma_left = normal_mean.amount - 3 * normal_std.amount
+    three_sigma_right = normal_mean.amount + 3 * normal_std.amount
+
+    # Create the plot
+    normal_fig, normal_ax = plt.subplots(figsize=(10, 6))
+
+    # Plot the distribution
+    normal_ax.plot(_normal_x, _normal_y, 'b-', linewidth=2)
+
+    # stdev intervals
+    normal_ax.fill_between(_normal_x, 0, _normal_y, where=(_normal_x >= one_sigma_left) & (_normal_x <= one_sigma_right), 
+                   alpha=0.3, color='red', label='68% (±1σ)')
+    normal_ax.fill_between(_normal_x, 0, _normal_y, where=(_normal_x >= two_sigma_left) & (_normal_x <= two_sigma_right), 
+                   alpha=0.2, color='green', label='95% (±2σ)')
+    normal_ax.fill_between(_normal_x, 0, _normal_y, where=(_normal_x >= three_sigma_left) & (_normal_x <= three_sigma_right), 
+                   alpha=0.1, color='blue', label='99.7% (±3σ)')
+
+    # vertical lines for the mean and standard deviations
+    normal_ax.axvline(x=normal_mean.amount, color='black', linestyle='-', linewidth=1.5, label='Mean (μ)')
+    normal_ax.axvline(x=one_sigma_left, color='red', linestyle='--', linewidth=1)
+    normal_ax.axvline(x=one_sigma_right, color='red', linestyle='--', linewidth=1)
+    normal_ax.axvline(x=two_sigma_left, color='green', linestyle='--', linewidth=1)
+    normal_ax.axvline(x=two_sigma_right, color='green', linestyle='--', linewidth=1)
+
+    # annotations
+    normal_ax.annotate(f'μ = {normal_mean.amount:.2f}', 
+               xy=(normal_mean.amount, max(_normal_y)*0.5),
+               xytext=(normal_mean.amount + 0.5, max(_normal_y)*0.8),
+               arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    normal_ax.annotate(f'σ = {normal_std.amount:.2f}', 
+               xy=(one_sigma_right, stats.norm.pdf(one_sigma_right, loc=normal_mean.amount, scale=normal_std.amount)),
+               xytext=(one_sigma_right + 0.5, max(_normal_y)*0.6),
+               arrowprops=dict(facecolor='red', shrink=0.05, width=1))
+
+    # labels and title
+    normal_ax.set_xlabel('Value')
+    normal_ax.set_ylabel('Probability Density')
+    normal_ax.set_title(f'Normal Distribution with μ = {normal_mean.amount:.2f} and σ = {normal_std.amount:.2f}')
+
+    # legend and grid
+    normal_ax.legend()
+    normal_ax.grid(alpha=0.3)
+
+    plt.tight_layout()
+    plt.gca()
+    return (
+        normal_ax,
+        normal_fig,
+        one_sigma_left,
+        one_sigma_right,
+        three_sigma_left,
+        three_sigma_right,
+        two_sigma_left,
+        two_sigma_right,
+    )
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        /// tip
+        The interactive visualization above demonstrates how standard deviation (σ) affects the shape of a normal distribution:
+
+        - The **red region** covers μ ± 1σ, containing approximately 68% of the probability
+        - The **green region** covers μ ± 2σ, containing approximately 95% of the probability
+        - The **blue region** covers μ ± 3σ, containing approximately 99.7% of the probability
+
+        This is known as the "68-95-99.7 rule" or the "empirical rule" and is a useful heuristic for understanding the spread of data.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🤔 Test Your Understanding
+
+        Choose what you believe are the correct options in the questions below:
+
+        <details>
+        <summary>The variance of a random variable can be negative.</summary>
+        ❌ False! Variance is defined as an expected value of squared deviations, and squares are always non-negative.
+        </details>
+
+        <details>
+        <summary>If X and Y are independent random variables, then Var(X + Y) = Var(X) + Var(Y).</summary>
+        ✅ True! This is one of the key properties of variance for independent random variables.
+        </details>
+
+        <details>
+        <summary>Multiplying a random variable by 2 multiplies its variance by 2.</summary>
+        ❌ False! Multiplying a random variable by a constant a multiplies its variance by a². So multiplying by 2 multiplies variance by 4.
+        </details>
+
+        <details>
+        <summary>Standard deviation is always equal to the square root of variance.</summary>
+        ✅ True! By definition, standard deviation σ = √Var(X).
+        </details>
+
+        <details>
+        <summary>If Var(X) = 0, then X must be a constant.</summary>
+        ✅ True! Zero variance means there is no spread around the mean, so X can only take one value.
+        </details>
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Key Takeaways
+
+        Variance gives us a way to measure how spread out a random variable is around its mean. It's like the "uncertainty" in our expectation - a high variance means individual outcomes can differ widely from what we expect on average.
+
+        Standard deviation brings this measure back to the original units, making it easier to interpret. For grades, a standard deviation of 10 points means typical grades fall within about 10 points of the average.
+
+        Variance pops up everywhere - from weather forecasts (how reliable is the predicted temperature?) to financial investments (how risky is this stock?) to quality control (how consistent is our manufacturing process?).
+
+        In our next notebook, we'll explore more properties of random variables and see how they combine to form more complex distributions.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Appendix (containing helper code):""")
+    return
+
+
+@app.cell(hide_code=True)
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import numpy as np
+    import scipy.stats as stats
+    import matplotlib.pyplot as plt
+    from wigglystuff import TangleSlider
+    return TangleSlider, np, plt, stats
+
+
+@app.cell(hide_code=True)
+def _(TangleSlider, mo):
+    # Create interactive elements using TangleSlider for a more inline experience
+    true_grade = mo.ui.anywidget(TangleSlider(
+        amount=70, 
+        min_value=50, 
+        max_value=90, 
+        step=5,
+        digits=0,
+        suffix=" points"
+    ))
+
+    grader_a_spread = mo.ui.anywidget(TangleSlider(
+        amount=10, 
+        min_value=5, 
+        max_value=20, 
+        step=1,
+        digits=0,
+        suffix=" points"
+    ))
+
+    grader_b_spread = mo.ui.anywidget(TangleSlider(
+        amount=2, 
+        min_value=1, 
+        max_value=5, 
+        step=0.5,
+        digits=1,
+        suffix=" points"
+    ))
+
+    grader_c_mix = mo.ui.anywidget(TangleSlider(
+        amount=0.2, 
+        min_value=0, 
+        max_value=1, 
+        step=0.05,
+        digits=2,
+        suffix=" proportion"
+    ))
+    return grader_a_spread, grader_b_spread, grader_c_mix, true_grade
+
+
+@app.cell(hide_code=True)
+def _(grader_a_spread, grader_b_spread, grader_c_mix, mo, true_grade):
+    # Create a grid layout for the interactive controls
+    controls = mo.vstack([
+        mo.md("### Adjust Parameters to See How Variance Changes"),
+        mo.hstack([
+            mo.md("**True grade:** The correct score that should be assigned is "),
+            true_grade,
+            mo.md(" out of 100.")
+        ], justify="start"),
+        mo.hstack([
+            mo.md("**Grader A:** Has a wide spread with standard deviation of "),
+            grader_a_spread,
+            mo.md(" points.")
+        ], justify="start"),
+        mo.hstack([
+            mo.md("**Grader B:** Has a narrow spread with standard deviation of "),
+            grader_b_spread,
+            mo.md(" points.")
+        ], justify="start"),
+        mo.hstack([
+            mo.md("**Grader C:** Has a mixture distribution with "),
+            grader_c_mix,
+            mo.md(" proportion of outliers.")
+        ], justify="start"),
+    ])
+    return (controls,)
+
+
+if __name__ == "__main__":
+    app.run()

probability/13_bernoulli_distribution.py

@@ -0,0 +1,427 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.3",
+#     "scipy==1.15.2",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.22"
+app = marimo.App(width="medium", app_title="Bernoulli Distribution")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Bernoulli Distribution
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/bernoulli/), by Stanford professor Chris Piech._
+
+        ## Parametric Random Variables
+
+        There are many classic and commonly-seen random variable abstractions that show up in the world of probability. At this point, we'll learn about several of the most significant parametric discrete distributions.
+
+        When solving problems, if you can recognize that a random variable fits one of these formats, then you can use its pre-derived Probability Mass Function (PMF), expectation, variance, and other properties. Random variables of this sort are called **parametric random variables**. If you can argue that a random variable falls under one of the studied parametric types, you simply need to provide parameters.
+
+        > A good analogy is a `class` in programming. Creating a parametric random variable is very similar to calling a constructor with input parameters.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Bernoulli Random Variables
+
+        A **Bernoulli random variable** (also called a boolean or indicator random variable) is the simplest kind of parametric random variable. It can take on two values: 1 and 0.
+
+        It takes on a 1 if an experiment with probability $p$ resulted in success and a 0 otherwise. 
+
+        Some example uses include:
+
+        - A coin flip (heads = 1, tails = 0)
+        - A random binary digit
+        - Whether a disk drive crashed
+        - Whether someone likes a Netflix movie
+
+        Here $p$ is the parameter, but different instances of Bernoulli random variables might have different values of $p$.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Key Properties of a Bernoulli Random Variable
+
+        If $X$ is declared to be a Bernoulli random variable with parameter $p$, denoted $X \sim \text{Bern}(p)$, it has the following properties:
+        """
+    )
+    return
+
+
+@app.cell
+def _(stats):
+    # Define the Bernoulli distribution function
+    def Bern(p):
+        return stats.bernoulli(p)
+    return (Bern,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Bernoulli Distribution Properties
+
+        $\begin{array}{lll}
+        \text{Notation:} & X \sim \text{Bern}(p) \\
+        \text{Description:} & \text{A boolean variable that is 1 with probability } p \\
+        \text{Parameters:} & p, \text{ the probability that } X = 1 \\
+        \text{Support:} & x \text{ is either 0 or 1} \\
+        \text{PMF equation:} & P(X = x) = 
+            \begin{cases}
+                p & \text{if }x = 1\\
+                1-p & \text{if }x = 0
+            \end{cases} \\
+        \text{PMF (smooth):} & P(X = x) = p^x(1-p)^{1-x} \\
+        \text{Expectation:} & E[X] = p \\
+        \text{Variance:} & \text{Var}(X) = p(1-p) \\
+        \end{array}$
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, p_slider):
+    # Visualization of the Bernoulli PMF
+    _p = p_slider.value
+
+    # Values for PMF
+    values = [0, 1]
+    probabilities = [1 - _p, _p]
+
+    # Relevant statistics
+    expected_value = _p
+    variance = _p * (1 - _p)
+
+    mo.md(f"""
+    ## PMF Graph for Bernoulli($p={_p:.2f}$)
+
+    Parameter $p$: {p_slider}
+
+    Expected value: $E[X] = {expected_value:.2f}$
+
+    Variance: $\\text{{Var}}(X) = {variance:.2f}$
+    """)
+    return expected_value, probabilities, values, variance
+
+
+@app.cell(hide_code=True)
+def _(expected_value, p_slider, plt, probabilities, values, variance):
+    # PMF
+    _p = p_slider.value
+    fig, ax = plt.subplots(figsize=(10, 6))
+
+    # Bar plot for PMF
+    ax.bar(values, probabilities, width=0.4, color='blue', alpha=0.7)
+
+    ax.set_xlabel('Values that X can take on')
+    ax.set_ylabel('Probability')
+    ax.set_title(f'PMF of Bernoulli Distribution with p = {_p:.2f}')
+
+    # x-axis limit
+    ax.set_xticks([0, 1])
+    ax.set_xlim(-0.5, 1.5)
+
+    # y-axis w/ some padding
+    ax.set_ylim(0, max(probabilities) * 1.1)
+
+    # Add expectation as vertical line
+    ax.axvline(x=expected_value, color='red', linestyle='--', 
+               label=f'E[X] = {expected_value:.2f}')
+
+    # Add variance annotation
+    ax.text(0.5, max(probabilities) * 0.8, 
+            f'Var(X) = {variance:.3f}', 
+            horizontalalignment='center',
+            bbox=dict(facecolor='white', alpha=0.7))
+
+    ax.legend()
+    plt.tight_layout()
+    plt.gca()
+    return ax, fig
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Proof: Expectation of a Bernoulli
+
+        If $X$ is a Bernoulli with parameter $p$, $X \sim \text{Bern}(p)$:
+
+        \begin{align}
+        E[X] &= \sum_x x \cdot (X=x) && \text{Definition of expectation} \\
+        &= 1 \cdot p + 0 \cdot (1-p) &&
+        X \text{ can take on values 0 and 1} \\
+        &= p && \text{Remove the 0 term}
+        \end{align}
+
+        ## Proof: Variance of a Bernoulli
+
+        If $X$ is a Bernoulli with parameter $p$, $X \sim \text{Bern}(p)$:
+
+        To compute variance, first compute $E[X^2]$:
+
+        \begin{align}
+        E[X^2]
+        &= \sum_x x^2 \cdot (X=x) &&\text{LOTUS}\\
+        &= 0^2 \cdot (1-p) + 1^2 \cdot p\\
+        &= p
+        \end{align}
+
+        \begin{align}
+        (X)
+        &= E[X^2] - E[X]^2&& \text{Def of variance} \\
+        &= p - p^2 && \text{Substitute }E[X^2]=p, E[X] = p \\
+        &= p (1-p) && \text{Factor out }p
+        \end{align}
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Indicator Random Variable
+
+        > **Definition**: An indicator variable is a Bernoulli random variable which takes on the value 1 if an **underlying event occurs**, and 0 _otherwise_.
+
+        Indicator random variables are a convenient way to convert the "true/false" outcome of an event into a number. That number may be easier to incorporate into an equation.
+
+        A random variable $I$ is an indicator variable for an event $A$ if $I = 1$ when $A$ occurs and $I = 0$ if $A$ does not occur. Indicator random variables are Bernoulli random variables, with $p = P(A)$. $I_A$ is a common choice of name for an indicator random variable.
+
+        Here are some properties of indicator random variables:
+
+        - $P(I=1)=P(A)$
+        - $E[I]=P(A)$
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    # Simulation of Bernoulli trials
+    mo.md(r"""
+    ## Simulation of Bernoulli Trials
+
+    Let's simulate Bernoulli trials to see the law of large numbers in action. We'll flip a biased coin repeatedly and observe how the proportion of successes approaches the true probability $p$.
+    """)
+
+    # UI element for simulation parameters
+    num_trials_slider = mo.ui.slider(10, 10000, value=1000, step=10, label="Number of trials")
+    p_sim_slider = mo.ui.slider(0.01, 0.99, value=0.65, step=0.01, label="Success probability (p)")
+    return num_trials_slider, p_sim_slider
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""## Simulation""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, num_trials_slider, p_sim_slider):
+    mo.hstack([num_trials_slider, p_sim_slider], justify='space-around')
+    return
+
+
+@app.cell(hide_code=True)
+def _(np, num_trials_slider, p_sim_slider, plt):
+    # Bernoulli trials
+    _num_trials = num_trials_slider.value
+    p = p_sim_slider.value
+
+    # Random Bernoulli trials
+    trials = np.random.binomial(1, p, size=_num_trials)
+
+    # Cumulative proportion of successes
+    cumulative_mean = np.cumsum(trials) / np.arange(1, _num_trials + 1)
+
+    # Results
+    plt.figure(figsize=(10, 6))
+    plt.plot(range(1, _num_trials + 1), cumulative_mean, label='Proportion of successes')
+    plt.axhline(y=p, color='r', linestyle='--', label=f'True probability (p={p})')
+
+    plt.xscale('log')  # Use log scale for better visualization
+    plt.xlabel('Number of trials')
+    plt.ylabel('Proportion of successes')
+    plt.title('Convergence of Sample Proportion to True Probability')
+    plt.legend()
+    plt.grid(True, alpha=0.3)
+
+    # Add annotation
+    plt.annotate('As the number of trials increases,\nthe proportion approaches p',
+                xy=(_num_trials, cumulative_mean[-1]), 
+                xytext=(_num_trials/5, p + 0.1),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    plt.tight_layout()
+    plt.gca()
+    return cumulative_mean, p, trials
+
+
+@app.cell(hide_code=True)
+def _(mo, np, trials):
+    # Calculate statistics from the simulation
+    num_successes = np.sum(trials)
+    num_trials = len(trials)
+    proportion = num_successes / num_trials
+
+    # Display the results
+    mo.md(f"""
+    ### Simulation Results
+
+    - Number of trials: {num_trials}
+    - Number of successes: {num_successes}
+    - Proportion of successes: {proportion:.4f}
+
+    This demonstrates how the sample proportion approaches the true probability $p$ as the number of trials increases.
+    """)
+    return num_successes, num_trials, proportion
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🤔 Test Your Understanding
+
+        Pick which of these statements about Bernoulli random variables you think are correct:
+
+        /// details | The variance of a Bernoulli random variable is always less than or equal to 0.25
+        ✅ Correct! The variance $p(1-p)$ reaches its maximum value of 0.25 when $p = 0.5$.
+        ///
+
+        /// details | The expected value of a Bernoulli random variable must be either 0 or 1
+        ❌ Incorrect! The expected value is $p$, which can be any value between 0 and 1.
+        ///
+
+        /// details | If $X \sim \text{Bern}(0.3)$ and $Y \sim \text{Bern}(0.7)$, then $X$ and $Y$ have the same variance
+        ✅ Correct! $\text{Var}(X) = 0.3 \times 0.7 = 0.21$ and $\text{Var}(Y) = 0.7 \times 0.3 = 0.21$.
+        ///
+
+        /// details | Two independent coin flips can be modeled as the sum of two Bernoulli random variables
+        ✅ Correct! The sum would follow a Binomial distribution with $n=2$.
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Applications of Bernoulli Random Variables
+
+        Bernoulli random variables are used in many real-world scenarios:
+
+        1. **Quality Control**: Testing if a manufactured item is defective (1) or not (0)
+
+        2. **A/B Testing**: Determining if a user clicks (1) or doesn't click (0) on a website button
+
+        3. **Medical Testing**: Checking if a patient tests positive (1) or negative (0) for a disease
+
+        4. **Election Modeling**: Modeling if a particular voter votes for candidate A (1) or not (0)
+
+        5. **Financial Markets**: Modeling if a stock price goes up (1) or down (0) in a simplified model
+
+        Because Bernoulli random variables are parametric, as soon as you declare a random variable to be of type Bernoulli, you automatically know all of its pre-derived properties!
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Summary
+
+        And that's a wrap on Bernoulli distributions! We've learnt the simplest of all probability distributions — the one that only has two possible outcomes. Flip a coin, check if an email is spam, see if your blind date shows up — these are all Bernoulli trials with success probability $p$. 
+
+        The beauty of Bernoulli is in its simplicity: just set $p$ (the probability of success) and you're good to go! The PMF gives us $P(X=1) = p$ and $P(X=0) = 1-p$, while expectation is simply $p$ and variance is $p(1-p)$. Oh, and when you're tracking whether specific events happen or not? That's an indicator random variable — just another Bernoulli in disguise!
+
+        Two key things to remember:
+
+        /// note
+        💡 **Maximum Variance**: A Bernoulli's variance $p(1-p)$ reaches its maximum at $p=0.5$, making a fair coin the most "unpredictable" Bernoulli random variable.
+
+        💡 **Instant Properties**: When you identify a random variable as Bernoulli, you instantly know all its properties—expectation, variance, PMF—without additional calculations.
+        ///
+
+        Next up: Binomial distribution—where we'll see what happens when we let Bernoulli trials have a party and add themselves together!
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""#### Appendix (containing helper code for the notebook)""")
+    return
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    from marimo import Html
+    return (Html,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import numpy as np
+    import matplotlib.pyplot as plt
+    from scipy import stats
+    import math
+
+    # Set style for consistent visualizations
+    plt.style.use('seaborn-v0_8-whitegrid')
+    plt.rcParams['figure.figsize'] = [10, 6]
+    plt.rcParams['font.size'] = 12
+
+    # Set random seed for reproducibility
+    np.random.seed(42)
+    return math, np, plt, stats
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    # Create a UI element for the parameter p
+    p_slider = mo.ui.slider(0.01, 0.99, value=0.65, step=0.01, label="Parameter p")
+    return (p_slider,)
+
+
+if __name__ == "__main__":
+    app.run()

probability/14_binomial_distribution.py

@@ -0,0 +1,545 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.4",
+#     "scipy==1.15.2",
+#     "altair==5.2.0",
+#     "wigglystuff==0.1.10",
+#     "pandas==2.2.3",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.24"
+app = marimo.App(width="medium", app_title="Binomial Distribution")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Binomial Distribution
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/binomial/), by Stanford professor Chris Piech._
+
+        In this section, we will discuss the binomial distribution. To start, imagine the following example:
+
+        Consider $n$ independent trials of an experiment where each trial is a "success" with probability $p$. Let $X$ be the number of successes in $n$ trials.
+
+        This situation is truly common in the natural world, and as such, there has been a lot of research into such phenomena. Random variables like $X$ are called **binomial random variables**. If you can identify that a process fits this description, you can inherit many already proved properties such as the PMF formula, expectation, and variance!
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Binomial Random Variable Definition
+
+        $X \sim \text{Bin}(n, p)$ represents a binomial random variable where:
+
+        - $X$ is our random variable (number of successes)
+        - $\text{Bin}$ indicates it follows a binomial distribution
+        - $n$ is the number of trials
+        - $p$ is the probability of success in each trial
+
+        ```
+        X ~ Bin(n, p)
+         ↑    ↑  ↑
+         |    |  +-- Probability of
+         |    |      success on each
+         |    |      trial
+         |    +-- Number of trials
+         |
+        Our random variable
+          is distributed
+          as a Binomial
+        ```
+
+        Here are a few examples of binomial random variables:
+
+        - Number of heads in $n$ coin flips
+        - Number of 1's in randomly generated length $n$ bit string
+        - Number of disk drives crashed in 1000 computer cluster, assuming disks crash independently
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Properties of Binomial Distribution
+
+        | Property | Formula |
+        |----------|---------|
+        | Notation | $X \sim \text{Bin}(n, p)$ |
+        | Description | Number of "successes" in $n$ identical, independent experiments each with probability of success $p$ |
+        | Parameters | $n \in \{0, 1, \dots\}$, the number of experiments<br>$p \in [0, 1]$, the probability that a single experiment gives a "success" |
+        | Support | $x \in \{0, 1, \dots, n\}$ |
+        | PMF equation | $P(X=x) = {n \choose x}p^x(1-p)^{n-x}$ |
+        | Expectation | $E[X] = n \cdot p$ |
+        | Variance | $\text{Var}(X) = n \cdot p \cdot (1-p)$ |
+
+        Let's explore how the binomial distribution changes with different parameters.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(TangleSlider, mo):
+    # Interactive elements using TangleSlider
+    n_slider = mo.ui.anywidget(TangleSlider(
+        amount=10, 
+        min_value=1, 
+        max_value=30, 
+        step=1,
+        digits=0,
+        suffix=" trials"
+    ))
+
+    p_slider = mo.ui.anywidget(TangleSlider(
+        amount=0.5, 
+        min_value=0.01, 
+        max_value=0.99, 
+        step=0.01,
+        digits=2,
+        suffix=" probability"
+    ))
+
+    # Grid layout for the interactive controls
+    controls = mo.vstack([
+        mo.md("### Adjust Parameters to See How Binomial Distribution Changes"),
+        mo.hstack([
+            mo.md("**Number of trials (n):** "),
+            n_slider
+        ], justify="start"),
+        mo.hstack([
+            mo.md("**Probability of success (p):** "),
+            p_slider
+        ], justify="start"),
+    ])
+    return controls, n_slider, p_slider
+
+
+@app.cell(hide_code=True)
+def _(controls):
+    controls
+    return
+
+
+@app.cell(hide_code=True)
+def _(n_slider, np, p_slider, plt, stats):
+    # Parameters from sliders
+    _n = int(n_slider.amount)
+    _p = p_slider.amount
+
+    # Calculate PMF
+    _x = np.arange(0, _n + 1)
+    _pmf = stats.binom.pmf(_x, _n, _p)
+
+    # Relevant stats
+    _mean = _n * _p
+    _variance = _n * _p * (1 - _p)
+    _std_dev = np.sqrt(_variance)
+
+    _fig, _ax = plt.subplots(figsize=(10, 6))
+
+    # Plot PMF as bars
+    _ax.bar(_x, _pmf, color='royalblue', alpha=0.7, label=f'PMF: P(X=k)')
+
+    # Add a line
+    _ax.plot(_x, _pmf, 'ro-', alpha=0.6, label='PMF line')
+
+    # Add vertical lines
+    _ax.axvline(x=_mean, color='green', linestyle='--', linewidth=2, 
+               label=f'Mean: {_mean:.2f}')
+
+    # Shade the stdev region
+    _ax.axvspan(_mean - _std_dev, _mean + _std_dev, alpha=0.2, color='green',
+               label=f'±1 Std Dev: {_std_dev:.2f}')
+
+    # Add labels and title
+    _ax.set_xlabel('Number of Successes (k)')
+    _ax.set_ylabel('Probability: P(X=k)')
+    _ax.set_title(f'Binomial Distribution with n={_n}, p={_p:.2f}')
+
+    # Annotations
+    _ax.annotate(f'E[X] = {_mean:.2f}', 
+                xy=(_mean, stats.binom.pmf(int(_mean), _n, _p)), 
+                xytext=(_mean + 1, max(_pmf) * 0.8),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    _ax.annotate(f'Var(X) = {_variance:.2f}', 
+                xy=(_mean, stats.binom.pmf(int(_mean), _n, _p) / 2), 
+                xytext=(_mean + 1, max(_pmf) * 0.6),
+                arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+    # Grid and legend
+    _ax.grid(alpha=0.3)
+    _ax.legend()
+
+    plt.tight_layout()
+    plt.gca()
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Relationship to Bernoulli Random Variables
+
+        One way to think of the binomial is as the sum of $n$ Bernoulli variables. Say that $Y_i$ is an indicator Bernoulli random variable which is 1 if experiment $i$ is a success. Then if $X$ is the total number of successes in $n$ experiments, $X \sim \text{Bin}(n, p)$:
+
+        $$X = \sum_{i=1}^n Y_i$$
+
+        Recall that the outcome of $Y_i$ will be 1 or 0, so one way to think of $X$ is as the sum of those 1s and 0s.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Binomial Probability Mass Function (PMF)
+
+        The most important property to know about a binomial is its [Probability Mass Function](https://marimo.app/https://github.com/marimo-team/learn/blob/main/probability/10_probability_mass_function.py):
+
+        $$P(X=k) = {n \choose k}p^k(1-p)^{n-k}$$
+
+        ```
+        P(X = k) = (n) p^k(1-p)^(n-k)
+         ↑           (k)
+         |            ↑
+         |            +-- Binomial coefficient:
+         |                number of ways to choose
+         |                k successes from n trials
+         |
+        Probability that our
+        variable takes on the
+        value k
+        ```
+
+        Recall, we derived this formula in Part 1. There is a complete example on the probability of $k$ heads in $n$ coin flips, where each flip is heads with probability $p$.
+
+        To briefly review, if you think of each experiment as being distinct, then there are ${n \choose k}$ ways of permuting $k$ successes from $n$ experiments. For any of the mutually exclusive permutations, the probability of that permutation is $p^k \cdot (1-p)^{n-k}$.
+
+        The name binomial comes from the term ${n \choose k}$ which is formally called the binomial coefficient.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Expectation of Binomial
+
+        There is an easy way to calculate the expectation of a binomial and a hard way. The easy way is to leverage the fact that a binomial is the sum of Bernoulli indicator random variables $X = \sum_{i=1}^{n} Y_i$ where $Y_i$ is an indicator of whether the $i$-th experiment was a success: $Y_i \sim \text{Bernoulli}(p)$. 
+
+        Since the [expectation of the sum](http://marimo.app/https://github.com/marimo-team/learn/blob/main/probability/11_expectation.py) of random variables is the sum of expectations, we can add the expectation, $E[Y_i] = p$, of each of the Bernoulli's:
+
+        \begin{align}
+        E[X] &= E\Big[\sum_{i=1}^{n} Y_i\Big] && \text{Since }X = \sum_{i=1}^{n} Y_i \\
+        &= \sum_{i=1}^{n}E[ Y_i] && \text{Expectation of sum} \\
+        &= \sum_{i=1}^{n}p && \text{Expectation of Bernoulli} \\
+        &= n \cdot p && \text{Sum $n$ times}
+        \end{align}
+
+        The hard way is to use the definition of expectation:
+
+        \begin{align}
+        E[X] &= \sum_{i=0}^n i \cdot P(X = i) && \text{Def of expectation} \\
+        &= \sum_{i=0}^n i \cdot {n \choose i} p^i(1-p)^{n-i} && \text{Sub in PMF} \\
+        & \cdots && \text{Many steps later} \\
+        &= n \cdot p
+        \end{align}
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Binomial Distribution in Python
+
+        As you might expect, you can use binomial distributions in code. The standardized library for binomials is `scipy.stats.binom`.
+
+        One of the most helpful methods that this package provides is a way to calculate the PMF. For example, say $n=5$, $p=0.6$ and you want to find $P(X=2)$, you could use the following code:
+        """
+    )
+    return
+
+
+@app.cell
+def _(stats):
+    # define variables for x, n, and p
+    _n = 5  # Integer value for n
+    _p = 0.6
+    _x = 2
+
+    # use scipy to compute the pmf
+    p_x = stats.binom.pmf(_x, _n, _p)
+
+    # use the probability for future work
+    print(f'P(X = {_x}) = {p_x:.4f}')
+    return (p_x,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Another particularly helpful function is the ability to generate a random sample from a binomial. For example, say $X$ represents the number of requests to a website. We can draw 100 samples from this distribution using the following code:""")
+    return
+
+
+@app.cell
+def _(n, p, stats):
+    n_int = int(n)
+
+    # samples from the binomial distribution
+    samples = stats.binom.rvs(n_int, p, size=100)
+
+    # Print the samples
+    print(samples)
+    return n_int, samples
+
+
+@app.cell(hide_code=True)
+def _(n_int, np, p, plt, samples, stats):
+    # Plot histogram of samples
+    plt.figure(figsize=(10, 5))
+    plt.hist(samples, bins=np.arange(-0.5, n_int+1.5, 1), alpha=0.7, color='royalblue', 
+             edgecolor='black', density=True)
+
+    # Overlay the PMF
+    x_values = np.arange(0, n_int+1)
+    pmf_values = stats.binom.pmf(x_values, n_int, p)
+    plt.plot(x_values, pmf_values, 'ro-', ms=8, label='Theoretical PMF')
+
+    # Add labels and title
+    plt.xlabel('Number of Successes')
+    plt.ylabel('Relative Frequency / Probability')
+    plt.title(f'Histogram of 100 Samples from Bin({n_int}, {p})')
+    plt.legend()
+    plt.grid(alpha=0.3)
+
+    # Annotate
+    plt.annotate('Sample mean: %.2f' % np.mean(samples), 
+                xy=(0.7, 0.9), xycoords='axes fraction',
+                bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.3))
+    plt.annotate('Theoretical mean: %.2f' % (n_int*p), 
+                xy=(0.7, 0.8), xycoords='axes fraction',
+                bbox=dict(boxstyle='round,pad=0.5', fc='lightgreen', alpha=0.3))
+
+    plt.tight_layout()
+    plt.gca()
+    return pmf_values, x_values
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        You might be wondering what a random sample is! A random sample is a randomly chosen assignment for our random variable. Above we have 100 such assignments. The probability that value $k$ is chosen is given by the PMF: $P(X=k)$. 
+
+        There are also functions for getting the mean, the variance, and more. You can read the [scipy.stats.binom documentation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.binom.html), especially the list of methods.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Interactive Exploration of Binomial vs. Negative Binomial
+
+        The standard binomial distribution is a special case of a broader family of distributions. One related distribution is the negative binomial, which can model count data with overdispersion (where the variance is larger than the mean).
+
+        Below, you can explore how the negative binomial distribution compares to a Poisson distribution (which can be seen as a limiting case of the binomial as $n$ gets large and $p$ gets small, with $np$ held constant).
+
+        Adjust the sliders to see how the parameters affect the distribution:
+
+        *Note: The interactive visualization in this section was inspired by work from [liquidcarbon on GitHub](https://github.com/liquidcarbon).*
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(alpha_slider, chart, equation, mo, mu_slider):
+    mo.vstack(
+        [
+            mo.md(f"## Negative Binomial Distribution (Poisson + Overdispersion)\n{equation}"),
+            mo.hstack([mu_slider, alpha_slider], justify="start"),
+            chart,
+        ], justify='space-around'
+    ).center()
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🤔 Test Your Understanding
+        Pick which of these statements about binomial distributions you think are correct:
+
+        /// details | The variance of a binomial distribution is always equal to its mean
+        ❌ Incorrect! The variance is $np(1-p)$ while the mean is $np$. They're only equal when $p=1$ (which is a degenerate case).
+        ///
+
+        /// details | If $X \sim \text{Bin}(n, p)$ and $Y \sim \text{Bin}(n, 1-p)$, then $X$ and $Y$ have the same variance
+        ✅ Correct! $\text{Var}(X) = np(1-p)$ and $\text{Var}(Y) = n(1-p)p$, which are the same.
+        ///
+
+        /// details | As the number of trials increases, the binomial distribution approaches a normal distribution
+        ✅ Correct! For large $n$, the binomial distribution can be approximated by a normal distribution with the same mean and variance.
+        ///
+
+        /// details | The PMF of a binomial distribution is symmetric when $p = 0.5$
+        ✅ Correct! When $p = 0.5$, the PMF is symmetric around $n/2$.
+        ///
+
+        /// details | The sum of two independent binomial random variables with the same $p$ is also a binomial random variable
+        ✅ Correct! If $X \sim \text{Bin}(n_1, p)$ and $Y \sim \text{Bin}(n_2, p)$ are independent, then $X + Y \sim \text{Bin}(n_1 + n_2, p)$.
+        ///
+
+        /// details | The maximum value of the PMF for $\text{Bin}(n,p)$ always occurs at $k = np$
+        ❌ Incorrect! The mode (maximum value of PMF) is either $\lfloor (n+1)p \rfloor$ or $\lceil (n+1)p-1 \rceil$ depending on whether $(n+1)p$ is an integer.
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Summary
+
+        So we've explored the binomial distribution, and honestly, it's one of the most practical probability distributions you'll encounter. Think about it — anytime you're counting successes in a fixed number of trials (like those coin flips we discussed), this is your go-to distribution.
+
+        I find it fascinating how the expectation is simply $np$. Such a clean, intuitive formula! And remember that neat visualization we saw earlier? When we adjusted the parameters, you could actually see how the distribution shape changes—becoming more symmetric as $n$ increases.
+
+        The key things to take away:
+
+        - The binomial distribution models the number of successes in $n$ independent trials, each with probability $p$ of success
+
+        - Its PMF is given by the formula $P(X=k) = {n \choose k}p^k(1-p)^{n-k}$, which lets us calculate exactly how likely any specific number of successes is
+
+        - The expected value is $E[X] = np$ and the variance is $Var(X) = np(1-p)$
+
+        - It's related to other distributions: it's essentially a sum of Bernoulli random variables, and connects to both the negative binomial and Poisson distributions
+
+        - In Python, the `scipy.stats.binom` module makes working with binomial distributions straightforward—you can generate random samples and calculate probabilities with just a few lines of code
+
+        You'll see the binomial distribution pop up everywhere—from computer science to quality control, epidemiology, and data science. Any time you have scenarios with binary outcomes over multiple trials, this distribution has you covered.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Appendix code (helper functions, variables, etc.):""")
+    return
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import numpy as np
+    import matplotlib.pyplot as plt
+    import scipy.stats as stats
+    import pandas as pd
+    import altair as alt
+    from wigglystuff import TangleSlider
+    return TangleSlider, alt, np, pd, plt, stats
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    alpha_slider = mo.ui.slider(
+        value=0.1,
+        steps=[0, 0.01, 0.02, 0.03, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1],
+        label="α (overdispersion)",
+        show_value=True,
+    )
+    mu_slider = mo.ui.slider(
+        value=100, start=1, stop=100, step=1, label="μ (mean)", show_value=True
+    )
+    return alpha_slider, mu_slider
+
+
+@app.cell(hide_code=True)
+def _():
+    equation = """
+    $$
+    P(X = k) = \\frac{\\Gamma(k + \\frac{1}{\\alpha})}{\\Gamma(k + 1) \\Gamma(\\frac{1}{\\alpha})} \\left( \\frac{1}{\\mu \\alpha + 1} \\right)^{\\frac{1}{\\alpha}} \\left( \\frac{\\mu \\alpha}{\\mu \\alpha + 1} \\right)^k
+    $$
+
+    $$
+    \\sigma^2 = \\mu + \\alpha \\mu^2
+    $$
+    """
+    return (equation,)
+
+
+@app.cell(hide_code=True)
+def _(alpha_slider, alt, mu_slider, np, pd, stats):
+    mu = mu_slider.value
+    alpha = alpha_slider.value
+    n = 1000 - mu if alpha == 0 else 1 / alpha
+    p = n / (mu + n)
+    x = np.arange(0, mu * 3 + 1, 1)
+    df = pd.DataFrame(
+        {
+            "x": x,
+            "y": stats.nbinom.pmf(x, n, p),
+            "y_poi": stats.nbinom.pmf(x, 1000 - mu, 1 - mu / 1000),
+        }
+    )
+    r1k = stats.nbinom.rvs(n, p, size=1000)
+    df["in 95% CI"] = df["x"].between(*np.percentile(r1k, q=[2.5, 97.5]))
+    base = alt.Chart(df)
+
+    chart_poi = base.mark_bar(
+        fillOpacity=0.25, width=100 / mu, fill="magenta"
+    ).encode(
+        x=alt.X("x").scale(domain=(-0.4, x.max() + 0.4), nice=False),
+        y=alt.Y("y_poi").scale(domain=(0, df.y_poi.max() * 1.1)).title(None),
+    )
+    chart_nb = base.mark_bar(fillOpacity=0.75, width=100 / mu).encode(
+        x="x",
+        y="y",
+        fill=alt.Fill("in 95% CI")
+        .scale(domain=[False, True], range=["#aaa", "#7c7"])
+        .legend(orient="bottom-right"),
+    )
+
+    chart = (chart_poi + chart_nb).configure_view(continuousWidth=450)
+    return alpha, base, chart, chart_nb, chart_poi, df, mu, n, p, r1k, x
+
+
+if __name__ == "__main__":
+    app.run()

probability/15_poisson_distribution.py

@@ -0,0 +1,805 @@
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "marimo",
+#     "matplotlib==3.10.0",
+#     "numpy==2.2.4",
+#     "scipy==1.15.2",
+#     "altair==5.2.0",
+#     "wigglystuff==0.1.10",
+#     "pandas==2.2.3",
+# ]
+# ///
+
+import marimo
+
+__generated_with = "0.11.25"
+app = marimo.App(width="medium", app_title="Poisson Distribution")
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        # Poisson Distribution
+
+        _This notebook is a computational companion to ["Probability for Computer Scientists"](https://chrispiech.github.io/probabilityForComputerScientists/en/part2/poisson/), by Stanford professor Chris Piech._
+
+        A Poisson random variable gives the probability of a given number of events in a fixed interval of time (or space). It makes the Poisson assumption that events occur with a known constant mean rate and independently of the time since the last event.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Poisson Random Variable Definition
+
+        $X \sim \text{Poisson}(\lambda)$ represents a Poisson random variable where:
+
+        - $X$ is our random variable (number of events)
+        - $\text{Poisson}$ indicates it follows a Poisson distribution
+        - $\lambda$ is the rate parameter (average number of events per time interval)
+
+        ```
+        X ~ Poisson(λ)
+         ↑     ↑    ↑
+         |     |    +-- Rate parameter:
+         |     |        average number of
+         |     |        events per interval
+         |     +-- Indicates Poisson
+         |         distribution
+         |
+        Our random variable
+          counting number of events
+        ```
+
+        The Poisson distribution is particularly useful when:
+
+        1. Events occur independently of each other
+        2. The average rate of occurrence is constant
+        3. Two events cannot occur at exactly the same instant
+        4. The probability of an event is proportional to the length of the time interval
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Properties of Poisson Distribution
+
+        | Property | Formula |
+        |----------|---------|
+        | Notation | $X \sim \text{Poisson}(\lambda)$ |
+        | Description | Number of events in a fixed time frame if (a) events occur with a constant mean rate and (b) they occur independently of time since last event |
+        | Parameters | $\lambda \in \mathbb{R}^{+}$, the constant average rate |
+        | Support | $x \in \{0, 1, \dots\}$ |
+        | PMF equation | $P(X=x) = \frac{\lambda^x e^{-\lambda}}{x!}$ |
+        | Expectation | $E[X] = \lambda$ |
+        | Variance | $\text{Var}(X) = \lambda$ |
+
+        Note that unlike many other distributions, the Poisson distribution's mean and variance are equal, both being $\lambda$.
+
+        Let's explore how the Poisson distribution changes with different rate parameters.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(TangleSlider, mo):
+    # interactive elements using TangleSlider
+    lambda_slider = mo.ui.anywidget(TangleSlider(
+        amount=5, 
+        min_value=0.1, 
+        max_value=20, 
+        step=0.1,
+        digits=1,
+        suffix=" events"
+    ))
+
+    # interactive controls
+    _controls = mo.vstack([
+        mo.md("### Adjust the Rate Parameter to See How Poisson Distribution Changes"),
+        mo.hstack([
+            mo.md("**Rate parameter (λ):** "),
+            lambda_slider,
+            mo.md("**events per interval.** Higher values shift the distribution rightward and make it more spread out.")
+        ], justify="start"),
+    ])
+    _controls
+    return (lambda_slider,)
+
+
+@app.cell(hide_code=True)
+def _(lambda_slider, np, plt, stats):
+    def create_poisson_pmf_plot(lambda_value):
+        """Create a visualization of Poisson PMF with annotations for mean and variance."""
+        # PMF for values
+        max_x = max(20, int(lambda_value * 3))  # Show at least up to 3*lambda
+        x = np.arange(0, max_x + 1)
+        pmf = stats.poisson.pmf(x, lambda_value)
+
+        # Relevant key statistics
+        mean = lambda_value  # For Poisson, mean = lambda
+        variance = lambda_value  # For Poisson, variance = lambda
+        std_dev = np.sqrt(variance)
+
+        # plot
+        fig, ax = plt.subplots(figsize=(10, 6))
+
+        # PMF as bars
+        ax.bar(x, pmf, color='royalblue', alpha=0.7, label=f'PMF: P(X=k)')
+
+        # for the PMF values
+        ax.plot(x, pmf, 'ro-', alpha=0.6, label='PMF line')
+
+        # Vertical lines - mean and key values
+        ax.axvline(x=mean, color='green', linestyle='--', linewidth=2, 
+                label=f'Mean: {mean:.2f}')
+
+        # Stdev region
+        ax.axvspan(mean - std_dev, mean + std_dev, alpha=0.2, color='green',
+                label=f'±1 Std Dev: {std_dev:.2f}')
+
+        ax.set_xlabel('Number of Events (k)')
+        ax.set_ylabel('Probability: P(X=k)')
+        ax.set_title(f'Poisson Distribution with λ={lambda_value:.1f}')
+
+        # annotations
+        ax.annotate(f'E[X] = {mean:.2f}', 
+                    xy=(mean, stats.poisson.pmf(int(mean), lambda_value)), 
+                    xytext=(mean + 1, max(pmf) * 0.8),
+                    arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+        ax.annotate(f'Var(X) = {variance:.2f}', 
+                    xy=(mean, stats.poisson.pmf(int(mean), lambda_value) / 2), 
+                    xytext=(mean + 1, max(pmf) * 0.6),
+                    arrowprops=dict(facecolor='black', shrink=0.05, width=1))
+
+        ax.grid(alpha=0.3)
+        ax.legend()
+
+        plt.tight_layout()
+        return plt.gca()
+
+    # Get parameter from slider and create plot
+    _lambda = lambda_slider.amount
+    create_poisson_pmf_plot(_lambda)
+    return (create_poisson_pmf_plot,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Poisson Intuition: Relation to Binomial Distribution
+
+        The Poisson distribution can be derived as a limiting case of the [binomial distribution](http://marimo.app/https://github.com/marimo-team/learn/blob/main/probability/14_binomial_distribution.py). 
+
+        Let's work on a practical example: predicting the number of ride-sharing requests in a specific area over a one-minute interval. From historical data, we know that the average number of requests per minute is $\lambda = 5$.
+
+        We could approximate this using a binomial distribution by dividing our minute into smaller intervals. For example, we can divide a minute into 60 seconds and treat each second as a [Bernoulli trial](http://marimo.app/https://github.com/marimo-team/learn/blob/main/probability/13_bernoulli_distribution.py) - either there's a request (success) or there isn't (failure).
+
+        Let's visualize this concept:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(fig_to_image, mo, plt):
+    def create_time_division_visualization():
+        # visualization of dividing a minute into 60 seconds
+        fig, ax = plt.subplots(figsize=(12, 2))
+
+        # Example events hardcoded at 2.75s and 7.12s
+        events = [2.75, 7.12]
+
+        # array of 60 rectangles
+        for i in range(60):
+            color = 'royalblue' if any(i <= e < i+1 for e in events) else 'lightgray'
+            ax.add_patch(plt.Rectangle((i, 0), 0.9, 1, color=color))
+
+        # markers for events
+        for e in events:
+            ax.plot(e, 0.5, 'ro', markersize=10)
+
+        # labels
+        ax.set_xlim(0, 60)
+        ax.set_ylim(0, 1)
+        ax.set_yticks([])
+        ax.set_xticks([0, 15, 30, 45, 60])
+        ax.set_xticklabels(['0s', '15s', '30s', '45s', '60s'])
+        ax.set_xlabel('Time (seconds)')
+        ax.set_title('One Minute Divided into 60 Second Intervals')
+
+        plt.tight_layout()
+        plt.gca()
+        return fig, events, i
+
+    # Create visualization and convert to image
+    _fig, _events, i = create_time_division_visualization()
+    _img = mo.image(fig_to_image(_fig), width="100%")
+
+    # explanation
+    _explanation = mo.md(
+        r"""
+        In this visualization:
+    
+        - Each rectangle represents a 1-second interval
+        - Blue rectangles indicate intervals where an event occurred
+        - Red dots show the actual event times (2.75s and 7.12s)
+
+        If we treat this as a binomial experiment with 60 trials (seconds), we can calculate probabilities using the binomial PMF. But there's a problem: what if multiple events occur within the same second? To address this, we can divide our minute into smaller intervals.
+        """
+    )
+    mo.vstack([_fig, _explanation])
+    return create_time_division_visualization, i
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        The total number of requests received over the minute can be approximated as the sum of the sixty indicator variables, which conveniently matches the description of a binomial — a sum of Bernoullis. 
+
+        Specifically, if we define $X$ to be the number of requests in a minute, $X$ is a binomial with $n=60$ trials. What is the probability, $p$, of a success on a single trial? To make the expectation of $X$ equal the observed historical average $\lambda$, we should choose $p$ so that:
+
+        \begin{align}
+        \lambda &= E[X] && \text{Expectation matches historical average} \\
+        \lambda &= n \cdot p && \text{Expectation of a Binomial is } n \cdot p \\
+        p &= \frac{\lambda}{n} && \text{Solving for $p$}
+        \end{align}
+
+        In this case, since $\lambda=5$ and $n=60$, we should choose $p=\frac{5}{60}=\frac{1}{12}$ and state that $X \sim \text{Bin}(n=60, p=\frac{5}{60})$. Now we can calculate the probability of different numbers of requests using the binomial PMF:
+
+        $P(X = x) = {n \choose x} p^x (1-p)^{n-x}$
+
+        For example:
+
+        \begin{align}
+        P(X=1) &= {60 \choose 1} (5/60)^1 (55/60)^{60-1} \approx 0.0295 \\
+        P(X=2) &= {60 \choose 2} (5/60)^2 (55/60)^{60-2} \approx 0.0790 \\
+        P(X=3) &= {60 \choose 3} (5/60)^3 (55/60)^{60-3} \approx 0.1389
+        \end{align}
+
+        This is a good approximation, but it doesn't account for the possibility of multiple events in a single second. One solution is to divide our minute into even more fine-grained intervals. Let's try 600 deciseconds (tenths of a second):
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(fig_to_image, mo, plt):
+    def create_decisecond_visualization(e_value):
+        # (Just showing the first 100 for clarity)
+        fig, ax = plt.subplots(figsize=(12, 2))
+
+        # Example events at 2.75s and 7.12s (convert to deciseconds)
+        events = [27.5, 71.2]
+    
+        for i in range(100):
+            color = 'royalblue' if any(i <= event_val < i + 1 for event_val in events) else 'lightgray'
+            ax.add_patch(plt.Rectangle((i, 0), 0.9, 1, color=color))
+
+        # Markers for events
+        for event in events:
+            if event < 100:  # Only show events in our visible range
+                ax.plot(event/10, 0.5, 'ro', markersize=10)  # Divide by 10 to convert to deciseconds
+
+        # Add labels
+        ax.set_xlim(0, 100)
+        ax.set_ylim(0, 1)
+        ax.set_yticks([])
+        ax.set_xticks([0, 20, 40, 60, 80, 100])
+        ax.set_xticklabels(['0s', '2s', '4s', '6s', '8s', '10s'])
+        ax.set_xlabel('Time (first 10 seconds shown)')
+        ax.set_title('One Minute Divided into 600 Decisecond Intervals (first 100 shown)')
+
+        plt.tight_layout()
+        plt.gca()
+        return fig
+
+    # Create viz and convert to image
+    _fig = create_decisecond_visualization(e_value=5)
+    _img = mo.image(fig_to_image(_fig), width="100%")
+
+    # Explanation
+    _explanation = mo.md(
+        r"""
+        With $n=600$ and $p=\frac{5}{600}=\frac{1}{120}$, we can recalculate our probabilities:
+
+        \begin{align}
+        P(X=1) &= {600 \choose 1} (5/600)^1 (595/600)^{600-1} \approx 0.0333 \\
+        P(X=2) &= {600 \choose 2} (5/600)^2 (595/600)^{600-2} \approx 0.0837 \\
+        P(X=3) &= {600 \choose 3} (5/600)^3 (595/600)^{600-3} \approx 0.1402
+        \end{align}
+
+        As we make our intervals smaller (increasing $n$), our approximation becomes more accurate.
+        """
+    )
+    mo.vstack([_fig, _explanation])
+    return (create_decisecond_visualization,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## The Binomial Distribution in the Limit
+
+        What happens if we continue dividing our time interval into smaller and smaller pieces? Let's explore how the probabilities change as we increase the number of intervals:
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    intervals_slider = mo.ui.slider(
+        start = 60, 
+        stop = 10000,
+        step=100,
+        value=600,
+        label="Number of intervals to divide a minute")
+    return (intervals_slider,)
+
+
+@app.cell(hide_code=True)
+def _(intervals_slider):
+    intervals_slider
+    return
+
+
+@app.cell(hide_code=True)
+def _(intervals_slider, np, pd, plt, stats):
+    def create_comparison_plot(n, lambda_value):
+        # Calculate probability
+        p = lambda_value / n
+
+        # Binomial probabilities
+        x_values = np.arange(0, 15)
+        binom_pmf = stats.binom.pmf(x_values, n, p)
+
+        # True Poisson probabilities
+        poisson_pmf = stats.poisson.pmf(x_values, lambda_value)
+
+        # DF for comparison
+        df = pd.DataFrame({
+            'Events': x_values,
+            f'Binomial(n={n}, p={p:.6f})': binom_pmf,
+            f'Poisson(λ=5)': poisson_pmf,
+            'Difference': np.abs(binom_pmf - poisson_pmf)
+        })
+
+        # Plot both PMFs
+        fig, ax = plt.subplots(figsize=(10, 6))
+
+        # Bar plot for the binomial
+        ax.bar(x_values - 0.2, binom_pmf, width=0.4, alpha=0.7, 
+            color='royalblue', label=f'Binomial(n={n}, p={p:.6f})')
+
+        # Bar plot for the Poisson
+        ax.bar(x_values + 0.2, poisson_pmf, width=0.4, alpha=0.7,
+            color='crimson', label='Poisson(λ=5)')
+
+        # Labels and title
+        ax.set_xlabel('Number of Events (k)')
+        ax.set_ylabel('Probability')
+        ax.set_title(f'Comparison of Binomial and Poisson PMFs with n={n}')
+        ax.legend()
+        ax.set_xticks(x_values)
+        ax.grid(alpha=0.3)
+
+        plt.tight_layout()
+        return df, fig, n, p
+
+    # Number of intervals from the slider
+    n = intervals_slider.value
+    _lambda = 5  # Fixed lambda for our example
+
+    # Cromparison plot
+    df, fig, n, p = create_comparison_plot(n, _lambda)
+    return create_comparison_plot, df, fig, n, p
+
+
+@app.cell(hide_code=True)
+def _(df, fig, fig_to_image, mo, n, p):
+    # table of values
+    _styled_df = df.style.format({
+        f'Binomial(n={n}, p={p:.6f})': '{:.6f}',
+        f'Poisson(λ=5)': '{:.6f}',
+        'Difference': '{:.6f}'
+    })
+
+    # Calculate the max absolute difference
+    _max_diff = df['Difference'].max()
+
+    # output
+    _chart = mo.image(fig_to_image(fig), width="100%")
+    _explanation = mo.md(f"**Maximum absolute difference between distributions: {_max_diff:.6f}**")
+    _table = mo.ui.table(df)
+
+    mo.vstack([_chart, _explanation, _table])
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        As you can see from the interactive comparison above, as the number of intervals increases, the binomial distribution approaches the Poisson distribution! This is not a coincidence - the Poisson distribution is actually the limiting case of the binomial distribution when:
+
+        - The number of trials $n$ approaches infinity
+        - The probability of success $p$ approaches zero
+        - The product $np = \lambda$ remains constant
+
+        This relationship is why the Poisson distribution is so useful - it's easier to work with than a binomial with a very large number of trials and a very small probability of success.
+
+        ## Derivation of the Poisson PMF
+
+        Let's derive the Poisson PMF by taking the limit of the binomial PMF as $n \to \infty$. We start with:
+
+        $P(X=x) = \lim_{n \rightarrow \infty} {n \choose x} (\lambda / n)^x(1-\lambda/n)^{n-x}$
+
+        While this expression looks intimidating, it simplifies nicely:
+
+        \begin{align}
+        P(X=x) 
+        &= \lim_{n \rightarrow \infty} {n \choose x} (\lambda / n)^x(1-\lambda/n)^{n-x}
+            && \text{Start: binomial in the limit}\\
+        &= \lim_{n \rightarrow \infty}
+            {n \choose x} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            \frac{(1-\lambda/n)^{n}}{(1-\lambda/n)^{x}} 
+            && \text{Expanding the power terms} \\
+        &= \lim_{n \rightarrow \infty}
+            \frac{n!}{(n-x)!x!} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            \frac{(1-\lambda/n)^{n}}{(1-\lambda/n)^{x}} 
+            && \text{Expanding the binomial term} \\
+        &= \lim_{n \rightarrow \infty}
+            \frac{n!}{(n-x)!x!} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            \frac{e^{-\lambda}}{(1-\lambda/n)^{x}} 
+            && \text{Using limit rule } \lim_{n \rightarrow \infty}(1-\lambda/n)^{n} = e^{-\lambda}\\
+        &= \lim_{n \rightarrow \infty}
+            \frac{n!}{(n-x)!x!} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            \frac{e^{-\lambda}}{1} 
+            && \text{As } n \to \infty \text{, } \lambda/n \to 0\\
+        &= \lim_{n \rightarrow \infty}
+            \frac{n!}{(n-x)!} \cdot
+            \frac{1}{x!} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            e^{-\lambda}
+            && \text{Rearranging terms}\\
+        &= \lim_{n \rightarrow \infty}
+            \frac{n^x}{1} \cdot
+            \frac{1}{x!} \cdot
+            \frac{\lambda^x}{n^x} \cdot
+            e^{-\lambda}
+            && \text{As } n \to \infty \text{, } \frac{n!}{(n-x)!} \approx n^x\\
+        &= \lim_{n \rightarrow \infty}
+            \frac{\lambda^x}{x!} \cdot
+            e^{-\lambda}
+            && \text{Canceling } n^x\\
+        &= 
+            \frac{\lambda^x \cdot e^{-\lambda}}{x!} 
+            && \text{Simplifying}\\
+        \end{align}
+
+        This gives us our elegant Poisson PMF formula: $P(X=x) = \frac{\lambda^x \cdot e^{-\lambda}}{x!}$
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Poisson Distribution in Python
+
+        Python's `scipy.stats` module provides functions to work with the Poisson distribution. Let's see how to calculate probabilities and generate random samples.
+
+        First, let's calculate some probabilities for our ride-sharing example with $\lambda = 5$:
+        """
+    )
+    return
+
+
+@app.cell
+def _(stats):
+    _lambda = 5
+
+    # Calculate probabilities for X = 1, 2, 3
+    p_1 = stats.poisson.pmf(1, _lambda)
+    p_2 = stats.poisson.pmf(2, _lambda)
+    p_3 = stats.poisson.pmf(3, _lambda)
+
+    print(f"P(X=1) = {p_1:.5f}")
+    print(f"P(X=2) = {p_2:.5f}")
+    print(f"P(X=3) = {p_3:.5f}")
+
+    # Calculate cumulative probability P(X ≤ 3)
+    p_leq_3 = stats.poisson.cdf(3, _lambda)
+    print(f"P(X≤3) = {p_leq_3:.5f}")
+
+    # Calculate probability P(X > 10)
+    p_gt_10 = 1 - stats.poisson.cdf(10, _lambda)
+    print(f"P(X>10) = {p_gt_10:.5f}")
+    return p_1, p_2, p_3, p_gt_10, p_leq_3
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""We can also generate random samples from a Poisson distribution and visualize their distribution:""")
+    return
+
+
+@app.cell(hide_code=True)
+def _(np, plt, stats):
+    def create_samples_plot(lambda_value, sample_size=1000):
+        # Random samples
+        samples = stats.poisson.rvs(lambda_value, size=sample_size)
+
+        # theoretical PMF
+        x_values = np.arange(0, max(samples) + 1)
+        pmf_values = stats.poisson.pmf(x_values, lambda_value)
+
+        # histograms to compare
+        fig, ax = plt.subplots(figsize=(10, 6))
+
+        # samples as a histogram
+        ax.hist(samples, bins=np.arange(-0.5, max(samples) + 1.5, 1), 
+                alpha=0.7, density=True, label='Random Samples')
+
+        # theoretical PMF
+        ax.plot(x_values, pmf_values, 'ro-', label='Theoretical PMF')
+
+        # labels and title
+        ax.set_xlabel('Number of Events')
+        ax.set_ylabel('Relative Frequency / Probability')
+        ax.set_title(f'1000 Random Samples from Poisson(λ={lambda_value})')
+        ax.legend()
+        ax.grid(alpha=0.3)
+
+        # annotations
+        ax.annotate(f'Sample Mean: {np.mean(samples):.2f}', 
+                    xy=(0.7, 0.9), xycoords='axes fraction',
+                    bbox=dict(boxstyle='round,pad=0.5', fc='yellow', alpha=0.3))
+        ax.annotate(f'Theoretical Mean: {lambda_value:.2f}', 
+                    xy=(0.7, 0.8), xycoords='axes fraction',
+                    bbox=dict(boxstyle='round,pad=0.5', fc='lightgreen', alpha=0.3))
+
+        plt.tight_layout()
+        return plt.gca()
+
+    # Use a lambda value of 5 for this example
+    _lambda = 5
+    create_samples_plot(_lambda)
+    return (create_samples_plot,)
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Changing Time Frames
+
+        One important property of the Poisson distribution is that the rate parameter $\lambda$ scales linearly with the time interval. If events occur at a rate of $\lambda$ per unit time, then over a period of $t$ units, the rate parameter becomes $\lambda \cdot t$.
+
+        For example, if a website receives an average of 5 requests per minute, what is the distribution of requests over a 20-minute period?
+
+        The rate parameter for the 20-minute period would be $\lambda = 5 \cdot 20 = 100$ requests.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    rate_slider = mo.ui.slider(
+        start = 0.1,
+        stop = 10,
+        step=0.1,
+        value=5,
+        label="Rate per unit time (λ)"
+    )
+
+    time_slider = mo.ui.slider(
+        start = 1,
+        stop = 60,
+        step=1,
+        value=20,
+        label="Time period (t units)"
+    )
+
+    controls = mo.vstack([
+        mo.md("### Adjust Parameters to See How Time Scaling Works"),
+        mo.hstack([rate_slider, time_slider], justify="space-between")
+    ])
+    return controls, rate_slider, time_slider
+
+
+@app.cell
+def _(controls):
+    controls.center()
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo, np, plt, rate_slider, stats, time_slider):
+    def create_time_scaling_plot(rate, time_period):
+        # scaled rate parameter
+        lambda_value = rate * time_period
+
+        # PMF for values
+        max_x = max(30, int(lambda_value * 1.5))
+        x = np.arange(0, max_x + 1)
+        pmf = stats.poisson.pmf(x, lambda_value)
+
+        # plot
+        fig, ax = plt.subplots(figsize=(10, 6))
+
+        # PMF as bars
+        ax.bar(x, pmf, color='royalblue', alpha=0.7, 
+            label=f'PMF: Poisson(λ={lambda_value:.1f})')
+
+        # vertical line for mean
+        ax.axvline(x=lambda_value, color='red', linestyle='--', linewidth=2,
+                label=f'Mean = {lambda_value:.1f}')
+
+        # labels and title
+        ax.set_xlabel('Number of Events')
+        ax.set_ylabel('Probability')
+        ax.set_title(f'Poisson Distribution Over {time_period} Units (Rate = {rate}/unit)')
+
+        # better visualization if lambda is large
+        if lambda_value > 10:
+            ax.set_xlim(lambda_value - 4*np.sqrt(lambda_value), 
+                         lambda_value + 4*np.sqrt(lambda_value))
+
+        ax.legend()
+        ax.grid(alpha=0.3)
+
+        plt.tight_layout()
+
+        # Create relevant info markdown
+        info_text = f"""
+        When the rate is **{rate}** events per unit time and we observe for **{time_period}** units:
+
+        - The expected number of events is **{lambda_value:.1f}**
+        - The variance is also **{lambda_value:.1f}**
+        - The standard deviation is **{np.sqrt(lambda_value):.2f}**
+        - P(X=0) = {stats.poisson.pmf(0, lambda_value):.4f} (probability of no events)
+        - P(X≥10) = {1 - stats.poisson.cdf(9, lambda_value):.4f} (probability of 10 or more events)
+        """
+
+        return plt.gca(), info_text
+
+    # parameters from sliders
+    _rate = rate_slider.value
+    _time = time_slider.value
+
+    # store
+    _plot, _info_text = create_time_scaling_plot(_rate, _time)
+
+    # Display info as markdown
+    info = mo.md(_info_text)
+
+    mo.vstack([_plot, info], justify="center")
+    return create_time_scaling_plot, info
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## 🤔 Test Your Understanding
+        Pick which of these statements about Poisson distributions you think are correct:
+
+        /// details | The variance of a Poisson distribution is always equal to its mean
+        ✅ Correct! For a Poisson distribution with parameter $\lambda$, both the mean and variance equal $\lambda$.
+        ///
+
+        /// details | The Poisson distribution can be used to model the number of successes in a fixed number of trials
+        ❌ Incorrect! That's the binomial distribution. The Poisson distribution models the number of events in a fixed interval of time or space, not a fixed number of trials.
+        ///
+
+        /// details | If $X \sim \text{Poisson}(\lambda_1)$ and $Y \sim \text{Poisson}(\lambda_2)$ are independent, then $X + Y \sim \text{Poisson}(\lambda_1 + \lambda_2)$
+        ✅ Correct! The sum of independent Poisson random variables is also a Poisson random variable with parameter equal to the sum of the individual parameters.
+        ///
+
+        /// details | As $\lambda$ increases, the Poisson distribution approaches a normal distribution
+        ✅ Correct! For large values of $\lambda$ (generally $\lambda > 10$), the Poisson distribution is approximately normal with mean $\lambda$ and variance $\lambda$.
+        ///
+
+        /// details | The probability of zero events in a Poisson process is always less than the probability of one event
+        ❌ Incorrect! For $\lambda < 1$, the probability of zero events ($e^{-\lambda}$) is actually greater than the probability of one event ($\lambda e^{-\lambda}$).
+        ///
+
+        /// details | The Poisson distribution has a single parameter $\lambda$, which always equals the average number of events per time period
+        ✅ Correct! The parameter $\lambda$ represents the average rate of events, and it uniquely defines the distribution.
+        ///
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(
+        r"""
+        ## Summary
+
+        The Poisson distribution is one of those incredibly useful tools that shows up all over the place. I've always found it fascinating how such a simple formula can model so many real-world phenomena - from website traffic to radioactive decay.
+
+        What makes the Poisson really cool is that it emerges naturally as we try to model rare events occurring over a continuous interval. Remember that visualization where we kept dividing time into smaller and smaller chunks? As we showed, when you take a binomial distribution and let the number of trials approach infinity while keeping the expected value constant, you end up with the elegant Poisson formula.
+
+        The key things to remember about the Poisson distribution:
+
+        - It models the number of events occurring in a fixed interval of time or space, assuming events happen at a constant average rate and independently of each other
+
+        - Its PMF is given by the elegantly simple formula $P(X=k) = \frac{\lambda^k e^{-\lambda}}{k!}$
+
+        - Both the mean and variance equal the parameter $\lambda$, which represents the average number of events per interval
+
+        - It's related to the binomial distribution as a limiting case when $n \to \infty$, $p \to 0$, and $np = \lambda$ remains constant
+
+        - The rate parameter scales linearly with the length of the interval - if events occur at rate $\lambda$ per unit time, then over $t$ units, the parameter becomes $\lambda t$
+
+        From modeling website traffic and customer arrivals to defects in manufacturing and radioactive decay, the Poisson distribution provides a powerful and mathematically elegant way to understand random occurrences in our world.
+        """
+    )
+    return
+
+
+@app.cell(hide_code=True)
+def _(mo):
+    mo.md(r"""Appendix code (helper functions, variables, etc.):""")
+    return
+
+
+@app.cell
+def _():
+    import marimo as mo
+    return (mo,)
+
+
+@app.cell(hide_code=True)
+def _():
+    import numpy as np
+    import matplotlib.pyplot as plt
+    import scipy.stats as stats
+    import pandas as pd
+    import altair as alt
+    from wigglystuff import TangleSlider
+    return TangleSlider, alt, np, pd, plt, stats
+
+
+@app.cell(hide_code=True)
+def _():
+    import io
+    import base64
+    from matplotlib.figure import Figure
+
+    # Helper function to convert mpl figure to an image format mo.image can hopefully handle
+    def fig_to_image(fig):
+        buf = io.BytesIO()
+        fig.savefig(buf, format='png')
+        buf.seek(0)
+        data = f"data:image/png;base64,{base64.b64encode(buf.read()).decode('utf-8')}"
+        return data
+    return Figure, base64, fig_to_image, io
+
+
+if __name__ == "__main__":
+    app.run()

python/006_dictionaries.py

@@ -196,13 +196,13 @@ def _():
 
 @app.cell
 def _(mo, nested_data):
-    mo.md(f"Alice's age: {nested_data["users"]["alice"]["age"]}")
+    mo.md(f"Alice's age: {nested_data['users']['alice']['age']}")
     return
 
 
 @app.cell
 def _(mo, nested_data):
-    mo.md(f"Bob's interests: {nested_data["users"]["bob"]["interests"]}")
+    mo.md(f"Bob's interests: {nested_data['users']['bob']['interests']}")
     return
 
 

scripts/build.py

@@ -0,0 +1,1523 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import argparse
+import json
+from typing import List, Dict, Any
+from pathlib import Path
+import re
+
+
+def export_html_wasm(notebook_path: str, output_dir: str, as_app: bool = False) -> bool:
+    """Export a single marimo notebook to HTML format.
+
+    Returns:
+        bool: True if export succeeded, False otherwise
+    """
+    output_path = notebook_path.replace(".py", ".html")
+
+    cmd = ["marimo", "export", "html-wasm"]
+    if as_app:
+        print(f"Exporting {notebook_path} to {output_path} as app")
+        cmd.extend(["--mode", "run", "--no-show-code"])
+    else:
+        print(f"Exporting {notebook_path} to {output_path} as notebook")
+        cmd.extend(["--mode", "edit"])
+
+    try:
+        output_file = os.path.join(output_dir, output_path)
+        os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+        cmd.extend([notebook_path, "-o", output_file])
+        print(f"Running command: {' '.join(cmd)}")
+        
+        # Use Popen to handle interactive prompts
+        process = subprocess.Popen(
+            cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True
+        )
+        
+        # Send 'Y' to the prompt
+        stdout, stderr = process.communicate(input="Y\n", timeout=60)
+        
+        if process.returncode != 0:
+            print(f"Error exporting {notebook_path}:")
+            print(f"Command: {' '.join(cmd)}")
+            print(f"Return code: {process.returncode}")
+            print(f"Stdout: {stdout}")
+            print(f"Stderr: {stderr}")
+            return False
+            
+        print(f"Successfully exported {notebook_path} to {output_file}")
+        return True
+    except subprocess.TimeoutExpired:
+        print(f"Timeout exporting {notebook_path} - command took too long to execute")
+        return False
+    except subprocess.CalledProcessError as e:
+        print(f"Error exporting {notebook_path}:")
+        print(e.stderr)
+        return False
+    except Exception as e:
+        print(f"Unexpected error exporting {notebook_path}: {e}")
+        return False
+
+
+def get_course_metadata(course_dir: Path) -> Dict[str, Any]:
+    """Extract metadata from a course directory."""
+    metadata = {
+        "id": course_dir.name,
+        "title": course_dir.name.replace("_", " ").title(),
+        "description": "",
+        "notebooks": []
+    }
+    
+    # Try to read README.md for description
+    readme_path = course_dir / "README.md"
+    if readme_path.exists():
+        with open(readme_path, "r", encoding="utf-8") as f:
+            content = f.read()
+            # Extract first paragraph as description
+            if content:
+                lines = content.split("\n")
+                # Skip title line if it exists
+                start_idx = 1 if lines and lines[0].startswith("#") else 0
+                description_lines = []
+                for line in lines[start_idx:]:
+                    if line.strip() and not line.startswith("#"):
+                        description_lines.append(line)
+                    elif description_lines:  # Stop at the next heading
+                        break
+                description = " ".join(description_lines).strip()
+                # Clean up the description
+                description = description.replace("_", "")
+                description = description.replace("[work in progress]", "")
+                description = description.replace("(https://github.com/marimo-team/learn/issues/51)", "")
+                # Remove any other GitHub issue links
+                description = re.sub(r'\[.*?\]\(https://github\.com/.*?/issues/\d+\)', '', description)
+                description = re.sub(r'https://github\.com/.*?/issues/\d+', '', description)
+                # Clean up any double spaces
+                description = re.sub(r'\s+', ' ', description).strip()
+                metadata["description"] = description
+    
+    return metadata
+
+
+def organize_notebooks_by_course(all_notebooks: List[str]) -> Dict[str, Dict[str, Any]]:
+    """Organize notebooks by course."""
+    courses = {}
+    
+    for notebook_path in all_notebooks:
+        path = Path(notebook_path)
+        course_id = path.parts[0]
+        
+        if course_id not in courses:
+            course_dir = Path(course_id)
+            courses[course_id] = get_course_metadata(course_dir)
+        
+        # Extract notebook info
+        filename = path.name
+        notebook_id = path.stem
+        
+        # Try to extract order from filename (e.g., 001_numbers.py -> 1)
+        order = 999
+        if "_" in notebook_id:
+            try:
+                order_str = notebook_id.split("_")[0]
+                order = int(order_str)
+            except ValueError:
+                pass
+        
+        # Create display name by removing order prefix and underscores
+        display_name = notebook_id
+        if "_" in notebook_id:
+            display_name = "_".join(notebook_id.split("_")[1:])
+        
+        # Convert display name to title case, but handle italics properly
+        parts = display_name.split("_")
+        formatted_parts = []
+        
+        i = 0
+        while i < len(parts):
+            if i + 1 < len(parts) and parts[i] == "" and parts[i+1] == "":
+                # Skip empty parts that might come from consecutive underscores
+                i += 2
+                continue
+                
+            if i + 1 < len(parts) and (parts[i] == "" or parts[i+1] == ""):
+                # This is an italics marker
+                if parts[i] == "":
+                    # Opening italics
+                    text_part = parts[i+1].replace("_", " ").title()
+                    formatted_parts.append(f"<em>{text_part}</em>")
+                    i += 2
+                else:
+                    # Text followed by italics marker
+                    text_part = parts[i].replace("_", " ").title()
+                    formatted_parts.append(text_part)
+                    i += 1
+            else:
+                # Regular text
+                text_part = parts[i].replace("_", " ").title()
+                formatted_parts.append(text_part)
+                i += 1
+        
+        display_name = " ".join(formatted_parts)
+        
+        courses[course_id]["notebooks"].append({
+            "id": notebook_id,
+            "path": notebook_path,
+            "display_name": display_name,
+            "order": order,
+            "original_number": notebook_id.split("_")[0] if "_" in notebook_id else ""
+        })
+    
+    # Sort notebooks by order
+    for course_id in courses:
+        courses[course_id]["notebooks"].sort(key=lambda x: x["order"])
+    
+    return courses
+
+
+def generate_eva_css() -> str:
+    """Generate Neon Genesis Evangelion inspired CSS with light/dark mode support."""
+    return """
+    :root {
+        /* Light mode colors (default) */
+        --eva-purple: #7209b7;
+        --eva-green: #1c7361;
+        --eva-orange: #e65100;
+        --eva-blue: #0039cb;
+        --eva-red: #c62828;
+        --eva-black: #f5f5f5;
+        --eva-dark: #e0e0e0;
+        --eva-terminal-bg: rgba(255, 255, 255, 0.9);
+        --eva-text: #333333;
+        --eva-border-radius: 4px;
+        --eva-transition: all 0.3s ease;
+    }
+    
+    /* Dark mode colors */
+    [data-theme="dark"] {
+        --eva-purple: #9a1eb3;
+        --eva-green: #1c7361;
+        --eva-orange: #ff6600;
+        --eva-blue: #0066ff;
+        --eva-red: #ff0000;
+        --eva-black: #111111;
+        --eva-dark: #222222;
+        --eva-terminal-bg: rgba(0, 0, 0, 0.85);
+        --eva-text: #e0e0e0;
+    }
+    
+    body {
+        background-color: var(--eva-black);
+        color: var(--eva-text);
+        font-family: 'Courier New', monospace;
+        margin: 0;
+        padding: 0;
+        line-height: 1.6;
+        transition: background-color 0.3s ease, color 0.3s ease;
+    }
+    
+    .eva-container {
+        max-width: 1200px;
+        margin: 0 auto;
+        padding: 2rem;
+    }
+    
+    .eva-header {
+        border-bottom: 2px solid var(--eva-green);
+        padding-bottom: 1rem;
+        margin-bottom: 2rem;
+        display: flex;
+        justify-content: space-between;
+        align-items: center;
+        position: sticky;
+        top: 0;
+        background-color: var(--eva-black);
+        z-index: 100;
+        backdrop-filter: blur(5px);
+        padding-top: 1rem;
+        transition: background-color 0.3s ease;
+    }
+    
+    [data-theme="light"] .eva-header {
+        background-color: rgba(245, 245, 245, 0.95);
+    }
+    
+    .eva-logo {
+        font-size: 2.5rem;
+        font-weight: bold;
+        color: var(--eva-green);
+        text-transform: uppercase;
+        letter-spacing: 2px;
+        text-shadow: 0 0 10px rgba(28, 115, 97, 0.5);
+    }
+    
+    [data-theme="light"] .eva-logo {
+        text-shadow: 0 0 10px rgba(28, 115, 97, 0.3);
+    }
+    
+    .eva-nav {
+        display: flex;
+        gap: 1.5rem;
+        align-items: center;
+    }
+    
+    .eva-nav a {
+        color: var(--eva-text);
+        text-decoration: none;
+        text-transform: uppercase;
+        font-size: 0.9rem;
+        letter-spacing: 1px;
+        transition: color 0.3s;
+        position: relative;
+        padding: 0.5rem 0;
+    }
+    
+    .eva-nav a:hover {
+        color: var(--eva-green);
+    }
+    
+    .eva-nav a:hover::after {
+        content: '';
+        position: absolute;
+        bottom: -5px;
+        left: 0;
+        width: 100%;
+        height: 2px;
+        background-color: var(--eva-green);
+        animation: scanline 1.5s linear infinite;
+    }
+    
+    .theme-toggle {
+        background: none;
+        border: none;
+        color: var(--eva-text);
+        cursor: pointer;
+        font-size: 1.2rem;
+        padding: 0.5rem;
+        margin-left: 1rem;
+        transition: color 0.3s;
+    }
+    
+    .theme-toggle:hover {
+        color: var(--eva-green);
+    }
+    
+    .eva-hero {
+        background-color: var(--eva-terminal-bg);
+        border: 1px solid var(--eva-green);
+        padding: 3rem 2rem;
+        margin-bottom: 3rem;
+        position: relative;
+        overflow: hidden;
+        border-radius: var(--eva-border-radius);
+        display: flex;
+        flex-direction: column;
+        align-items: flex-start;
+        background-image: linear-gradient(45deg, rgba(0, 0, 0, 0.9), rgba(0, 0, 0, 0.7)), url('https://raw.githubusercontent.com/marimo-team/marimo/main/docs/_static/marimo-logotype-thick.svg');
+        background-size: cover;
+        background-position: center;
+        background-blend-mode: overlay;
+        transition: background-color 0.3s ease, border-color 0.3s ease;
+    }
+    
+    [data-theme="light"] .eva-hero {
+        background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.9), rgba(255, 255, 255, 0.7)), url('https://raw.githubusercontent.com/marimo-team/marimo/main/docs/_static/marimo-logotype-thick.svg');
+    }
+    
+    .eva-hero::before {
+        content: '';
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 2px;
+        background-color: var(--eva-green);
+        animation: scanline 3s linear infinite;
+    }
+    
+    .eva-hero h1 {
+        font-size: 2.5rem;
+        margin-bottom: 1rem;
+        color: var(--eva-green);
+        text-transform: uppercase;
+        letter-spacing: 2px;
+        text-shadow: 0 0 10px rgba(28, 115, 97, 0.5);
+    }
+    
+    [data-theme="light"] .eva-hero h1 {
+        text-shadow: 0 0 10px rgba(28, 115, 97, 0.3);
+    }
+    
+    .eva-hero p {
+        font-size: 1.1rem;
+        max-width: 800px;
+        margin-bottom: 2rem;
+        line-height: 1.8;
+    }
+    
+    .eva-features {
+        display: grid;
+        grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+        gap: 2rem;
+        margin-bottom: 3rem;
+    }
+    
+    .eva-feature {
+        background-color: var(--eva-terminal-bg);
+        border: 1px solid var(--eva-blue);
+        padding: 1.5rem;
+        border-radius: var(--eva-border-radius);
+        transition: var(--eva-transition);
+        position: relative;
+        overflow: hidden;
+    }
+    
+    .eva-feature:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 10px 20px rgba(0, 102, 255, 0.2);
+    }
+    
+    .eva-feature-icon {
+        font-size: 2rem;
+        margin-bottom: 1rem;
+        color: var(--eva-blue);
+    }
+    
+    .eva-feature h3 {
+        font-size: 1.3rem;
+        margin-bottom: 1rem;
+        color: var(--eva-blue);
+    }
+    
+    .eva-section-title {
+        font-size: 2rem;
+        color: var(--eva-green);
+        margin-bottom: 2rem;
+        text-transform: uppercase;
+        letter-spacing: 2px;
+        text-align: center;
+        position: relative;
+        padding-bottom: 1rem;
+    }
+    
+    .eva-section-title::after {
+        content: '';
+        position: absolute;
+        bottom: 0;
+        left: 50%;
+        transform: translateX(-50%);
+        width: 100px;
+        height: 2px;
+        background-color: var(--eva-green);
+    }
+    
+    /* Flashcard view for courses */
+    .eva-courses {
+        display: grid;
+        grid-template-columns: repeat(auto-fill, minmax(350px, 1fr));
+        gap: 2rem;
+    }
+    
+    .eva-course {
+        background-color: var(--eva-terminal-bg);
+        border: 1px solid var(--eva-purple);
+        border-radius: var(--eva-border-radius);
+        transition: var(--eva-transition), height 0.4s cubic-bezier(0.19, 1, 0.22, 1);
+        position: relative;
+        overflow: hidden;
+        height: 350px;
+        display: flex;
+        flex-direction: column;
+    }
+    
+    .eva-course:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 10px 20px rgba(154, 30, 179, 0.3);
+    }
+    
+    .eva-course::after {
+        content: '';
+        position: absolute;
+        bottom: 0;
+        left: 0;
+        width: 100%;
+        height: 2px;
+        background-color: var(--eva-purple);
+        animation: scanline 2s linear infinite;
+    }
+    
+    .eva-course-badge {
+        position: absolute;
+        top: 15px;
+        right: -40px;
+        background: linear-gradient(135deg, var(--eva-orange) 0%, #ff9500 100%);
+        color: var(--eva-black);
+        font-size: 0.65rem;
+        padding: 0.3rem 2.5rem;
+        text-transform: uppercase;
+        font-weight: bold;
+        z-index: 3;
+        letter-spacing: 1px;
+        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.3);
+        transform: rotate(45deg);
+        text-shadow: 0 1px 1px rgba(255, 255, 255, 0.2);
+        border-top: 1px solid rgba(255, 255, 255, 0.3);
+        border-bottom: 1px solid rgba(0, 0, 0, 0.2);
+        white-space: nowrap;
+        overflow: hidden;
+    }
+    
+    .eva-course-badge i {
+        margin-right: 4px;
+        font-size: 0.7rem;
+    }
+    
+    [data-theme="light"] .eva-course-badge {
+        box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2);
+        text-shadow: 0 1px 1px rgba(255, 255, 255, 0.4);
+    }
+    
+    .eva-course-badge::before {
+        content: '';
+        position: absolute;
+        left: 0;
+        top: 0;
+        width: 100%;
+        height: 100%;
+        background: linear-gradient(to right, transparent, rgba(255, 255, 255, 0.3), transparent);
+        animation: scanline 2s linear infinite;
+    }
+    
+    .eva-course-header {
+        padding: 1rem 1.5rem;
+        cursor: pointer;
+        display: flex;
+        justify-content: space-between;
+        align-items: center;
+        border-bottom: 1px solid rgba(154, 30, 179, 0.3);
+        z-index: 2;
+        background-color: var(--eva-terminal-bg);
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: 3.5rem;
+        box-sizing: border-box;
+    }
+    
+    .eva-course-title {
+        font-size: 1.3rem;
+        color: var(--eva-purple);
+        text-transform: uppercase;
+        letter-spacing: 1px;
+        margin: 0;
+    }
+    
+    .eva-course-toggle {
+        color: var(--eva-purple);
+        font-size: 1.5rem;
+        transition: transform 0.4s cubic-bezier(0.19, 1, 0.22, 1);
+    }
+    
+    .eva-course.active .eva-course-toggle {
+        transform: rotate(180deg);
+    }
+    
+    .eva-course-front {
+        display: flex;
+        flex-direction: column;
+        justify-content: space-between;
+        padding: 1.5rem;
+        margin-top: 3.5rem;
+        transition: opacity 0.3s ease, transform 0.3s ease;
+        position: absolute;
+        top: 0;
+        left: 0;
+        width: 100%;
+        height: calc(100% - 3.5rem);
+        background-color: var(--eva-terminal-bg);
+        z-index: 1;
+        box-sizing: border-box;
+    }
+    
+    .eva-course.active .eva-course-front {
+        opacity: 0;
+        transform: translateY(-10px);
+        pointer-events: none;
+    }
+    
+    .eva-course-description {
+        margin-top: 0.5rem;
+        margin-bottom: 1.5rem;
+        font-size: 0.9rem;
+        line-height: 1.6;
+        flex-grow: 1;
+        overflow: hidden;
+        display: -webkit-box;
+        -webkit-line-clamp: 4;
+        -webkit-box-orient: vertical;
+        max-height: 150px;
+    }
+    
+    .eva-course-stats {
+        display: flex;
+        justify-content: space-between;
+        font-size: 0.8rem;
+        color: var(--eva-text);
+        opacity: 0.7;
+    }
+    
+    .eva-course-content {
+        position: absolute;
+        top: 3.5rem;
+        left: 0;
+        width: 100%;
+        height: calc(100% - 3.5rem);
+        padding: 1.5rem;
+        background-color: var(--eva-terminal-bg);
+        transition: opacity 0.3s ease, transform 0.3s ease;
+        opacity: 0;
+        transform: translateY(10px);
+        pointer-events: none;
+        overflow-y: auto;
+        z-index: 1;
+        box-sizing: border-box;
+    }
+    
+    .eva-course.active .eva-course-content {
+        opacity: 1;
+        transform: translateY(0);
+        pointer-events: auto;
+    }
+    
+    .eva-course.active {
+        height: auto;
+        min-height: 350px;
+        max-height: 800px;
+        transition: height 0.4s cubic-bezier(0.19, 1, 0.22, 1), transform 0.3s ease, box-shadow 0.3s ease;
+    }
+    
+    .eva-notebooks {
+        margin-top: 1rem;
+        display: grid;
+        grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
+        gap: 0.75rem;
+    }
+    
+    .eva-notebook {
+        margin-bottom: 0.5rem;
+        padding: 0.75rem;
+        border-left: 2px solid var(--eva-blue);
+        transition: all 0.25s ease;
+        display: flex;
+        align-items: center;
+        background-color: rgba(0, 0, 0, 0.2);
+        border-radius: 0 var(--eva-border-radius) var(--eva-border-radius) 0;
+        opacity: 1;
+        transform: translateX(0);
+    }
+    
+    [data-theme="light"] .eva-notebook {
+        background-color: rgba(0, 0, 0, 0.05);
+    }
+    
+    .eva-notebook:hover {
+        background-color: rgba(0, 102, 255, 0.1);
+        padding-left: 1rem;
+        transform: translateX(3px);
+    }
+    
+    .eva-notebook a {
+        color: var(--eva-text);
+        text-decoration: none;
+        display: block;
+        font-size: 0.9rem;
+        flex-grow: 1;
+    }
+    
+    .eva-notebook a:hover {
+        color: var(--eva-blue);
+    }
+    
+    .eva-notebook-number {
+        color: var(--eva-blue);
+        font-size: 0.8rem;
+        margin-right: 0.75rem;
+        opacity: 0.7;
+        min-width: 24px;
+        font-weight: bold;
+    }
+    
+    .eva-button {
+        display: inline-block;
+        background-color: transparent;
+        color: var(--eva-green);
+        border: 1px solid var(--eva-green);
+        padding: 0.7rem 1.5rem;
+        text-decoration: none;
+        text-transform: uppercase;
+        font-size: 0.9rem;
+        letter-spacing: 1px;
+        transition: var(--eva-transition);
+        cursor: pointer;
+        border-radius: var(--eva-border-radius);
+        position: relative;
+        overflow: hidden;
+    }
+    
+    .eva-button:hover {
+        background-color: var(--eva-green);
+        color: var(--eva-black);
+    }
+    
+    .eva-button::after {
+        content: '';
+        position: absolute;
+        top: 0;
+        left: -100%;
+        width: 100%;
+        height: 100%;
+        background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.2), transparent);
+        transition: 0.5s;
+    }
+    
+    .eva-button:hover::after {
+        left: 100%;
+    }
+    
+    .eva-course-button {
+        margin-top: 1rem;
+        margin-bottom: 1rem;
+        align-self: center;
+    }
+    
+    .eva-cta {
+        background-color: var(--eva-terminal-bg);
+        border: 1px solid var(--eva-orange);
+        padding: 3rem 2rem;
+        margin: 4rem 0;
+        text-align: center;
+        border-radius: var(--eva-border-radius);
+        position: relative;
+        overflow: hidden;
+    }
+    
+    .eva-cta h2 {
+        font-size: 2rem;
+        color: var(--eva-orange);
+        margin-bottom: 1.5rem;
+        text-transform: uppercase;
+    }
+    
+    .eva-cta p {
+        max-width: 600px;
+        margin: 0 auto 2rem;
+        font-size: 1.1rem;
+    }
+    
+    .eva-cta .eva-button {
+        color: var(--eva-orange);
+        border-color: var(--eva-orange);
+    }
+    
+    .eva-cta .eva-button:hover {
+        background-color: var(--eva-orange);
+        color: var(--eva-black);
+    }
+    
+    .eva-footer {
+        margin-top: 4rem;
+        padding-top: 2rem;
+        border-top: 2px solid var(--eva-green);
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+        gap: 2rem;
+    }
+    
+    .eva-footer-logo {
+        max-width: 200px;
+        margin-bottom: 1rem;
+    }
+    
+    .eva-footer-links {
+        display: flex;
+        gap: 1.5rem;
+        margin-bottom: 1.5rem;
+    }
+    
+    .eva-footer-links a {
+        color: var(--eva-text);
+        text-decoration: none;
+        transition: var(--eva-transition);
+    }
+    
+    .eva-footer-links a:hover {
+        color: var(--eva-green);
+    }
+    
+    .eva-social-links {
+        display: flex;
+        gap: 1.5rem;
+        margin-bottom: 1.5rem;
+    }
+    
+    .eva-social-links a {
+        color: var(--eva-text);
+        font-size: 1.5rem;
+        transition: var(--eva-transition);
+    }
+    
+    .eva-social-links a:hover {
+        color: var(--eva-green);
+        transform: translateY(-3px);
+    }
+    
+    .eva-footer-copyright {
+        font-size: 0.9rem;
+        text-align: center;
+    }
+    
+    .eva-search {
+        position: relative;
+        margin-bottom: 3rem;
+    }
+    
+    .eva-search input {
+        width: 100%;
+        padding: 1rem;
+        background-color: var(--eva-terminal-bg);
+        border: 1px solid var(--eva-green);
+        color: var(--eva-text);
+        font-family: 'Courier New', monospace;
+        font-size: 1rem;
+        border-radius: var(--eva-border-radius);
+        outline: none;
+        transition: var(--eva-transition);
+    }
+    
+    .eva-search input:focus {
+        box-shadow: 0 0 10px rgba(28, 115, 97, 0.3);
+    }
+    
+    .eva-search input::placeholder {
+        color: rgba(224, 224, 224, 0.5);
+    }
+    
+    [data-theme="light"] .eva-search input::placeholder {
+        color: rgba(51, 51, 51, 0.5);
+    }
+    
+    .eva-search-icon {
+        position: absolute;
+        right: 1rem;
+        top: 50%;
+        transform: translateY(-50%);
+        color: var(--eva-green);
+        font-size: 1.2rem;
+    }
+    
+    @keyframes scanline {
+        0% {
+            transform: translateX(-100%);
+        }
+        100% {
+            transform: translateX(100%);
+        }
+    }
+    
+    @keyframes blink {
+        0%, 100% {
+            opacity: 1;
+        }
+        50% {
+            opacity: 0;
+        }
+    }
+    
+    .eva-cursor {
+        display: inline-block;
+        width: 10px;
+        height: 1.2em;
+        background-color: var(--eva-green);
+        margin-left: 2px;
+        animation: blink 1s infinite;
+        vertical-align: middle;
+    }
+    
+    @media (max-width: 768px) {
+        .eva-courses {
+            grid-template-columns: 1fr;
+        }
+        
+        .eva-header {
+            flex-direction: column;
+            align-items: flex-start;
+            padding: 1rem;
+        }
+        
+        .eva-nav {
+            margin-top: 1rem;
+            flex-wrap: wrap;
+        }
+        
+        .eva-hero {
+            padding: 2rem 1rem;
+        }
+        
+        .eva-hero h1 {
+            font-size: 2rem;
+        }
+        
+        .eva-features {
+            grid-template-columns: 1fr;
+        }
+        
+        .eva-footer {
+            flex-direction: column;
+            align-items: center;
+            text-align: center;
+        }
+        
+        .eva-notebooks {
+            grid-template-columns: 1fr;
+        }
+    }
+    
+    .eva-course.closing .eva-course-content {
+        opacity: 0;
+        transform: translateY(10px);
+        transition: opacity 0.2s ease, transform 0.2s ease;
+    }
+    
+    .eva-course.closing .eva-course-front {
+        opacity: 1;
+        transform: translateY(0);
+        transition: opacity 0.3s ease 0.1s, transform 0.3s ease 0.1s;
+    }
+    """
+
+
+def get_html_header():
+    """Generate the HTML header with CSS and meta tags."""
+    return """<!DOCTYPE html>
+<html lang="en" data-theme="light">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Marimo Learn - Interactive Educational Notebooks</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
+    <style>
+{css}
+    </style>
+  </head>
+<body>
+    <div class="eva-container">
+        <header class="eva-header">
+            <div class="eva-logo">MARIMO LEARN</div>
+            <nav class="eva-nav">
+                <a href="#features">Features</a>
+                <a href="#courses">Courses</a>
+                <a href="#contribute">Contribute</a>
+                <a href="https://docs.marimo.io" target="_blank">Documentation</a>
+                <a href="https://github.com/marimo-team/learn" target="_blank">GitHub</a>
+                <button id="themeToggle" class="theme-toggle" aria-label="Toggle dark/light mode">
+                    <i class="fas fa-moon"></i>
+                </button>
+            </nav>
+        </header>"""
+
+
+def get_html_hero_section():
+    """Generate the hero section of the page."""
+    return """
+        <section class="eva-hero">
+            <h1>Interactive Learning with Marimo<span class="eva-cursor"></span></h1>
+            <p>
+                A curated collection of educational notebooks covering computer science, 
+                mathematics, data science, and more. Built with marimo - the reactive 
+                Python notebook that makes data exploration delightful.
+            </p>
+            <a href="#courses" class="eva-button">Explore Courses</a>
+        </section>"""
+
+
+def get_html_features_section():
+    """Generate the features section of the page."""
+    return """
+        <section id="features">
+            <h2 class="eva-section-title">Why Marimo Learn?</h2>
+            <div class="eva-features">
+                <div class="eva-feature">
+                    <div class="eva-feature-icon"><i class="fas fa-bolt"></i></div>
+                    <h3>Reactive Notebooks</h3>
+                    <p>Experience the power of reactive programming with marimo notebooks that automatically update when dependencies change.</p>
+                </div>
+                <div class="eva-feature">
+                    <div class="eva-feature-icon"><i class="fas fa-code"></i></div>
+                    <h3>Learn by Doing</h3>
+                    <p>Interactive examples and exercises help you understand concepts through hands-on practice.</p>
+                </div>
+                <div class="eva-feature">
+                    <div class="eva-feature-icon"><i class="fas fa-graduation-cap"></i></div>
+                    <h3>Comprehensive Courses</h3>
+                    <p>From Python basics to advanced optimization techniques, our courses cover a wide range of topics.</p>
+                </div>
+            </div>
+        </section>"""
+
+
+def get_html_courses_start():
+    """Generate the beginning of the courses section."""
+    return """
+        <section id="courses">
+            <h2 class="eva-section-title">Explore Courses</h2>
+            <div class="eva-search">
+                <input type="text" id="courseSearch" placeholder="Search courses and notebooks...">
+                <span class="eva-search-icon"><i class="fas fa-search"></i></span>
+            </div>
+            <div class="eva-courses">"""
+
+
+def generate_course_card(course, notebook_count, is_wip):
+    """Generate HTML for a single course card."""
+    html = f'<div class="eva-course" data-course-id="{course["id"]}">\n'
+    
+    # Add WIP badge if needed
+    if is_wip:
+        html += '    <div class="eva-course-badge"><i class="fas fa-code-branch"></i> In Progress</div>\n'
+    
+    html += f'''    <div class="eva-course-header">
+        <h2 class="eva-course-title">{course["title"]}</h2>
+        <span class="eva-course-toggle"><i class="fas fa-chevron-down"></i></span>
+    </div>
+    <div class="eva-course-front">
+        <p class="eva-course-description">{course["description"]}</p>
+        <div class="eva-course-stats">
+            <span><i class="fas fa-book"></i> {notebook_count} notebook{"s" if notebook_count != 1 else ""}</span>
+        </div>
+        <button class="eva-button eva-course-button">View Notebooks</button>
+    </div>
+    <div class="eva-course-content">
+        <div class="eva-notebooks">
+'''
+    
+    # Add notebooks
+    for i, notebook in enumerate(course["notebooks"]):
+        notebook_number = notebook.get("original_number", f"{i+1:02d}")
+        html += f'''            <div class="eva-notebook">
+                <span class="eva-notebook-number">{notebook_number}</span>
+                <a href="{notebook["path"].replace(".py", ".html")}" data-notebook-title="{notebook["display_name"]}">{notebook["display_name"]}</a>
+            </div>
+'''
+
+    html += '''        </div>
+    </div>
+</div>
+'''
+    return html
+
+
+def generate_course_cards(courses):
+    """Generate HTML for all course cards."""
+    html = ""
+    
+    # Define the custom order for courses
+    course_order = ["python", "probability", "polars", "optimization", "functional_programming"]
+    
+    # Create a dictionary of courses by ID for easy lookup
+    courses_by_id = {course["id"]: course for course in courses.values()}
+    
+    # Determine which courses are "work in progress" based on description or notebook count
+    work_in_progress = set()
+    for course_id, course in courses_by_id.items():
+        # Consider a course as "work in progress" if it has few notebooks or contains specific phrases
+        if (len(course["notebooks"]) < 5 or 
+            "work in progress" in course["description"].lower() or
+            "help us add" in course["description"].lower() or
+            "check back later" in course["description"].lower()):
+            work_in_progress.add(course_id)
+    
+    # First output courses in the specified order
+    for course_id in course_order:
+        if course_id in courses_by_id:
+            course = courses_by_id[course_id]
+            
+            # Skip if no notebooks
+            if not course["notebooks"]:
+                continue
+            
+            # Count notebooks
+            notebook_count = len(course["notebooks"])
+            
+            # Determine if this course is a work in progress
+            is_wip = course_id in work_in_progress
+            
+            html += generate_course_card(course, notebook_count, is_wip)
+            
+            # Remove from the dictionary so we don't output it again
+            del courses_by_id[course_id]
+    
+    # Then output any remaining courses alphabetically
+    sorted_remaining_courses = sorted(courses_by_id.values(), key=lambda x: x["title"])
+    
+    for course in sorted_remaining_courses:
+        # Skip if no notebooks
+        if not course["notebooks"]:
+            continue
+        
+        # Count notebooks
+        notebook_count = len(course["notebooks"])
+        
+        # Determine if this course is a work in progress
+        is_wip = course["id"] in work_in_progress
+        
+        html += generate_course_card(course, notebook_count, is_wip)
+    
+    return html
+
+
+def get_html_courses_end():
+    """Generate the end of the courses section."""
+    return """            </div>
+        </section>"""
+
+
+def get_html_contribute_section():
+    """Generate the contribute section."""
+    return """
+        <section id="contribute" class="eva-cta">
+            <h2>Contribute to Marimo Learn</h2>
+            <p>
+                Help us expand our collection of educational notebooks. Whether you're an expert in machine learning, 
+                statistics, or any other field, your contributions are welcome!
+            </p>
+            <a href="https://github.com/marimo-team/learn" target="_blank" class="eva-button">
+                <i class="fab fa-github"></i> Contribute on GitHub
+            </a>
+        </section>"""
+
+
+def get_html_footer():
+    """Generate the page footer."""
+    return """
+        <footer class="eva-footer">
+            <div class="eva-footer-logo">
+                <a href="https://marimo.io" target="_blank">
+                    <img src="https://marimo.io/logotype-wide.svg" alt="Marimo" width="200">
+                </a>
+            </div>
+            <div class="eva-social-links">
+                <a href="https://github.com/marimo-team" target="_blank" aria-label="GitHub"><i class="fab fa-github"></i></a>
+                <a href="https://marimo.io/discord?ref=learn" target="_blank" aria-label="Discord"><i class="fab fa-discord"></i></a>
+                <a href="https://twitter.com/marimo_io" target="_blank" aria-label="Twitter"><i class="fab fa-twitter"></i></a>
+                <a href="https://www.youtube.com/@marimo-team" target="_blank" aria-label="YouTube"><i class="fab fa-youtube"></i></a>
+                <a href="https://www.linkedin.com/company/marimo-io" target="_blank" aria-label="LinkedIn"><i class="fab fa-linkedin"></i></a>
+            </div>
+            <div class="eva-footer-links">
+                <a href="https://marimo.io" target="_blank">Website</a>
+                <a href="https://docs.marimo.io" target="_blank">Documentation</a>
+                <a href="https://github.com/marimo-team/learn" target="_blank">GitHub</a>
+            </div>
+            <div class="eva-footer-copyright">
+                © 2025 Marimo Inc. All rights reserved.
+            </div>
+        </footer>"""
+
+
+def get_html_scripts():
+    """Generate the JavaScript for the page."""
+    return """
+    <script>
+        // Set light theme as default immediately
+        document.documentElement.setAttribute('data-theme', 'light');
+        
+        document.addEventListener('DOMContentLoaded', function() {
+            // Theme toggle functionality
+            const themeToggle = document.getElementById('themeToggle');
+            const themeIcon = themeToggle.querySelector('i');
+            
+            // Update theme icon based on current theme
+            updateThemeIcon('light');
+            
+            // Check localStorage for saved theme preference
+            const savedTheme = localStorage.getItem('theme');
+            if (savedTheme && savedTheme !== 'light') {
+                document.documentElement.setAttribute('data-theme', savedTheme);
+                updateThemeIcon(savedTheme);
+            }
+            
+            // Toggle theme when button is clicked
+            themeToggle.addEventListener('click', () => {
+                const currentTheme = document.documentElement.getAttribute('data-theme');
+                const newTheme = currentTheme === 'dark' ? 'light' : 'dark';
+                
+                document.documentElement.setAttribute('data-theme', newTheme);
+                localStorage.setItem('theme', newTheme);
+                updateThemeIcon(newTheme);
+            });
+            
+            function updateThemeIcon(theme) {
+                if (theme === 'dark') {
+                    themeIcon.className = 'fas fa-sun';
+                } else {
+                    themeIcon.className = 'fas fa-moon';
+                }
+            }
+            
+            // Terminal typing effect for hero text
+            const heroTitle = document.querySelector('.eva-hero h1');
+            const heroText = document.querySelector('.eva-hero p');
+            const cursor = document.querySelector('.eva-cursor');
+            
+            const originalTitle = heroTitle.textContent;
+            const originalText = heroText.textContent.trim();
+            
+            heroTitle.textContent = '';
+            heroText.textContent = '';
+            
+            let titleIndex = 0;
+            let textIndex = 0;
+            
+            function typeTitle() {
+                if (titleIndex < originalTitle.length) {
+                    heroTitle.textContent += originalTitle.charAt(titleIndex);
+                    titleIndex++;
+                    setTimeout(typeTitle, 50);
+                } else {
+                    cursor.style.display = 'none';
+                    setTimeout(typeText, 500);
+                }
+            }
+            
+            function typeText() {
+                if (textIndex < originalText.length) {
+                    heroText.textContent += originalText.charAt(textIndex);
+                    textIndex++;
+                    setTimeout(typeText, 20);
+                }
+            }
+            
+            typeTitle();
+            
+            // Course toggle functionality - flashcard style
+            const courseHeaders = document.querySelectorAll('.eva-course-header');
+            const courseButtons = document.querySelectorAll('.eva-course-button');
+            
+            // Function to toggle course
+            function toggleCourse(course) {
+                const isActive = course.classList.contains('active');
+                
+                // First close all courses with a slight delay for better visual effect
+                document.querySelectorAll('.eva-course.active').forEach(c => {
+                    if (c !== course) {
+                        // Add a closing class for animation
+                        c.classList.add('closing');
+                        // Remove active class after a short delay
+                        setTimeout(() => {
+                            c.classList.remove('active');
+                            c.classList.remove('closing');
+                        }, 300);
+                    }
+                });
+                
+                // Toggle the clicked course
+                if (!isActive) {
+                    // Add a small delay before opening to allow others to close
+                    setTimeout(() => {
+                        course.classList.add('active');
+                        
+                        // Check if the course has any notebooks
+                        const notebooks = course.querySelectorAll('.eva-notebook');
+                        const content = course.querySelector('.eva-course-content');
+                        
+                        if (notebooks.length === 0 && !content.querySelector('.eva-empty-message')) {
+                            // If no notebooks, show a message
+                            const emptyMessage = document.createElement('p');
+                            emptyMessage.className = 'eva-empty-message';
+                            emptyMessage.textContent = 'No notebooks available in this course yet.';
+                            emptyMessage.style.color = 'var(--eva-text)';
+                            emptyMessage.style.fontStyle = 'italic';
+                            emptyMessage.style.opacity = '0.7';
+                            emptyMessage.style.textAlign = 'center';
+                            emptyMessage.style.padding = '1rem 0';
+                            content.appendChild(emptyMessage);
+                        }
+                        
+                        // Animate notebooks to appear sequentially
+                        notebooks.forEach((notebook, index) => {
+                            notebook.style.opacity = '0';
+                            notebook.style.transform = 'translateX(-10px)';
+                            setTimeout(() => {
+                                notebook.style.opacity = '1';
+                                notebook.style.transform = 'translateX(0)';
+                            }, 50 + (index * 30)); // Stagger the animations
+                        });
+                    }, 100);
+                }
+            }
+            
+            // Add click event to course headers
+            courseHeaders.forEach(header => {
+                header.addEventListener('click', function(e) {
+                    e.preventDefault();
+                    e.stopPropagation();
+                    
+                    const currentCourse = this.closest('.eva-course');
+                    toggleCourse(currentCourse);
+                });
+            });
+            
+            // Add click event to course buttons
+            courseButtons.forEach(button => {
+                button.addEventListener('click', function(e) {
+                    e.preventDefault();
+                    e.stopPropagation();
+                    
+                    const currentCourse = this.closest('.eva-course');
+                    toggleCourse(currentCourse);
+                });
+            });
+            
+            // Search functionality with improved matching
+            const searchInput = document.getElementById('courseSearch');
+            const courses = document.querySelectorAll('.eva-course');
+            const notebooks = document.querySelectorAll('.eva-notebook');
+            
+            searchInput.addEventListener('input', function() {
+                const searchTerm = this.value.toLowerCase();
+                
+                if (searchTerm === '') {
+                    // Reset all visibility
+                    courses.forEach(course => {
+                        course.style.display = 'block';
+                        course.classList.remove('active');
+                    });
+                    
+                    notebooks.forEach(notebook => {
+                        notebook.style.display = 'flex';
+                    });
+                    
+                    // Open the first course with notebooks by default when search is cleared
+                    for (let i = 0; i < courses.length; i++) {
+                        const courseNotebooks = courses[i].querySelectorAll('.eva-notebook');
+                        if (courseNotebooks.length > 0) {
+                            courses[i].classList.add('active');
+                            break;
+                        }
+                    }
+                    
+                    return;
+                }
+                
+                // First hide all courses
+                courses.forEach(course => {
+                    course.style.display = 'none';
+                    course.classList.remove('active');
+                });
+                
+                // Then show courses and notebooks that match the search
+                let hasResults = false;
+                
+                // Track which courses have matching notebooks
+                const coursesWithMatchingNotebooks = new Set();
+                
+                // First check notebooks
+                notebooks.forEach(notebook => {
+                    const notebookTitle = notebook.querySelector('a').getAttribute('data-notebook-title').toLowerCase();
+                    const matchesSearch = notebookTitle.includes(searchTerm);
+                    
+                    notebook.style.display = matchesSearch ? 'flex' : 'none';
+                    
+                    if (matchesSearch) {
+                        const course = notebook.closest('.eva-course');
+                        coursesWithMatchingNotebooks.add(course.getAttribute('data-course-id'));
+                        hasResults = true;
+                    }
+                });
+                
+                // Then check course titles and descriptions
+                courses.forEach(course => {
+                    const courseId = course.getAttribute('data-course-id');
+                    const courseTitle = course.querySelector('.eva-course-title').textContent.toLowerCase();
+                    const courseDescription = course.querySelector('.eva-course-description').textContent.toLowerCase();
+                    
+                    const courseMatches = courseTitle.includes(searchTerm) || courseDescription.includes(searchTerm);
+                    
+                    // Show course if it matches or has matching notebooks
+                    if (courseMatches || coursesWithMatchingNotebooks.has(courseId)) {
+                        course.style.display = 'block';
+                        course.classList.add('active');
+                        hasResults = true;
+                        
+                        // If course matches but doesn't have matching notebooks, show all its notebooks
+                        if (courseMatches && !coursesWithMatchingNotebooks.has(courseId)) {
+                            course.querySelectorAll('.eva-notebook').forEach(nb => {
+                                nb.style.display = 'flex';
+                            });
+                        }
+                    }
+                });
+            });
+            
+            // Open the first course with notebooks by default
+            let firstCourseWithNotebooks = null;
+            for (let i = 0; i < courses.length; i++) {
+                const courseNotebooks = courses[i].querySelectorAll('.eva-notebook');
+                if (courseNotebooks.length > 0) {
+                    firstCourseWithNotebooks = courses[i];
+                    break;
+                }
+            }
+            
+            if (firstCourseWithNotebooks) {
+                firstCourseWithNotebooks.classList.add('active');
+            } else if (courses.length > 0) {
+                // If no courses have notebooks, just open the first one
+                courses[0].classList.add('active');
+            }
+            
+            // Smooth scrolling for anchor links
+            document.querySelectorAll('a[href^="#"]').forEach(anchor => {
+                anchor.addEventListener('click', function(e) {
+                    e.preventDefault();
+                    
+                    const targetId = this.getAttribute('href');
+                    const targetElement = document.querySelector(targetId);
+                    
+                    if (targetElement) {
+                        window.scrollTo({
+                            top: targetElement.offsetTop - 100,
+                            behavior: 'smooth'
+                        });
+                    }
+                });
+            });
+        });
+    </script>"""
+
+
+def get_html_footer_closing():
+    """Generate closing HTML tags."""
+    return """
+  </div>
+  </body>
+</html>"""
+
+
+def generate_index(courses: Dict[str, Dict[str, Any]], output_dir: str) -> None:
+    """Generate the index.html file with Neon Genesis Evangelion aesthetics."""
+    print("Generating index.html")
+
+    index_path = os.path.join(output_dir, "index.html")
+    os.makedirs(output_dir, exist_ok=True)
+
+    try:
+        with open(index_path, "w", encoding="utf-8") as f:
+            # Build the page HTML from individual components
+            header = get_html_header().format(css=generate_eva_css())
+            hero = get_html_hero_section()
+            features = get_html_features_section()
+            courses_start = get_html_courses_start()
+            course_cards = generate_course_cards(courses)
+            courses_end = get_html_courses_end()
+            contribute = get_html_contribute_section()
+            footer = get_html_footer()
+            scripts = get_html_scripts()
+            closing = get_html_footer_closing()
+            
+            # Write all elements to the file
+            f.write(header)
+            f.write(hero)
+            f.write(features)
+            f.write(courses_start)
+            f.write(course_cards)
+            f.write(courses_end)
+            f.write(contribute)
+            f.write(footer)
+            f.write(scripts)
+            f.write(closing)
+            
+    except IOError as e:
+        print(f"Error generating index.html: {e}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Build marimo notebooks")
+    parser.add_argument(
+        "--output-dir", default="_site", help="Output directory for built files"
+    )
+    parser.add_argument(
+        "--course-dirs", nargs="+", default=None, 
+        help="Specific course directories to build (default: all directories with .py files)"
+    )
+    args = parser.parse_args()
+
+    # Find all course directories (directories containing .py files)
+    all_notebooks: List[str] = []
+    
+    # Directories to exclude from course detection
+    excluded_dirs = ["scripts", "env", "__pycache__", ".git", ".github", "assets"]
+    
+    if args.course_dirs:
+        course_dirs = args.course_dirs
+    else:
+        # Automatically detect course directories (any directory with .py files)
+        course_dirs = []
+        for item in os.listdir("."):
+            if (os.path.isdir(item) and 
+                not item.startswith(".") and 
+                not item.startswith("_") and
+                item not in excluded_dirs):
+                # Check if directory contains .py files
+                if list(Path(item).glob("*.py")):
+                    course_dirs.append(item)
+    
+    print(f"Found course directories: {', '.join(course_dirs)}")
+    
+    for directory in course_dirs:
+        dir_path = Path(directory)
+        if not dir_path.exists():
+            print(f"Warning: Directory not found: {dir_path}")
+            continue
+
+        notebooks = [str(path) for path in dir_path.rglob("*.py") 
+                    if not path.name.startswith("_") and "/__pycache__/" not in str(path)]
+        all_notebooks.extend(notebooks)
+
+    if not all_notebooks:
+        print("No notebooks found!")
+        return
+
+    # Export notebooks sequentially
+    successful_notebooks = []
+    for nb in all_notebooks:
+        # Determine if notebook should be exported as app or notebook
+        # For now, export all as notebooks
+        if export_html_wasm(nb, args.output_dir, as_app=False):
+            successful_notebooks.append(nb)
+
+    # Organize notebooks by course (only include successfully exported notebooks)
+    courses = organize_notebooks_by_course(successful_notebooks)
+    
+    # Generate index with organized courses
+    generate_index(courses, args.output_dir)
+    
+    # Save course data as JSON for potential use by other tools
+    courses_json_path = os.path.join(args.output_dir, "courses.json")
+    with open(courses_json_path, "w", encoding="utf-8") as f:
+        json.dump(courses, f, indent=2)
+    
+    print(f"Build complete! Site generated in {args.output_dir}")
+    print(f"Successfully exported {len(successful_notebooks)} out of {len(all_notebooks)} notebooks")
+
+
+if __name__ == "__main__":
+    main()

scripts/preview.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import argparse
+import webbrowser
+import time
+import sys
+from pathlib import Path
+
+def main():
+    parser = argparse.ArgumentParser(description="Build and preview marimo notebooks site")
+    parser.add_argument(
+        "--port", default=8000, type=int, help="Port to run the server on"
+    )
+    parser.add_argument(
+        "--no-build", action="store_true", help="Skip building the site (just serve existing files)"
+    )
+    parser.add_argument(
+        "--output-dir", default="_site", help="Output directory for built files"
+    )
+    args = parser.parse_args()
+    
+    # Store the current directory
+    original_dir = os.getcwd()
+    
+    try:
+        # Build the site if not skipped
+        if not args.no_build:
+            print("Building site...")
+            build_script = Path("scripts/build.py")
+            if not build_script.exists():
+                print(f"Error: Build script not found at {build_script}")
+                return 1
+                
+            result = subprocess.run(
+                [sys.executable, str(build_script), "--output-dir", args.output_dir], 
+                check=False
+            )
+            if result.returncode != 0:
+                print("Warning: Build process completed with errors.")
+        
+        # Check if the output directory exists
+        output_dir = Path(args.output_dir)
+        if not output_dir.exists():
+            print(f"Error: Output directory '{args.output_dir}' does not exist.")
+            return 1
+            
+        # Change to the output directory
+        os.chdir(args.output_dir)
+        
+        # Open the browser
+        url = f"http://localhost:{args.port}"
+        print(f"Opening {url} in your browser...")
+        webbrowser.open(url)
+        
+        # Start the server
+        print(f"Starting server on port {args.port}...")
+        print("Press Ctrl+C to stop the server")
+        
+        # Use the appropriate Python executable
+        subprocess.run([sys.executable, "-m", "http.server", str(args.port)])
+        
+        return 0
+    except KeyboardInterrupt:
+        print("\nServer stopped.")
+        return 0
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+    finally:
+        # Always return to the original directory
+        os.chdir(original_dir)
+
+if __name__ == "__main__":
+    sys.exit(main())