Spaces:

Tinkering
/

Pytorch-day-prez

Running

App Files Files Community

Molbap HF Staff commited on 8 days ago

Commit

d25266e

verified ·

1 Parent(s): 78bf448

Upload folder using huggingface_hub

Browse files

Files changed (14) hide show

.gitattributes +1 -0
assets/Pytorch_day_botleft.png +0 -0
assets/fastload.png +0 -0
assets/head_logo.svg +17 -0
assets/linux foundation.png +0 -0
assets/model_growth.html +0 -0
assets/normalize_time_torch.webp +0 -0
assets/py2.png +0 -0
assets/screenpage.png +0 -0
assets/screenpage2.png +0 -0
assets/torchlogo.png +0 -0
assets/visual_debugger.png +3 -0
revamped_index.html +326 -0
v0_index.html +314 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/visual_debugger.png filter=lfs diff=lfs merge=lfs -text

assets/Pytorch_day_botleft.png ADDED Viewed

assets/fastload.png ADDED Viewed

assets/head_logo.svg ADDED Viewed

assets/linux foundation.png ADDED Viewed

assets/model_growth.html ADDED Viewed

The diff for this file is too large to render. See raw diff

assets/normalize_time_torch.webp ADDED Viewed

assets/py2.png ADDED Viewed

assets/screenpage.png ADDED Viewed

assets/screenpage2.png ADDED Viewed

assets/torchlogo.png ADDED Viewed

assets/visual_debugger.png ADDED Viewed

Git LFS Details

SHA256: 87aa0537215a8e926d2d5c05f0789e8815a8ef34b7fe778cc703303abfc7da5d
Pointer size: 131 Bytes
Size of remote file: 477 kB

revamped_index.html ADDED Viewed

	@@ -0,0 +1,326 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>PyTorch × Transformers Journey</title>
+  <!-- Google Fonts -->
+  <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;800&family=Fira+Code:wght@400;600&display=swap" rel="stylesheet" />
+  <!-- Reveal.js core & dark theme base -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reset.css" />
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reveal.css" />
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/theme/black.css" id="theme" />
+  <!-- Highlight.js -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/styles/github-dark.min.css" />
+  <!-- Animations -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@4/animate.min.css" />
+  <style>
+    :root {
+      --accent-primary: #ee4c2c; /* PyTorch orange‑red */
+      --accent-secondary: #ffb347; /* lighter highlight */
+      --bg-gradient-start: #1b1b1b;
+      --bg-gradient-end: #242424;
+    }
+    html, body { font-family: 'Inter', sans-serif; }
+    .reveal .slides {
+      background: linear-gradient(135deg, var(--bg-gradient-start), var(--bg-gradient-end));
+    }
+    .reveal h1, .reveal h2, .reveal h3 { color: var(--accent-primary); font-weight: 800; letter-spacing: -0.5px; }
+    .reveal pre code { font-family: 'Fira Code', monospace; font-size: 0.75em; }
+    .reveal section img, .reveal section svg { border-radius: 1rem; box-shadow: 0 8px 22px rgba(0,0,0,0.4); }
+    .fragment.highlight-current-blue.visible { color: var(--accent-secondary) !important; }
+    /* slide-density patch */
+    .reveal h1 { font-size: 2.6rem; line-height: 1.1; }
+    .reveal h2 { font-size: 1.9rem; line-height: 1.15; }
+    .reveal h3 { font-size: 1.4rem; line-height: 1.2; }
+    .reveal p, .reveal li { font-size: 0.9rem; line-height: 1.35; }
+    .reveal pre code { font-size: 0.67em; }
+    @media (max-width: 1024px) { .reveal h1{font-size:2.2rem;} .reveal h2{font-size:1.6rem;} }
+    .reveal table td, .reveal table th { font-size: 0.85rem; padding: 4px 8px; }
+  </style>
+</head>
+<body>
+  <div class="reveal">
+    <div class="slides">
+      <!-- 1 · Opening -->
+      <section data-auto-animate>
+        <h1 class="animate__animated animate__fadeInDown">PyTorch × Transformers Journey</h1>
+        <h3 class="animate__animated animate__fadeInDown animate__delay-1s">Pythonicity, Autodiff &amp; Modularity in Modern AI</h3>
+        <p class="animate__animated animate__fadeInUp animate__delay-2s">Pablo Montalvo‑Leroux · ML Engineer @ Hugging Face</p>
+      </section>
+      <!-- 2 · 2016: Backprop & Birth Pangs -->
+      <section>
+        <h2>2016‑2018: Backprop &amp; Birth Pangs</h2>
+        <ul>
+          <li>Hand‑crafted chain‑rule; frameworks such as Theano and CNTK appeared then vanished.</li>
+          <li>MLPs → RNNs → LSTMs — until <strong>BERT</strong> detonated the field in 2018.</li>
+          <li class="fragment">Reproducibility was painful ✗ — until Transformers met PyTorch ✓.</li>
+        </ul>
+      </section>
+      <!-- 3 · Static vs Dynamic Graphs -->
+      <section>
+        <h2>Static vs Dynamic Graphs</h2>
+        <p class="fragment">Static graphs require you to compile, wait, and cross fingers the bug reproduces.</p>
+        <p class="fragment">Dynamic graphs mean you can drop <code>pdb.set_trace()</code> anywhere and continue iterating.</p>
+        <p class="fragment"><code>torch.compile</code> gives the best of both worlds: write dynamically, ship something ahead‑of‑time optimised.</p>
+      </section>
+      <!-- 4 · Dynamic Graphs Enabled Contribution -->
+      <section>
+        <h2>Dynamic Graphs Enabled Contribution</h2>
+        <ul>
+          <li>Developers debug at line‑rate — no cold‑start recompiles.</li>
+          <li>Pull‑requests remained reproducible overnight, which accelerated trust.</li>
+          <li>Static‑graph alternatives stalled and the community consolidated around PyTorch.</li>
+        </ul>
+      </section>
+      <!-- 5 · Paper Tonight → Tweak Tomorrow -->
+      <section>
+        <h2>Clone the Paper Tonight → Tweak Tomorrow</h2>
+        <p>Research cadence is measured in <strong>hours</strong>; any friction kills momentum.</p>
+        <ul>
+          <li class="fragment">2018: BERT fine‑tuning required printing tensors live rather than recompiling graphs.</li>
+          <li class="fragment">Community PRs merged overnight — credibility snowballed for both PyTorch and Transformers.</li>
+        </ul>
+      </section>
+      <!-- 6 · One Model · One File -->
+      <section>
+        <h2>“One Model · One File” — Why it Matters</h2>
+        <pre><code class="language-python" data-trim data-noescape>
+# modeling_bert.py  — single source of truth 🗄️
+class BertConfig(PretrainedConfig):
+    ...
+class BertSelfAttention(nn.Module):
+    ...
+class BertLayer(nn.Module):
+    ...
+class BertModel(PreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.embeddings = BertEmbeddings(config)
+        self.encoder = nn.ModuleList(
+            [BertLayer(config) for _ in range(config.num_hidden_layers)]
+        )
+        self.init_weights()
+        </code></pre>
+        <ul>
+          <li>All layers, forward pass, and <code>from_pretrained()</code> logic live together.</li>
+          <li>No cross‑file inheritance maze — copy to Colab, hack, and run.</li>
+          <li>Reviewers diff one file; merge time dropped from days to hours.</li>
+        </ul>
+      </section>
+      <!-- 7 · Transformers Grew With Python -->
+      <section>
+        <h2>Transformers Grew with Python</h2>
+        <ul>
+          <li>The library prioritises hackability, which in turn accelerates adoption.</li>
+          <li>Python is slow by default, so we lean on compiled CUDA kernels and Triton for raw speed.</li>
+          <li>The new <strong>Kernel Hub</strong> means Transformers automatically uses a faster op the moment it is published — no application changes required.</li>
+        </ul>
+      </section>
+      <!-- 8 · Back to Python: Mary Shelley Mode -->
+      <section>
+        <h2>Back to Python: Modular “Mary Shelley” Mode</h2>
+        <p>Compose new blocks via subclassing and selective override.</p>
+        <pre><code class="language-python" data-trim data-noescape>
+class LlamaRotaryLoRA(LlamaAttention):
+    def __init__(...):
+        super().__init__(...)
+        self.q_proj = LoRA(self.q_proj)  # swap in LoRA
+        self.apply_rotary()              # keep RoPE
+        </code></pre>
+      </section>
+      <!-- 9 · Logit Debugger -->
+      <section>
+        <h2>Logit Debugger: Trust but Verify</h2>
+        <ul>
+          <li>Attach a hook to every <code>nn.Module</code>; dump logits layer‑by‑layer.</li>
+          <li>Spot ε‑level drifts — LayerNorm precision, FP16 underflow, etc.</li>
+          <li>JSON traces are diffable in CI, so regressions stay caught.</li>
+        </ul>
+      </section>
+      <!-- 10 · DTensor & TP API -->
+      <section>
+        <h2>DTensor & Tensor‑Parallel API</h2>
+        <ul>
+          <li>Logical tensor views unlock device‑mesh sharding.</li>
+          <li>The <code>tp_plan</code> JSON keeps model code pristine and declarative.</li>
+          <li>We regularly validate 100‑billion‑parameter checkpoints inside HF test infra.</li>
+        </ul>
+        <img data-src="assets/mesh.svg" alt="Device mesh" />
+      </section>
+      <!-- 11 · Zero‑Config Parallelism -->
+      <section>
+        <h2>Zero‑Config Parallelism</h2>
+        <pre><code class="language-json" data-trim data-noescape>{
+  "layer.*.self_attn.q_proj": "colwise",
+  "layer.*.self_attn.k_proj": "colwise",
+  "layer.*.self_attn.v_proj": "colwise",
+  "layer.*.self_attn.o_proj": "rowwise"
+}</code></pre>
+        <pre><code class="language-python" data-trim data-noescape>
+def translate_to_torch_parallel_style(style: str):
+    if style == "colwise":
+        return ColwiseParallel()
+    elif style == "rowwise":
+        return RowwiseParallel()
+        </code></pre>
+        <p class="fragment">One JSON file loads a 17‑billion‑parameter Llama‑4 on 8 GPUs; tweak the plan, not the network.</p>
+      </section>
+      <!-- 12 · Cache Allocator -->
+      <section>
+        <h2>Load Faster &amp; Stronger: Cache Allocator</h2>
+        <p>Zero‑copy weight sharding shaves <strong>15 %</strong> VRAM on A100 while cutting load time below 60 s for a 100‑B model.</p>
+        <img data-src="assets/memory_bars.svg" alt="Memory bars" />
+      </section>
+      <!-- 13 · Modular Transformers: GLM Example -->
+      <section>
+        <h2>Modular Transformers: GLM by Example</h2>
+        <pre><code class="language-python" data-trim>
+class GlmMLP(Phi3MLP):
+    pass
+class GlmAttention(LlamaAttention):
+    def __init__(self, config, layer_idx=None):
+        super().__init__(config, layer_idx)
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim,
+            config.hidden_size,
+            bias=False,
+        )
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+    # Slightly different RoPE
+    ...
+class GlmForCausalLM(LlamaForCausalLM):
+    pass
+        </code></pre>
+        <p>AST magic expands this 40‑line prototype into a full modelling file, ready for training.</p>
+      </section>
+      <!-- 14 · Rise of Multimodality -->
+      <section>
+        <h2>Rise of Multimodality</h2>
+        <pre><code class="language-python" data-trim data-noescape>
+processor = AutoProcessor.from_pretrained("Qwen/Qwen3-8B")
+model = AutoModelForConditionalGeneration.from_pretrained("Qwen/Qwen3-8B")
+        </code></pre>
+        <p class="fragment">Same API across text, vision, and audio: learn once, apply everywhere.</p>
+      </section>
+      <!-- 15 · Why Python wins -->
+      <section>
+        <h2>Why Python Wins</h2>
+        <ul>
+          <li>Low entry barrier attracts newcomers and domain specialists alike.</li>
+          <li>High‑level semantics concisely express low‑level intent.</li>
+          <li>The C++/Rust back‑end remains accessible for critical paths.</li>
+        </ul>
+      </section>
+      <!-- 16 · Where Python can bite -->
+      <section>
+        <h2>Where Python can bite 🐍</h2>
+        <ul>
+          <li class="fragment">Interpreter overhead hurts microkernels (token‑by‑token decoding).</li>
+          <li class="fragment">The GIL throttles concurrent host‑side work.</li>
+          <li class="fragment">Fresh research code is easy to leave unoptimised.</li>
+        </ul>
+        <p class="fragment">Mitigations: Triton, compiled custom ops, compile‑time fallbacks, and callable kernels.</p>
+      </section>
+      <!-- 17 · Kernel Hub -->
+      <section>
+        <h2>Kernel Hub: Optimised Ops from the Community</h2>
+        <p>Kernel Hub lets any Python program <em>download and hot‑load</em> compiled CUDA/C++ kernels directly from the Hugging Face Hub at runtime.</p>
+        <ul>
+          <li><strong>Portable</strong> – kernels work from arbitrary paths outside <code>PYTHONPATH</code>.</li>
+          <li><strong>Unique</strong> – load multiple versions of the same op side‑by‑side in one process.</li>
+          <li><strong>Compatible</strong> – every kernel targets all recent PyTorch wheels (CUDA, ROCm, CPU) and C‑library ABIs.</li>
+        </ul>
+        <p class="fragment">🚀 <strong>Quick start</strong> (requires <code>torch >= 2.5</code>):</p>
+        <pre><code class="language-bash" data-trim>pip install kernels</code></pre>
+        <pre><code class="language-python" data-trim data-noescape>
+import torch
+from kernels import get_kernel
+# Download optimised kernels from the Hugging Face Hub
+activation = get_kernel("kernels-community/activation")
+x = torch.randn(10, 10, dtype=torch.float16, device="cuda")
+y = torch.empty_like(x)
+activation.gelu_fast(y, x)
+print(y)
+        </code></pre>
+        <p class="fragment">Same Transformer code — now with a <strong>3× faster</strong> GELU on A100s.</p>
+      </section>
+      <!-- 18 · API design lessons -->
+      <section>
+        <h2>API Design Lessons</h2>
+        <ul>
+          <li>Make easy things obvious, and hard things merely possible.</li>
+          <li>Keep the paper‑to‑repository delta minimal for new models.</li>
+          <li>Hide sharding mechanics; expose developer intent.</li>
+        </ul>
+        <p class="fragment">We tune radios without learning RF theory — ML frameworks should feel as frictionless.</p>
+      </section>
+      <!-- 19 · Model Growth by Modality -->
+      <section>
+        <h2>Model Growth by Modality</h2>
+        <iframe src="model_growth.html" width="100%" height="600" style="border:none;"></iframe>
+      </section>
+      <!-- 20 · Takeaways -->
+      <section>
+        <h2>Takeaways &amp; The Future</h2>
+        <ul>
+          <li>PyTorch and <code>transformers</code> have grown symbiotically for eight years—expect the spiral to continue.</li>
+          <li>Pythonicity plus pragmatism keeps the barrier to innovation low.</li>
+          <li>Open‑source models are shipping faster, larger, and more multimodal than ever.</li>
+        </ul>
+        <p><a href="https://huggingface.co/transformers/contribute" target="_blank">hf.co/transformers/contribute</a></p>
+      </section>
+    </div>
+  </div>
+  <!-- Reveal.js core -->
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reveal.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/plugin/highlight/highlight.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/plugin/notes/notes.js"></script>
+  <!-- Plotly for interactive charts -->
+  <script src="https://cdn.plot.ly/plotly-2.31.1.min.js"></script>
+  <script>
+    Reveal.initialize({
+      hash: true,
+      slideNumber: true,
+      transition: 'slide',
+      backgroundTransition: 'convex',
+      plugins: [ RevealHighlight, RevealNotes ]
+    });
+  </script>
+</body>
+</html>

v0_index.html ADDED Viewed

	@@ -0,0 +1,314 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>PyTorch × Transformers Journey</title>
+  <!-- Google Fonts -->
+  <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;800&family=Fira+Code:wght@400;600&display=swap" rel="stylesheet" />
+  <!-- Reveal.js core & dark theme base -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reset.css" />
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reveal.css" />
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/theme/black.css" id="theme" />
+  <!-- Highlight.js -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/styles/github-dark.min.css" />
+  <!-- Animations -->
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/animate.css@4/animate.min.css" />
+  <!-- Custom palette from uploaded .pptx (approximated) -->
+  <style>
+    :root {
+      /* Extracted dominant colors from PPTX master */
+      --accent-primary: #ee4c2c; /* PyTorch orange‑red */
+      --accent-secondary: #ffb347; /* lighter highlight */
+      --bg-gradient-start: #1b1b1b;
+      --bg-gradient-end: #242424;
+    }
+    html, body {
+      font-family: 'Inter', sans-serif;
+    }
+    .reveal .slides {
+      background: linear-gradient(135deg, var(--bg-gradient-start), var(--bg-gradient-end));
+    }
+    .reveal h1,
+    .reveal h2,
+    .reveal h3 {
+      color: var(--accent-primary);
+      font-weight: 800;
+      letter-spacing: -0.5px;
+    }
+    .reveal pre code {
+      font-family: 'Fira Code', monospace;
+      font-size: 0.75em;
+    }
+    .reveal section img,
+    .reveal section svg {
+      border-radius: 1rem;
+      box-shadow: 0 8px 22px rgba(0,0,0,0.4);
+    }
+    .fragment.highlight-current-blue.visible {
+      color: var(--accent-secondary) !important;
+    }
+    /* ———  slide-density patch  ——— */
+    .reveal h1        { font-size: 2.6rem; line-height: 1.1; }   /* was huge */
+    .reveal h2        { font-size: 1.9rem; line-height: 1.15; }
+    .reveal h3        { font-size: 1.4rem; line-height: 1.2;  }
+    .reveal p,
+    .reveal li        { font-size: 0.9rem; line-height: 1.35; }
+    .reveal pre code  { font-size: 0.67em; }                     /* keep code readable */
+    @media (max-width: 1024px) {                                  /* fallback for small screens */
+      .reveal h1 { font-size: 2.2rem; }
+      .reveal h2 { font-size: 1.6rem; }
+    }
+    /* tweak table cells */
+    .reveal table td,
+    .reveal table th { font-size: 0.85rem; padding: 4px 8px; }
+  </style>
+</head>
+<body>
+  <div class="reveal">
+    <div class="slides">
+      <!-- 1. Opening -->
+      <section data-auto-animate>
+        <h1 class="animate__animated animate__fadeInDown">PyTorch × Transformers Journey</h1>
+        <h3 class="animate__animated animate__fadeInDown animate__delay-1s">Pythonicity, Autodiff & Modularity in Modern AI</h3>
+        <p class="animate__animated animate__fadeInUp animate__delay-2s">Pablo Montalvo‑Leroux · ML Engineer @ Hugging Face</p>
+      </section>
+      <!-- 4. Static vs Dynamic Graphs -->
+      <section>
+        <h2>Transformers and Static vs Dynamic Graphs</h2>
+        <p class="fragment">Static graphs and transformers existence are inversely correlated</p>
+        <p class="fragment">torch.compile ≈ sweet spot: author in dynamic, ship in static.</p>
+      </section>
+      <section><h2>Clone the paper tonight → tweak tomorrow</h2><p>Research cadence ≈ <strong>hours</strong>; any friction kills momentum.</p><ul><li class="fragment">2018: BERT fine-tunes needed <em>live</em> tensor prints, not graph recompiles.</li><li class="fragment">Community PRs were merged overnight → library credibility snowballed.</li></ul></section>
+      <!-- 6. Mary Shelley mode (unchanged) -->
+      <!--I also need you to include actual wins from transformersm it is a SOURCE OF MODEL TRUTH. We have 315 model defined and more every release - it is the reference, with very simple definitions. The Mary Shelley /Frankeinstein new addition is modular. One Decision we took that was extremely useful a long time ago was to have one file == one model, without any inheritance. All the code defining a model would be self-contained, so anyone could hack it and take a part here and there and modify it. It worked: it drove up the adoption, researchers & students were able to use it. However, we're not putting python first in this design, and we changed it recently WHILE keeping the original intent. Now, we have an AST-baed toposorting algorithm that parses a modular file, say it uses the vision encoder from Qwen 2.5VL, the layernorm of a cohere model, the preprocessing of an allenai model, the attention is original... then this modular file can use all the inheritance across transformers, but it will not be run. Once the algorithm expands the modular file, it becomes a modeling file, very verbose (can be thousands of lines), but still hackable: everything that defines a model is contained in it, and everyone can use it. Everything will work, including autodiff, thanks to torch in the backend.
+      That unlocks as well something I want to talk about: researchers and practicioners can define a model thanks to modular if they have a new hypothesis on say a layernorm, it is very easy to have a pytorch file defining the model AND have it compatible with transformers almost instantly, thus reactivating the power of pythonic inheritance. Paradigms exist for a reason. -->
+      <!-- 3. One Model / One File -->
+            <section>
+              <h2>Transformers Philosophy: One Model · One File pattern</h2>
+              <pre><code data-trim data-noescape class="language-python">
+      class BertEmbeddings(nn.Module):
+          …
+      class BertModel(BertPreTrainedModel):
+          …
+              </code></pre>
+              <p class="fragment">Atomic PRs → faster reviews → community velocity.</p>
+            </section>
+      <section>
+        <h2>Back to Python: Modular Mary Shelley Mode</h2>
+        <p>Compose new blocks via subclass & override.</p>
+        <pre><code data-trim data-noescape class="language-python">
+class LlamaRotaryLoRA(LlamaAttention):
+    def __init__(…):
+        super().__init__(…)
+        self.q_proj = LoRA(self.q_proj)
+        self.apply_rotary()
+        </code></pre>
+      </section>
+      <!-- 8. DTensor & TP API -->
+      <!--(New) Slide 9 · Tensor Parallelism — Split the Math, Not the Model
+    What problem? 175 B-param giants overflow a single GPU.
+    Core idea shard each linear kernel across devices
+    Partition	How it shards	Comm op	Notes
+    Column-wise	W = [W₀‖W₁] – split on out-features	AllGather outputs	Input is broadcast; biases split
+    Row-wise	Wᵀ = [W₀; W₁] – split on in-features	AllReduce outputs	Biases replicated
+        Rule of thumb – do column-wise first, row-wise second → halves the data you move.
+        Speaker cue: “Think Lego® bricks: first slice them long-ways, then stack the halves.”
+    (New) Slide 10 · Sequence ↔ Tensor: Keeping LayerNorms Honest
+    Tensor-parallel linear + attention can’t see the whole hidden state; LayerNorm/Dropout must.
+    Sequence Parallelism bridges the gap:
+        After row-wise: ReduceScatter → distribute tokens across GPUs.
+        LayerNorm / Dropout run locally (full hidden dim, partial sequence).
+        Before next TP region: AllGather to rebuild complete activations.
+    Caveats
+        Extra comm, but still intra-node (NVLink/XeLink)
+        Works layer-by-layer; no model-wide refactor needed.
+    Speaker cue: “Think relay race—baton = activations, hand-offs cost time, so we keep them short.”
+    (New) Slide 11 · Transformers “tp_plan” — Zero-Config Sharding
+    from transformers import AutoModelForCausalLM
+    model = AutoModelForCausalLM.from_pretrained(
+            "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+            torch_dtype="bfloat16",
+            tp_plan="auto")
+    print(model._tp_plan)
+    Example plan (8 GPUs):
+    {
+      "layer.*.self_attn.q_proj": "colwise",
+      "layer.*.self_attn.k_proj": "colwise",
+      "layer.*.self_attn.v_proj": "colwise",
+      "layer.*.self_attn.o_proj": "rowwise",
+      "...": "..."
+    }
+    Wins
+        One line to load a 17 B-param model on 8 GPUs.
+        Plan stored as JSON → editable, version-controlled, readable by chip vendors.
+        Built on DTensor, DeviceMesh; future-proof for expert/moe shards.
+    Speaker cue: “The plan is metadata—the model code stays pristine. Change the JSON, not the network.”-->
+      <section>
+        <h2>DTensor & Tensor‑Parallel API</h2>
+        <ul>
+          <li>Logical tensor view · device mesh</li>
+          <li><code>tp_plan</code> keeps module code intact</li>
+          <li>100B param validation inside HF tests</li>
+        </ul>
+        <img data-src="assets/mesh.svg" alt="Device mesh" />
+      </section>
+      <!-- 9. Cache Allocator -->
+      <section>
+        <h2>Smarter Memory: Cache Allocator</h2>
+        <p>0‑copy weight partitioning · 15 % RAM cut on A100</p>
+        <img data-src="assets/memory_bars.svg" alt="Memory bars" />
+      </section>
+      <!-- 11. Multimodal Rise -->
+      <section>
+        <h2>Rise of Multimodality</h2>
+        <pre><code data-trim data-noescape class="language-python">
+            processor = AutoProcessor.from_pretrained("Qwen/Qwen3-8B")
+            model = AutoModelForConditionalGeneration.from_pretrained("Qwen/Qwen3-8B")
+        </code></pre>
+        <p class="fragment">Same API across text · vision · audio.</p>
+      </section>
+      <!-- 12. Why Python wins -->
+      <section>
+        <h2>Why Python Wins</h2>
+        <ul>
+          <li>Low entry barrier</li>
+          <li>High‑level semantics express low‑level intent</li>
+          <li>Seamless C++/Rust extension points</li>
+        </ul>
+      </section>
+      <section>
+        <h2>Where Python can bite 🐍</h2>
+        <ul>
+          <li class="fragment">Interpreter overhead on microkernels (token‑by‑token decode)</li>
+          <li class="fragment">GIL can throttle async host‑side work</li>
+          <li class="fragment">Easy to under-optimize code that is fresh out of the lab.</li>
+        </ul>
+        <p class="fragment">Mitigations: Triton, compiled custom ops, compile‑time fallback, and callable kernels!</p>
+      </section>
+      <!-- 10. Community Kernels -->
+      <section>
+        <h2>Community Kernels</h2>
+        <p> New initiative </p>
+        <p> https://huggingface.co/kernels-community </p>
+      </section>
+      <!-- 14. Lessons for API designers -->
+      <section>
+        <h2>API Design Lessons</h2>
+        <ul>
+          <li>Make easy things obvious, hard things possible</li>
+          <li>Paper‑to‑repository difference should be minimal</li>
+          <li>Hide sharding, expose intent</li>
+        </ul>
+        <p class="fragment">We want to facilitate adoption. How does a radio work? Would you know how to tune it? </p>
+          <p class="fragment">  How does a computer work? Should you know how it does to be able to navigate the web? </p>
+      </section>
+      <section>
+        <h2>Model Growth by Modality</h2>
+        <iframe src="model_growth.html" width="100%" height="600" style="border:none;"></iframe>
+      </section>
+      <!-- 16. Takeaways -->
+      <section>
+        <h2>Takeaways & The Future </h2>
+        <ul>
+          <li>PyTorch & HF:Transformers grow symbiotically</li>
+          <li>Pythonicity × pragmatism drive adoption</li>
+          <li>Open-source models are being shipped more than ever and accomplishing more than ever thanks to initiatives such as ours</li>
+        </ul>
+        <p><a href="https://huggingface.co/transformers/contribute" target="_blank">hf.co/transformers/contribute</a></p>
+      </section>
+    </div>
+  </div>
+  <!-- Reveal.js core -->
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/dist/reveal.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/plugin/highlight/highlight.js"></script>
+  <script src="https://cdn.jsdelivr.net/npm/reveal.js@5/plugin/notes/notes.js"></script>
+  <!-- Plotly for interactive charts -->
+  <script src="https://cdn.plot.ly/plotly-2.31.1.min.js"></script>
+  <script>
+    /* Initialise Reveal with extras */
+    Reveal.initialize({
+      hash: true,
+      slideNumber: true,
+      transition: 'slide',
+      backgroundTransition: 'convex',
+      plugins: [ RevealHighlight, RevealNotes ]
+    });
+    /* LOC growth plot */
+    const locGrowth = document.getElementById('loc-growth');
+    if (locGrowth) {
+      const years = ['2018', '2020', '2022', '2025'];
+      const loc = [200, 40_000, 120_000, 315_000];
+      Plotly.newPlot(locGrowth, [{
+        x: years,
+        y: loc,
+        type: 'scatter',
+        mode: 'lines+markers',
+        hovertemplate: '%{y:,} LOC in %{x}<extra></extra>'
+      }], {title: 'Lines‑of‑Code growth (log‑scale)', yaxis: {type: 'log'}}, {displayModeBar: false});
+    }
+    /* compile benchmark */
+    const compilePlot = document.getElementById('compile-plot');
+    if (compilePlot) {
+      Plotly.newPlot(compilePlot, [{
+        x: ['Baseline', 'torch.compile'],
+        y: [100, 62],
+        type: 'bar',
+        text: ['1×', '1.6×'],
+        textposition: 'auto'
+      }], {
+        title: 'Decoder LM latency ↓',
+        margin: {t: 40, l: 40, r: 40, b: 40}
+      }, {displayModeBar: false});
+    }
+  </script>
+</body>
+</html>