full test suite

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "editor.formatOnSave": true
+}

app.js

@@ -1,26 +1,27 @@
-// import { __internal_XetBlob as XetBlobUnbundled } from "@huggingface/hub-unbundled";
-import { __internal_XetBlob as XetBlobEsm } from "@huggingface/hub-esm";
-import { __internal_XetBlob as XetBlob } from "@huggingface/hub";
+import {
+  __internal_XetBlob as XetBlob,
+  __internal_sha256,
+} from "@huggingface/hub";
 
 const lfsContent = "0123456789".repeat(1_000_000);
 
 const XetBlobs = {
   // "hub-unbundled": XetBlobUnbundled,
   "hub-esm": XetBlobEsm,
-  "hub": XetBlob
-}
+  hub: XetBlob,
+};
 
 async function test(id) {
   const blob = new Blob([lfsContent]);
 
   try {
     const blob = new XetBlobs[id]({
-        repo: {
-            type: "model",
-            name: "celinah/xet-experiments",
-        },
-        hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
-        size: 5_234_139_343,
+      repo: {
+        type: "model",
+        name: "celinah/xet-experiments",
+      },
+      hash: "7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b",
+      size: 5_234_139_343,
     });
 
     const text = await blob.slice(10, 22).text();
@@ -29,8 +30,148 @@ async function test(id) {
     document.getElementById(id).textContent = err.message;
   }
 }
-window.document.addEventListener("DOMContentLoaded", () => {
-  // test("hub-unbundled");
-  test("hub-esm");
-  test("hub");
-});
+
+async function checkRange(n) {
+  // Get selected option
+  const select = document.getElementById("file");
+  const option = select.options[select.selectedIndex];
+
+  // hash is in data-xet-hash attribute
+  const xetHash = option.getAttribute("data-xet-hash");
+  const sha = option.getAttribute("data-sha");
+  const size = +option.getAttribute("data-size");
+  const path = `https://huggingface.co/celinah/xet-experiments/resolve/main/${option.value}`;
+
+  const output = document.getElementById("output");
+  output.textContent = "";
+
+  const blob = new XetBlob({
+    repo: {
+      type: "model",
+      name: "celinah/xet-experiments",
+    },
+    hash: xetHash,
+    size,
+  });
+
+  const [own, bridge] = await Promise.all([
+    blob.slice(0, n).arrayBuffer(),
+    fetch(path, { headers: { Range: `bytes=0-${n - 1}` } }).then((res) =>
+      res.arrayBuffer()
+    ),
+  ]);
+
+  output.textContent += `Own: ${own.byteLength}\nBridge: ${bridge.byteLength}\n`;
+  // Check if the first n bytes are the same
+  const array1 = new Uint8Array(own);
+  const array2 = new Uint8Array(bridge);
+
+  let different = false;
+  for (let i = 0; i < n; i++) {
+    if (array1[i] !== array2[i]) {
+      output.textContent += `Difference at ${i}\n`;
+      different = true;
+      break;
+    }
+  }
+
+  if (!different) {
+    output.textContent += "No differences\n";
+  }
+}
+
+async function download() {
+  const select = document.getElementById("file");
+  const option = select.options[select.selectedIndex];
+  const xetHash = option.getAttribute("data-xet-hash");
+  const size = +option.getAttribute("data-size");
+
+  const output = document.getElementById("output");
+  output.textContent = "";
+
+  const blob = new XetBlob({
+    repo: {
+      type: "model",
+      name: "celinah/xet-experiments",
+    },
+    hash: xetHash,
+    size,
+    listener: (progress) => {
+      output.textContent += JSON.stringify(progress) + "\n";
+      output.textContent =
+        output.textContent.split("\n").slice(-1000).join("\n") + "\n";
+    },
+  });
+
+  const arrayBuffer = await blob.arrayBuffer();
+  const blob2 = new Blob([arrayBuffer]);
+  const url = URL.createObjectURL(blob2);
+
+  const a = document.createElement("a");
+  a.href = url;
+  a.download = option.value;
+  a.click();
+}
+
+async function checkSha() {
+  const select = document.getElementById("file");
+  const option = select.options[select.selectedIndex];
+  const xetHash = option.getAttribute("data-xet-hash");
+  const sha = option.getAttribute("data-sha");
+  const size = +option.getAttribute("data-size");
+
+  const output = document.getElementById("output");
+  output.textContent = "";
+
+  const blob = new XetBlob({
+    repo: {
+      type: "model",
+      name: "celinah/xet-experiments",
+    },
+    hash: xetHash,
+    size,
+  });
+
+  const iterator = __internal_sha256(blob);
+
+  while (1) {
+    const result = await iterator.next();
+
+    if (result.done) {
+      const hash = result.value;
+      output.textContent += `SHA: ${hash}\n`;
+      output.textContent += `Expected: ${sha}\n`;
+      output.textContent += `Match: ${hash === sha}\n`;
+      break;
+    }
+
+    output.textContent = `Progress: ${result.value * 100}%\n`;
+  }
+}
+
+// On loaded, add event listeners
+document.addEventListener(
+  "DOMContentLoaded",
+  () => {
+    document.getElementById("10kB").addEventListener("click", () => {
+      checkRange(10_000);
+    });
+
+    document.getElementById("100kB").addEventListener("click", () => {
+      checkRange(100_000);
+    });
+
+    document.getElementById("1MB").addEventListener("click", () => {
+      checkRange(1_000_000);
+    });
+
+    document.getElementById("sequential").addEventListener("click", () => {
+      checkRange(100_000).then(() => checkRange(1_000_000));
+    });
+
+    document.getElementById("download").addEventListener("click", download);
+
+    document.getElementById("sha").addEventListener("click", checkSha);
+  },
+  { once: true }
+);

index.html

@@ -1,36 +1,101 @@
 <!DOCTYPE html>
 <html>
-	<head>
-		<meta charset="UTF-8" />
-		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
-		<script src="https://cdn.tailwindcss.com"></script>
-		<!-- polyfill for firefox + import maps -->
-		<script src="https://unpkg.com/[email protected]/dist/es-module-shims.js"></script>
-		<script type="importmap">
-			{
-				"imports": {
-                    "@huggingface/hub-esm": "https://esm.sh/@huggingface/[email protected].2-xet",
-					"@huggingface/hub": "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm"
-				}
-			}
-		</script>
-		<script type="module" src="./app.js"></script>
-	</head>
-	<body>
-		<div class="container mx-auto pt-8 flex flex-col gap-4">
-          
-          <h2 class="font-bold">@huggingface/[email protected]</h2>
-          
-			<p class="text-gray-400 text-sm">https://esm.sh/@huggingface/[email protected]-xet</p>
-          
-          <pre id="hub-esm" class="bg-gray-100 rounded p-3 mb-8 text-sm">
-Output will be here (loading slice of <a href="https://huggingface.co/celinah/xet-experiments/blob/main/model5GB.safetensors" class="underline">this file</a>)</pre>
-			
-			<p class="text-gray-400 text-sm">https://cdn.jsdelivr.net/npm/@huggingface/[email protected]/+esm</p>
-                    
-          <pre id="hub" class="bg-gray-100 rounded p-3 mb-8 text-sm">
-Output will be here (loading slice of <a href="https://huggingface.co/celinah/xet-experiments/blob/main/model5GB.safetensors" class="underline">this file</a>)</pre
-			>
-		</div>
-	</body>
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <script src="https://cdn.tailwindcss.com"></script>
+    <!-- polyfill for firefox + import maps -->
+    <script src="https://unpkg.com/[email protected]/dist/es-module-shims.js"></script>
+    <script type="importmap">
+      {
+        "imports": {
+          "@huggingface/hub": "https://cdn.jsdelivr.net/npm/@huggingface/hub@1.1.0-xet.1/+esm"
+        }
+      }
+    </script>
+    <script type="module" src="./app.js"></script>
+  </head>
+  <body>
+    <div class="container mx-auto pt-8 flex flex-col gap-4">
+      <h2 class="font-bold">@huggingface/[email protected]-xet</h2>
+
+      <p>
+        Select a file from
+        <a href="https://huggingface.co/celinah/xet-experiments"
+          >celinah/xet-experiments</a
+        >
+      </p>
+
+      <select id="file">
+        <option
+          value="dataset1MB.zip"
+          data-size="1400865"
+          data-sha="1e5477bbbe868845be3b852d80eec2f1da6570340860668549a6dd5da0b00054"
+          data-xet-hash="87e19088fd7d31e4a77be3f993d85031f5ecbc8f5ad58668880516209e6ed663"
+        >
+          dataset1MB.zip (1,4MB)
+        </option>
+        <option
+          value="large_text.txt"
+          data-size="62914580"
+          data-xet-hash="794efea76d8cb372bbe1385d9e51c3384555f3281e629903ecb6abeff7d54eec"
+          data-sha="c27f98578d9363b27db0bc1cbd9c692f8e6e90ae98c38cee7bc0a88829debd17"
+        >
+          large_text.txt (62.9MB)
+        </option>
+        <option
+          value="model133MB.safetensors"
+          data-size="133610664"
+          data-sha="35bb6ef696cc88b3d18b7520273dfd25315f6c47d265142ba13bba38678c92b6"
+          data-xet-hash="bdfd0a89d314f83b7fefed034f2c61133d4b49f7e9eff920dcd4c7062e64adf0"
+        >
+          model133MB.safetensors (134MB)
+        </option>
+        <option
+          value="model133MB_bis.safetensors"
+          data-size="133610664"
+          data-sha="35bb6ef696cc88b3d18b7520273dfd25315f6c47d265142ba13bba38678c92b6"
+          data-xet-hash="bdfd0a89d314f83b7fefed034f2c61133d4b49f7e9eff920dcd4c7062e64adf0"
+        >
+          model133MB.safetensors (134MB)
+        </option>
+        <option
+          value="model5GB.safetensors"
+          data-size="5234139343"
+          data-sha="b933b099f335d0cc95380200bbc3d1dc4b240f0c81e6de1a192b4921142403f8"
+          data-xet-hash="7b3b6d07673a88cf467e67c1f7edef1a8c268cbf66e9dd9b0366322d4ab56d9b"
+        >
+          model5GB.safetensors (5.2GB)
+        </option>
+        <option
+          value="rand1GB_0.safetensors"
+          data-size="1073741824"
+          data-sha="5360d7847d01cad7a0d5dd9a3f87d4fb3535ab1becbc5fa7796bb81cfb16acd7"
+          data-xet-hash="602ffd88da44b7b1cdf94a4abfdbdcf211fe36de2c71d08b5c491fabd3fd92d8"
+        >
+          rand1GB_0.safetensors (1.1GB)
+        </option>
+        <option
+          value="rand500MB"
+          data-size="524288000"
+          data-sha="8059a3147859009f1841e7df52364f20891709898ebf6150e4d1685103ab7e8f"
+          data-xet-hash="6d73bd467f35d5b9017f8a6aa358889f10c85a4e3278497382c3d1baaba4fe5a"
+        >
+          rand500MB (524MB)
+        </option>
+      </select>
+      <div class="flex gap-2 flex-wrap">
+        <button type="button" id="10kB">Check first 10 kB</button>
+        <button type="button" id="100kB">Check first 100 kB</button>
+        <button type="button" id="1MB">Check first MB</button>
+        <button type="button" id="sequential">
+          Check 100kB then 1MB in succession
+        </button>
+        <button type="button" id="sha">Check full file's sha</button>
+        <button type="button" id="download">Download full file</button>
+      </div>
+
+      <pre id="output"></pre>
+    </div>
+  </body>
 </html>