Inference that never leaves this tab.

import { ModelNano } from "./nano.js";

const engine = await ModelNano.create({
  modelUrl: "./model-nano-q4km.gguf",
  cache: "opfs",  // keep the weights in browser storage
});

const messages = [{ role: "user", content: "Explain WebGPU in one line." }];

for await (const ev of engine.generate(messages, { maxTokens: 256 })) {
  if (ev.type === "token") write(ev.text);
  if (ev.type === "done") console.log(ev.stats.tokPerSec);
}

const engine = await ModelNano.create({
  modelUrl,
  maxContext: 4096,
  onProgress({ phase, loaded, total }) {
    // phase: "download" | "upload" | "compile" | "warmup"
    bar.value = loaded / total;
  },
});

// what was loaded, and where it runs
const { quant, layers, adapter, vramBytes, loadMs } = engine.info;

const controller = new AbortController();
stopButton.onclick = () => controller.abort();

for await (const ev of engine.generate(messages, { signal: controller.signal })) {
  if (ev.type === "done") console.log(ev.finishReason);  // "abort"
}

engine.reset();  // start a new conversation

{ type: "token", text, id, thinking }
{ type: "done", finishReason: "stop" | "length" | "abort", stats }

stats: { promptTokens, cachedTokens, promptMs, ttftMs, genTokens, genMs, tokPerSec }