Spaces:

transformers-community
/

Transformers-tenets

Running

App Files Files Community

Molbap HF Staff commited on Oct 3

Commit

fc6c5e6

1 Parent(s): 3ff5fdf

update

Browse files

Files changed (42) hide show

app/dist/_astro/{index.beJ178IL.css → index.7hgRH84_.css} +0 -0
app/dist/_astro/{index.beJ178IL.css.gz → index.7hgRH84_.css.gz} +2 -2
app/dist/images/transformers/big_picture_zoomout.png +3 -0
app/dist/images/transformers/cluster_wave2vec2.png +3 -0
app/dist/images/transformers/detr_island.png +3 -0
app/dist/images/transformers/llama_center.png +3 -0
app/dist/images/transformers/llama_glm_attn.png +3 -0
app/dist/images/transformers/timeline_llava.png +3 -0
app/dist/index.html +125 -99
app/dist/index.html.gz +2 -2
app/dist/llama_center.png +3 -0
app/dist/llama_glm_attn.png +3 -0
app/public/images/transformers/big_picture_zoomout.png +3 -0
app/public/images/transformers/cluster_wave2vec2.png +3 -0
app/public/images/transformers/detr_island.png +3 -0
app/public/images/transformers/llama_center.png +3 -0
app/public/images/transformers/llama_glm_attn.png +3 -0
app/public/images/transformers/timeline_llava.png +3 -0
app/public/llama_center.png +3 -0
app/public/llama_glm_attn.png +3 -0
app/src/components/Hero.astro +0 -13
app/src/content/article.mdx +121 -59
app/src/content/embeds/banner.html +7 -0
app/src/content/embeds/transformers/tp-plan.html +22 -23
app/src/content/new_article.mdx +1 -0
app/src/styles/_base.css +4 -4
src/distill.js +0 -0
src/fragments/attention-visualizer.html +0 -45
src/fragments/d3-graph.html +0 -12
src/fragments/dependency-graph.html +0 -6
src/fragments/glm-compare.html +0 -149
src/fragments/loc-growth.html +0 -6
src/fragments/memory-profiler.html +0 -16
src/fragments/model-timeline.html +0 -6
src/fragments/model-visualisation.html +0 -0
src/fragments/terminal.html +0 -43
src/fragments/tp-plan.html +0 -24
src/fragments/warmup_demo.html +0 -398
src/index.js +0 -21
src/style.css +0 -741
src/transformers-custom.css +0 -741
webpack.config.js +4 -1

app/dist/_astro/{index.beJ178IL.css → index.7hgRH84_.css} RENAMED Viewed

The diff for this file is too large to render. See raw diff

app/dist/_astro/{index.beJ178IL.css.gz → index.7hgRH84_.css.gz} RENAMED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b295619b7b1df79e3034566f78788f08d65d11da3dd3d3c5ac113a256470d828
-size 18469

 version https://git-lfs.github.com/spec/v1
+oid sha256:b4e9905ab57ee091f7ff64f4356323970823bacad6cf723107e41b4696e3e6a9
+size 18473

app/dist/images/transformers/big_picture_zoomout.png ADDED Viewed

Git LFS Details

SHA256: 6b48173ad33c50e9b1b7f674bb21948da982db04e4a927cf0cecee45bc749297
Pointer size: 131 Bytes
Size of remote file: 218 kB

app/dist/images/transformers/cluster_wave2vec2.png ADDED Viewed

Git LFS Details

SHA256: ad2931607cfd522cbccddc8047ee7fd6ee3945a2d818fe72b6c6b08c58e062b3
Pointer size: 130 Bytes
Size of remote file: 55.4 kB

app/dist/images/transformers/detr_island.png ADDED Viewed

Git LFS Details

SHA256: 6f6daf8ce4f8e71a0a9b3c60f2a7a18aacf1812a54337cf345b9005eaa251125
Pointer size: 130 Bytes
Size of remote file: 18.5 kB

app/dist/images/transformers/llama_center.png ADDED Viewed

Git LFS Details

SHA256: 3ec2caa493f919717ece1366836e156d8d05a3bf09ef4313ea502d5130a82cb0
Pointer size: 131 Bytes
Size of remote file: 406 kB

app/dist/images/transformers/llama_glm_attn.png ADDED Viewed

Git LFS Details

SHA256: 6b2c88d5eb3d461d791e7e280f74e54d05f01babb02ea0536b50386b7b1b1b8a
Pointer size: 131 Bytes
Size of remote file: 138 kB

app/dist/images/transformers/timeline_llava.png ADDED Viewed

Git LFS Details

SHA256: 2bd0469fa24737bf309c1225005b62d7f0a9d722df0e56a18c578a1327cf94fc
Pointer size: 131 Bytes
Size of remote file: 621 kB

app/dist/index.html CHANGED Viewed

@@ -1,7 +1,7 @@
 <!DOCTYPE html><html lang="en" data-theme="light" data-toc-auto-collapse="1"> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Maintain the unmaintainable:
 1M python loc, 400+ models</title><meta name="description" content="A peek into software engineering for the transformers library"><link rel="canonical" href="http://localhost:4321/"><meta property="og:type" content="article"><meta property="og:title" content="Maintain the unmaintainable:
-1M python loc, 400+ models"><meta property="og:description" content="A peek into software engineering for the transformers library"><meta property="og:url" content="http://localhost:4321/"><meta property="og:image" content="/thumb.auto.jpg"><meta property="article:published_time" content="October 2, 2025"><meta property="article:author" content="Pablo Montalvo"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="Maintain the unmaintainable:
-1M python loc, 400+ models"><meta name="twitter:description" content="A peek into software engineering for the transformers library"><meta name="twitter:image" content="/thumb.auto.jpg"><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","headline":"Maintain the unmaintainable:\n1M python loc, 400+ models","description":"A peek into software engineering for the transformers library","datePublished":"October 2, 2025","author":[{"@type":"Person","name":"Pablo Montalvo"}],"keywords":"transformers, engineering, design-philosophy","mainEntityOfPage":"http://localhost:4321/","image":["/thumb.auto.jpg"]}</script><script>
       (() => {
         try {
           const saved = localStorage.getItem("theme");
@@ -12,8 +12,8 @@
           document.documentElement.setAttribute("data-theme", theme);
         } catch {}
       })();
-    </script><script type="module" src="/scripts/color-palettes.js"></script><!-- TO MANAGE PROPERLY --><script src="https://cdn.plot.ly/plotly-3.0.0.min.js" charset="utf-8"></script><link rel="stylesheet" href="/_astro/index.beJ178IL.css"><script type="module" src="/_astro/hoisted.DK-CdsVg.js"></script>
-<script type="module" src="/_astro/page.CH0W_C1Z.js"></script></head> <body> <button id="theme-toggle" aria-label="Toggle color theme" data-astro-cid-x3pjskd3> <svg class="icon light" width="20" height="20" viewBox="0 0 24 24" aria-hidden="true" focusable="false" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" data-astro-cid-x3pjskd3> <circle cx="12" cy="12" r="5" data-astro-cid-x3pjskd3></circle> <line x1="12" y1="1" x2="12" y2="4" data-astro-cid-x3pjskd3></line> <line x1="12" y1="20" x2="12" y2="23" data-astro-cid-x3pjskd3></line> <line x1="1" y1="12" x2="4" y2="12" data-astro-cid-x3pjskd3></line> <line x1="20" y1="12" x2="23" y2="12" data-astro-cid-x3pjskd3></line> <line x1="4.22" y1="4.22" x2="6.34" y2="6.34" data-astro-cid-x3pjskd3></line> <line x1="17.66" y1="17.66" x2="19.78" y2="19.78" data-astro-cid-x3pjskd3></line> <line x1="4.22" y1="19.78" x2="6.34" y2="17.66" data-astro-cid-x3pjskd3></line> <line x1="17.66" y1="6.34" x2="19.78" y2="4.22" data-astro-cid-x3pjskd3></line> </svg> <svg class="icon dark" width="20" height="20" viewBox="0 0 24 24" aria-hidden="true" focusable="false" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" data-astro-cid-x3pjskd3> <path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z" data-astro-cid-x3pjskd3></path> </svg>  </button>  <section class="hero" data-astro-cid-bbe6dxrz> <h1 class="hero-title" data-astro-cid-bbe6dxrz>Maintain the unmaintainable:<br/>1M python loc, 400+ models</h1> <div class="hero-banner" data-astro-cid-bbe6dxrz> <figure class="html-embed"><div class="html-embed__card is-frameless"><div id="frag-7ye2najyjjf"><style>
 @import url('https://fonts.googleapis.com/css2?family=Inter:wght@500;600&display=swap');
 .banner-container {
@@ -137,6 +137,13 @@ node.append('text')
   .attr('dy','-1.1em')
   .text(d => shortId(d.id));
 // Forces tuned for wide, short aspect
 const sim = d3.forceSimulation(graph.nodes)
   .force('link', d3.forceLink(graph.links).id(d => d.id).distance(150).strength(0.4))
@@ -160,14 +167,12 @@ function dragEnd(e,d){ if(!e.active) sim.alphaTarget(0); d.fx=d.fy=null; }
 // Fit on first paint (no zoom UI for banner)
 window.addEventListener('resize', () => location.reload());
 </script>
-</div></div></figure> <p class="hero-desc" data-astro-cid-bbe6dxrz>A peek into software engineering for the transformers library</p> </div> </section> <header class="meta" aria-label="Article meta information" data-astro-cid-bbe6dxrz> <div class="meta-container" data-astro-cid-bbe6dxrz> <div class="meta-container-cell" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Author</h3> <ul class="authors" data-astro-cid-bbe6dxrz> <li data-astro-cid-bbe6dxrz> <a href="https://huggingface.co/Molbap" data-astro-cid-bbe6dxrz>Pablo Montalvo</a>   </li> </ul> </div> <div class="meta-container-cell meta-container-cell--affiliations" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Affiliation</h3> <p data-astro-cid-bbe6dxrz> <a href="https://huggingface.co" target="_blank" rel="noopener noreferrer" data-astro-cid-bbe6dxrz> Hugging Face </a> </p> </div>  <div class="meta-container-cell meta-container-cell--published" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Published</h3> <p data-astro-cid-bbe6dxrz>October 2, 2025</p> </div> <!-- {doi && (
       <div class="meta-container-cell">
         <h3>DOI</h3>
         <p><a href={`https://doi.org/${doi}`} target="_blank" rel="noopener noreferrer">{doi}</a></p>
       </div>
-    )} --> <div class="meta-container-cell meta-container-cell--pdf" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>PDF</h3> <p data-astro-cid-bbe6dxrz> <a class="button" href="/maintain-the-unmaintainable-1m-python-loc-400-models.pdf" download="maintain-the-unmaintainable-1m-python-loc-400-models.pdf" aria-label="Download PDF maintain-the-unmaintainable-1m-python-loc-400-models.pdf" data-astro-cid-bbe6dxrz>
-Download PDF
-</a> </p> </div> </div> </header>  <section class="content-grid"> <nav class="table-of-contents" aria-label="Table of Contents" data-auto-collapse="1"> <div class="title">Table of Contents</div> <div id="article-toc-placeholder"></div> </nav> <details class="table-of-contents-mobile"> <summary>Table of Contents</summary> <div id="article-toc-mobile-placeholder"></div> </details> <script>
   // Build TOC from article headings (h2/h3/h4) and render into the sticky aside
   const buildTOC = () => {
     const holder = document.getElementById('article-toc-placeholder');
@@ -407,39 +412,43 @@ Download PDF
 <p>How do you keep such a ship afloat, made of so many moving, unrelated parts, contributed to by a buzzing hivemind? Especially as the pace of ML research accelerates? We receive constant feedback on everything from function signatures with hundreds of arguments to duplicated code and optimization concerns, and we listen to all of it, or try to. The library’s usage keeps on growing, and we are a small team of maintainers and contributors, backed by hundreds of open-source community members.
 We continue to support all new models and expect to do so for the foreseeable future.</p>
 <p>This post dissects the design philosophy that makes this possible. It’s the result of an evolution from our older principles, detailed on our previous <a href="https://huggingface.co/docs/transformers/en/philosophy">philosophy</a> page, as well as its accompanying <a href="https://huggingface.co/blog/transformers-design-philosophy">blog post from 2022</a>. More recently (and we strongly recommend the read) we published a blog post about <a href="https://huggingface.co/blog/faster-transformers">recent upgrades to transformers</a>, focusing on what makes the library faster today. All of these developments are only made possible thanks to these principles.</p>
-<p>We codify the “tenets” that guide our development, demonstrate how they are implemented in code, and show the measurable impact they have on the library’s sustainability and growth.</p>
-<p>For any OSS maintainer, power user, or contributor, this is the map to understanding, using, and building upon <code>transformers</code>, but not only: any project of comparable size will require you to make deep choices, not only on design and choice of abstraction, but on the very mindset of the software you are building.</p>
 <p><a href="#source-of-truth">Tenets exemplified</a> will have their summary available on hover.</p>
 <p><a href="https://huggingface.co/blog/welcome-openai-gpt-oss">External links</a> to articles will help you solidify your knowledge.</p>
-<p><a href="#generated-modeling">Several interactive visualisations</a> are available as you go - scroll, zoom, drag away.</p>
-<div class="crumbs"><p>Throughout this post, you’ll find breadcrumb boxes like this one. They summarize what you just learned, connect it to the tenets, and point to what’s coming <strong>Next</strong>. Think of them as narrative signposts to help you keep track.</p></div>
 <h2 id="the-core-tenets-of-transformers"><a href="#the-core-tenets-of-transformers">The core tenets of transformers</a></h2>
 <p>We summarize the foundations on which we’ve built everything, and write the “tenets” of the library.  They behave like <em>software interfaces</em>, hence it is crucial that they are explicitly written down. However opinionated they are, they have evolved over time.</p>
-<p>Note that the library <em>evolved</em> towards these principles, and that they <em>emerged</em> from decisions taken, and once emerged they were recognized as critical.</p>
-<div class="tenet-list"><ol><li class="tenet"><a id="source-of-truth"></a><strong>Source of Truth</strong><p>We aim to be a <a href="#https://huggingface.co/blog/transformers-model-definition">source of truth for all model definitions</a>. This is not a tenet, but something that still guides our decisions. Model implementations should be reliable, reproducible, and faithful to the original performances.</p><em>This overarching guideline ensures quality and reproducibility across all models in the library.</em></li><li class="tenet"><a id="one-model-one-file"></a><strong>One Model, One File</strong><p>All inference and training core logic has to be visible, top‑to‑bottom, to maximize each model’s hackability.</p><em>Every model should be completely understandable and hackable by reading a single file from top to bottom.</em></li><li class="tenet"><a id="code-is-product"></a><strong>Code is Product</strong><p>Optimize for reading, diffing, and tweaking, our users are power users. Variables can be explicit, full words, even several words, readability is primordial.</p><em>Code quality matters as much as functionality - optimize for human readers, not just computers.</em></li><li class="tenet"><a id="standardize-dont-abstract"></a><strong>Standardize, Don’t Abstract</strong><p>If it’s model behavior, keep it in the file; abstractions only for generic infra.</p><em>Model-specific logic belongs in the model file, not hidden behind abstractions.</em></li><li class="tenet"><a id="do-repeat-yourself"></a><strong>DRY* (DO Repeat Yourself)</strong><p>Copy when it helps users; keep successors in sync without centralizing behavior.</p><p><strong>Amendment:</strong> With the introduction and global adoption of <a href="#modular">modular</a> transformers, we do not repeat any logic in the modular files, but end user files remain faithful to the original tenet.</p><em>Strategic duplication can improve readability and maintainability when done thoughtfully.</em></li><li class="tenet"><a id="minimal-user-api"></a><strong>Minimal User API</strong><p>Config, model, preprocessing; from_pretrained, save_pretrained, push_to_hub. We want the least amount of codepaths. Reading should be obvious, configurations should be obvious.</p><em>Keep the public interface simple and predictable, users should know what to expect.</em></li><li class="tenet"><a id="backwards-compatibility"></a><strong>Backwards Compatibility</strong><p>Evolve by additive standardization, never break public APIs.</p><p>Any artifact that was once on the hub and loadable with transformers should be usable indefinitely with the same interface. Further, public methods should not change to avoid breaking dependencies.</p><em>Once something is public, it stays public, evolution through addition, not breaking changes.</em></li><li class="tenet"><a id="consistent-public-surface"></a><strong>Consistent Public Surface</strong><p>Same argument names, same outputs, hidden states and attentions exposed, enforced by tests. This is a goal we have as well as a tenet.</p><em>All models should feel familiar - consistent interfaces reduce cognitive load.</em></li></ol></div>
-<p>When a PR is merged, it is because the contribution is worthwhile, and that the  <code>transformers</code> team finds the design of the contribution to be aligned with what is above.</p>
-<p>Does all the code in the library follow strictly these tenets? No. The library is a gigantic house with connected nooks, corridors, crannies everywhere built by thousands of different workers. We <em>try</em> to make it so all the code added is compliant, because if we fail and merge it, we cannot change it lest we break <a href="#backwards-compatibility">backwards compatibility</a>.</p>
-<p>For instance, one function essential to the implementation of <a href="https://huggingface.co/papers/2104.09864">Rotary Positional Embeddings</a> is identical in 70 <code>modeling_&lt;file&gt;.py</code> across <code>src/transformers/models/.</code>  Why keep it? Because we want all the model logic to be <a href="#one-model-one-file">contained in the modeling file</a>. In order to do that, we <a href="#do-repeat-yourself">do repeat ourselves</a>.</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="python"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> rotate_half</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(x):</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">    &quot;&quot;&quot;Rotates half the hidden dims of the input.&quot;&quot;&quot;</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">    x1 </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> x[</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">...</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, : x.shape[</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">] </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">//</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 2</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">]</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">    x2 </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> x[</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">...</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, x.shape[</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">] </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">//</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 2</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> :]</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">    return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.cat((</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">x2, x1), </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">dim</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
 <span class="line"></span></code></pre></div>
-<p>You can use a simple regex to look at all methods of a given name across your codebase and look at their differences and similarities, that’s what I did (+ a hash to avoid quadraticity).</p>
-<p>We want all models to have self-contained modeling code.</p>
-<p>Every core functionality <em>must</em> be in the modeling code, every non-core functionality <em>can</em> be outside of it.</p>
-<p>This comes as a great cost. Enter the <code>#Copied from...</code> mechanism: for a long time, these comments were indicating that some code was copied from another model, saving time both for the reviewers and for the CI. But the LOC count kept creeping up. Each new model copied over hundreds of lines that we considered largely boilerplate, yet, we could not remove them.</p>
-<p>We needed to separate both principles that were so far intertwined, <a href="#do-repeat-yourself">repetition</a> and <a href="#one-model-one-file">hackability</a>.</p>
-<p>What was the solution to this?</p>
-<div class="crumbs"><p>Read the code in one place (<a href="#one-model-one-file">One Model, One File</a>). Keep semantics local (<a href="#standardize-dont-abstract">Standardize, Don’t Abstract</a>). Allow strategic duplication for end users (<a href="#do-repeat-yourself">DRY*</a>). Keep the public surface minimal and stable (<a href="#minimal-user-api">Minimal API</a>, <a href="#backwards-compatibility">Backwards Compatibility</a>, <a href="#consistent-public-surface">Consistent Surface</a>). <strong>Next:</strong> how modular transformers honor these while removing boilerplate.</p></div>
 <h2 id="-modular-transformers"><a href="#-modular-transformers"><a id="modular"></a> Modular transformers</a></h2>
-<p>Transformers is an opinionated library. The previous <a href="https://huggingface.co/docs/transformers/en/philosophy">philosophy</a> page, and the <a href="https://huggingface.co/blog/transformers-design-philosophy">blog post</a> were already pointing at the drawbacks mentioned just above, which have been iteratively addressed. <a href="https://huggingface.co/docs/transformers/en/modular_transformers"><code>modular</code> transformers were introduced</a>, allowing a form of inheritance without breaking <a href="#one-model-one-file">One model, One file</a>.</p>
-<p>We amended the principle of <a href="#do-repeat-yourself">DRY*</a> by removing progressively all pieces of code that were “copied from” another file.</p>
-<p>It works as follows. In order to contribute a model, say for instance  define a <code>modular_</code> file that can inherit from <em>any function across all other modeling, configuration and processor files</em>.
-This modular file can use inheritance across models: and then, it will be unravelled into a fully functional modeling file.</p>
 <summary id="generated-modeling">Auto-generated modeling code</summary>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-971efks994o"><div class="code-compare" style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin: 1.5rem 0;">
     <div class="code-column" style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
         <div class="code-header" style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
             modular_glm.py
@@ -588,10 +597,13 @@ class GlmRMSNorm(nn.Module):
     <strong>Left:</strong> Clean modular definition with inheritance.
     <strong>Right:</strong> Auto-expanded version with all inherited functionality visible.
 </p></div></div></figure>
-<p>As you can see, we can now define any model as a <em>modular</em> of another.</p>
 <p>You might think “well that’s just how inheritance works”. The crucial difference is that we do <em>visibly</em> what is essentially the <em>compiler</em>’s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it <a href="#one-model-one-file">all in one piece</a>.</p>
 <p>What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.</p>
-<p>When <code>AutoModel.from_pretrained(...)</code> is called, it is indeed the modeling (right side) that is ran, and all the tests are run on the modeling code.</p>
 <p>What does that give us?</p>
 <div class="crumbs"><p><strong>TL;DR:</strong> A small <code>modular_*.py</code> declares reuse; the expanded modeling file stays visible (<a href="#one-model-one-file">One Model, One File tenet preserved</a>). Reviewers and contributors maintain the shard, not the repetition.</p><p><strong>Next:</strong> the measurable effect on effective LOC and maintenance cost.</p></div>
 <h3 id="a-maintainable-control-surface"><a href="#a-maintainable-control-surface">A maintainable control surface</a></h3>
@@ -599,14 +611,15 @@ class GlmRMSNorm(nn.Module):
 However, if a model has a modular_<em>.py and a corresponding automatically generated modeling_</em>/.py, we only count the LOC under the modular file. The modeling code has no maintenance cost as it is strictly dependent on the modular file.</p>
 <p>That gives an “effective LOC” curve: the 𝗺𝗮𝗶𝗻𝘁𝗲𝗻𝗮𝗻𝗰𝗲 𝘀𝘂𝗿𝗳𝗮𝗰𝗲.</p>
 <p>Measured on git history, raw <code>modeling_*.py</code> grew at ~362 LOC/day before modular; counting only modular shards yields ~25 LOC/day after — about <strong>15× lower</strong>. The effective curve (blue line below) represents the <strong>maintenance surface</strong> today: what maintainers actually read and review.</p>
-<p>Less code to hand-maintain means fewer places to break. LOC is not a direct measure of complexity, but they correlate in review effort and change risk.</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-b9s6rzaeqn8"><iframe
 src="https://molbap-loc-1.hf.space"
 style="width:100%; height:900px; border:0"
 allow="clipboard-read; clipboard-write; fullscreen"
 referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
-<p>If you zoom in, you’ll notice there’s a sharp drop near the end, it’s essentially due to us <a href="https://github.com/huggingface/transformers/commit/4df2529d79d75f44e70396df5888a32ffa02d61e#diff-60849db3e9922197854ef1cac92bf4aba08b5d7fd3fe6f3c16a3511e29e0eacc">removing support for Jax and TensorFlow</a> library-wide.</p>
 <p>But this was not the only effort that allowed us to reduce maintenance load.</p>
 <p>We recently underwent a deep refactor of the attention implementation. You’ve likely heard about <a href="https://huggingface.co/docs/text-generation-inference/en/conceptual/flash_attention">flash attention</a> and its several variants.</p>
 <p>The <em>attention computation</em> itself happens at a <em>lower</em> level of abstraction than the model itself.</p>
@@ -632,41 +645,40 @@ referrerpolicy="no-referrer-when-downgrade"
 <div class="crumbs"><p>Attention semantics remain in <code>eager_attention_forward</code>; faster backends are opt-in via config. We inform via types/annotations rather than enforce rigid kwargs, preserving integrations.</p><p><strong>Next:</strong> parallel partitioning is declared as a plan, not through model surgery.</p></div>
 <h3 id="-configurable-tensor-parallelism"><a href="#-configurable-tensor-parallelism"><a id="simpler-tensor-parallelism"></a> Configurable Tensor Parallelism</a></h3>
 <p>If you’re not familiar with the different flavours of parallelism, I recommend checking out <a href="https://huggingface.co/blog/accelerate-nd-parallel">this blog post</a> first, and of course a full <a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook">dive into the ultra-scale playbook</a> is always recommended.</p>
-<p>The essential part is that, as <a href="https://huggingface.co/docs/transformers/v4.56.2/perf_train_gpu_many#tensor-parallelism">the documentation states</a> when tensors get too large to fit on a single GPU, they are sliced along a particular dimension and every slice is sent to a different GPU.</p>
 <p>Why does it matter?</p>
-<p>Because we want to avoid code modifications that are unrelated to the model.
-We choose to place the level of abstraction higher than the device placement: a matrix multiplication - a <code>nn.Linear</code> layer - should be always expressed in the same way, regardless of how it is placed.</p>
-<p>Hence, we want to touch <a href="#minimal-user-api">minimally</a> to the modeling code, and only modify it when <em>architectural changes</em> are involved. For instance, for tensor parallelism, we instead now specify a simple <code>tp_plan</code>.</p>
-<p>The alternative would be to modify parent classes specific to their</p>
-<p>It is written once in the config and passed to <code>.from_pretrained()</code>. The plan maps module name patterns to partitioning strategies. Strategies are resolved by the internal <code>ParallelInterface</code>, which wires to sharding implementations <code>ColwiseParallel</code>, <code>RowwiseParallel</code>, packed variants, and so on.</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-cnu461t3tc"><pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
-base_model_tp_plan = {
-    "layers.*.self_attn.q_proj": "colwise",
-    "layers.*.self_attn.k_proj": "colwise",
-    "layers.*.self_attn.v_proj": "colwise",
-    "layers.*.self_attn.o_proj": "rowwise",
-    "layers.*.mlp.gate_proj": "colwise",
-    "layers.*.mlp.up_proj":   "colwise",
-    "layers.*.mlp.down_proj": "rowwise",
-}
-# Runtime
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-model_id = "your/model-or-local-checkpoint"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    dtype=torch.bfloat16,
-    tp_plan=base_model_tp_plan,   # <-- plan defined above
-)
-tok = AutoTokenizer.from_pretrained(model_id)
-inputs = tok("Hello", return_tensors="pt").to(model.device)
-out = model(**inputs)</code></pre></div></div></figure>
-<p>Which allows a user to run with multiple processes per node, e.g. 4 GPUs:</p>
 <p><code>torchrun --nproc-per-node 4 demo.py</code></p>
-<p>Semantics stay in the model (a Linear stays a Linear), distribution is orthogonal and declared via strings: “colwise” splits columns of weights/bias across ranks; “rowwise” splits rows; packed variants shard fused weights; The mapping keys accept glob patterns like <code>layers.*.mlp.down_proj</code> to target repeated submodules.</p>
-<div class="crumbs"><p>Sharding is configuration (<code>tp_plan</code>), not edits to <code>Linear</code>s. Glob patterns target repeated blocks; modeling semantics stay intact. <strong>Next:</strong> per-layer attention/caching schedules declared in config, not hardcoded.</p></div>
 <h3 id="-layers-attentions-and-caches"><a href="#-layers-attentions-and-caches"><a id="layers-attentions-caches"></a> Layers, attentions and caches</a></h3>
 <p>Following the same logic, the <em>nature</em> of attention and per-layer caching should not be hardcoded. We should be able to specify in the configuration how each layer is implemented. Thus, we define a mapping like:</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="python"><code><span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">ALLOWED_LAYER_TYPES</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (</span></span>
@@ -694,11 +706,11 @@ out = model(**inputs)</code></pre></div></div></figure>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">class</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> GlmRMSNorm</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">nn</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">Module</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">):</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">    ...</span></span>
 <span class="line"></span></code></pre></div>
-<p>This also opens another contribution path: GPU specialists can contribute optimized kernels to the kernel hub, and have them usable in <code>transformers</code>. You can check on the <a href="https://huggingface.co/blog/hello-hf-kernels">kernel community blog post</a> to learn more about it!</p>
 <p>Even more resources have been added, like the formidable <a href="https://github.com/huggingface/kernel-builder">kernel builder</a> with its connected resources to <a href="https://github.com/huggingface/kernel-builder/blob/main/docs/writing-kernels.md">help you build kernels with it</a> and <a href="https://github.com/huggingface/kernel-builder/blob/main/docs/nix.md">with nix</a>.</p>
-<div class="crumbs"><p>Models define semantics; kernels define how to run them faster. Use annotations to borrow community forwards while keeping a consistent public surface. <strong>Next:</strong> what modularity looks like across the repo.</p></div>
-<h2 id="modular-developments"><a href="#modular-developments">Modular developments</a></h2>
-<p>Now, we have a form of inheritance in our codebase. Some models become standards, and model contributors are given the opportunity to <em>define standards</em>. Pushing the boundaries of scientific knowledge can translate into the boundaries of engineering if this effort is made, and we’re striving for it.
 It’s hard to conceptualize very large libraries and how their components interact with each other, regardless of your cognitive abilities for abstractions.
 So I wanted to take a look at the current <strong>state of modularity</strong> across the repository. How many models are defined using components of others?</p>
 <p>To get this graph, I used the heuristic of modular inheritance.</p>
@@ -707,31 +719,45 @@ So I wanted to take a look at the current <strong>state of modularity</strong> a
 <li>In this <code>modular</code> file, what models, configurations and processings are imported?</li>
 <li>Recurse through the model list that way.</li>
 </ol>
-<p>So what do we see? Llama is a basis for many models, and it shows.
-Radically different architectures such as mamba have spawned their own dependency subgraph.</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-3popmj9j337"><iframe
 src="https://molbap-dependencies-1.hf.space"
 style="width:100%; height:680px; border:0"
 allow="clipboard-read; clipboard-write; fullscreen"
 referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
 <p>In the case of VLMs, there’s far too many vision-based architectures that are not yet defined as modulars of other existing archs. In other words, there is no strong reference point in terms of software for vision models.
-As you can see, there is a small DETR island, a little llava pocket, and so on, but it’s not comparable to the centrality observed for llama.</p>
-<p>Another problem is, this visualization only shows <code>modular</code> models. Several models still do NOT have a modular file.</p>
-<p>How do we spot them, and how do we identify modularisable models?</p>
-<div class="crumbs"><p>Graph reading guide: nodes are models; edges are modular imports. Llama-lineage is a hub; several VLMs remain islands — engineering opportunity for shared parents. <strong>Next:</strong> timeline + similarity signals to spot candidates.</p></div>
 <h3 id="many-models-but-not-enough-yet-are-alike"><a href="#many-models-but-not-enough-yet-are-alike">Many models, but not enough yet, are alike</a></h3>
-<p>I looked into Jaccard similarity, which we use to measure set differences, to find similarities across models. I know that code is more than a set of characters stringed together. We also tried code-embedding models that ranked candidates better in practice, but for this post we stick to the deterministic Jaccard index. You can take a look at <a href="https://github.com/huggingface/transformers/pull/41289">the corresponding PR</a> for the embedding method.</p>
-<p>It is interesting, for our comparison, to look at <em>when</em> we deployed this modular logic and what was its rippling effect on the library. You can check the <a href="https://huggingface.co/spaces/Molbap/transformers-modular-refactor">larger space</a> to play around, but the gist is: adding modular allowed to connect more and more models to solid reference points.</p>
 <p>Yet, we still have a lot of gaps to fill.</p>
-<p>Zoom out below - it’s full of models. You can click on a node to see its connections better, or use the text box to search for a model.</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-n93vsx8na5e">    <iframe
   src="https://molbap-timeline-1.hf.space"
   style="width:100%; height:680px; border:0"
   allow="clipboard-read; clipboard-write; fullscreen"
   referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
-<p>If you’ve checked out llava, you’ve seen that llava_video is a red node, connected by a red edge to llava: it’s a candidate, something that we can <em>likely</em> remodularize, <a href="#backwards-compatibility">not touching the actual model</a> but being much more readable with <a href="#do-repeat-yourself">DRY*</a>.</p>
 <div class="crumbs"><p>Similarity metrics (Jaccard index or embeddings) surfaces likely parents; the timeline shows consolidation after modular landed. Red nodes/edges = candidates (e.g., <code>llava_video</code> → <code>llava</code>) for refactors that preserve behavior. <strong>Next:</strong> concrete VLM choices that avoid leaky abstractions.</p></div>
 <h3 id="vlm-improvements-avoiding-abstraction"><a href="#vlm-improvements-avoiding-abstraction">VLM improvements, avoiding abstraction</a></h3>
 <p>We don’t yet have a cookbook for common VLM patterns (image token scatter, multi‑tower encoders, cross‑attention bridges). This is one of the main improvement points where we can work.</p>
@@ -787,13 +813,14 @@ That means every decision we make to abstract something else has to be extremely
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">        return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> special_image_mask, special_video_mask</span></span>
 <span class="line"></span></code></pre></div>
 <p>But this is <em>within</em> the modeling file, not in the <code>PreTrainedModel</code> base class. It will not move away from it, because it’d break the <a href="#one-model-one-file">self-contained logic</a> of the model.</p>
-<div class="crumbs"><p>Keep VLM embedding mix in the modeling file (semantics), standardize safe helpers (e.g., placeholder masking), don’t migrate behavior to <code>PreTrainedModel</code>. <strong>Next:</strong> pipeline-level wins that came from PyTorch-first choices (fast processors).</p></div>
 <h3 id="on-image-processing-and-processors"><a href="#on-image-processing-and-processors">On image processing and processors</a></h3>
-<p>Choosing to be a <code>torch</code>-first software meant relieving a tremendous amount of support from <code>jax </code> and <code>TensorFlow</code> , and it also meant that we could be more lenient into the amount of torch-dependent utilities that we were able to add. One of these is the <em>fast processing</em> of images. Where they were before assumed to be minimal ndarrays, making stronger assumptions and enforcing <code>torch</code> and <code>torchvision</code> native inputs allowed up to speed up massively the processing time for each model.</p>
-<p>The gains in performance are immense, up to 20x speed for most models when compiled torchvision ops. Further, it allows to have the whole pipeline solely on GPU.</p>
 <p><img src="/images/transformers/fast_image_processors.png" alt="Fast Image Processors Performance"/>
 <p class="figure-legend">Thanks <a href="https://huggingface.co/yonigozlan">Yoni Gozlan</a> for the great work!</p></p>
-<div class="crumbs"><p>Torch-first lets processors assume torch/torchvision and run the whole pipeline on GPU; big per-model speedups. <strong>Next:</strong> how this lowers friction for contributors and downstream users.</p></div>
 <h2 id="reduce-barrier-to-entrycontribution"><a href="#reduce-barrier-to-entrycontribution">Reduce barrier to entry/contribution</a></h2>
 <p>This is an overall objective: there’s no <code>transformers</code> without its community.</p>
 <p>Having a framework means forcing users into it. It restrains flexibility and creativity, which are the fertile soil for new ideas to grow.</p>
@@ -803,7 +830,7 @@ That means every decision we make to abstract something else has to be extremely
 <strong>Next:</strong> power tools enabled by a consistent API.</p></div>
 <h3 id="-models-popularity"><a href="#-models-popularity"><a id="encoders-ftw"></a> Models popularity</a></h3>
 <p>Talking about dependencies, we can take a look at the number of downloads as a measure of popularity. One thing we see is the prominence of encoders, despite the apparent prevalence of decoder LLMs. The reason is that encoders are used to generate embeddings, which have multiple downstream uses. Just check out <a href="https://huggingface.co/blog/embeddinggemma">EmbeddingGemma</a> for a modern recap. Hence, it is vital to keep the encoders portion of the library viable, usable, fine-tune-able.</p>
-<div><figure class="html-embed"><div class="html-embed__card"><div id="frag-0yx46sf56oel"><html>
 <head><meta charset="utf-8" /></head>
 <body>
     <div>                        <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
@@ -4700,7 +4727,7 @@ return Plotly;
 <p>All models have the same API for attention computation, thanks to <a href="#external-attention-classes">the externalisation of attention classes</a>.</p>
 <p>This uniformity allows us to build cool tools to visualize the inner workings of the attention mechanism.</p>
 <p>One particular piece of machinery is the <code>attention mask</code>. Here you see the famous bidirectional attention pattern for the whole prefix (text + image) in PaliGemma and all Gemma2+ models, contrasting with the usual “causal-only” models.</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-xi8y9of6l1"><!-- Minimal HTML fragment: terminal-style ASCII attention masks -->
 <div style="max-width: 940px; margin: 16px 0; border:1px solid #2a2f3a; border-radius:8px; background:#0b0f19; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', monospace; color:#e5e7eb;">
     <div style="display:flex; align-items:center; gap:8px; padding:8px 10px; border-bottom:1px solid #1f2430; background:#111827; border-top-left-radius:8px; border-top-right-radius:8px;">
       <span style="width:10px; height:10px; background:#ef4444; border-radius:50%; display:inline-block;"></span>
@@ -4745,16 +4772,15 @@ return Plotly;
     </div>
   </div>
   </div></div></figure>
-<div class="crumbs"><p>Uniform attention APIs enable cross-model diagnostics (e.g., PaliGemma prefix bidirectionality vs causal).
-<strong>Next:</strong> whole-model tracing for ports and regressions.</p></div>
 <h3 id="logging-entire-model-activations"><a href="#logging-entire-model-activations">Logging entire model activations</a></h3>
-<p>Further, because it is all PyTorch (and it is even more now that we support only PyTorch), we can easily <a href="https://huggingface.co/docs/transformers/internal/model_debugging_utils">debug any model</a> when we want to add it to transformers. We now have a power-user tool for porting or adding models, that wraps a forward pass, intercepts every submodule call, and logs shapes, dtypes, and sample statistics of inputs/outputs to nested JSON.</p>
-<p>It just works with PyTorch models and is especially useful when aligning outputs with a reference implementation, aligned with our <a href="#source-of-truth">core guideline</a>.</p>
 <p><img src="/images/transformers/model_debugger.png" alt="Model debugger interface"/></p>
-<div class="crumbs"><p>Forward interception and nested JSON logging align ports to reference implementations, reinforcing “Source of Truth.” <strong>Next:</strong> CUDA warmup reduces load-time stalls without touching modeling semantics.</p></div>
 <h3 id="cooking-faster-cuda-warmups"><a href="#cooking-faster-cuda-warmups">Cooking faster CUDA warmups</a></h3>
-<p>Having a clean <em>external</em> API allows us to work on the <a href="#code-is-product">true inner workings of transformers</a>. One of the few recent additions was the <em>CUDA warmup</em> via <code>caching_allocator_warmup</code> which improved massively the loading footprint by pre-allocating GPU memory to avoid malloc bottlenecks during model loading, achieving a 7x factor for an 8B model, 6x for a 32B, you can check out <a href="https://github.com/huggingface/transformers/pull/36380">the source</a>!</p>
-<figure class="html-embed"><div class="html-embed__card"><div id="frag-d8fi5kw4x4v"><style>
 /* 1) Scope tokens to the widget */
 .warmup-demo{
   --page-bg:#ffffff;
@@ -5056,7 +5082,7 @@ return Plotly;
     }
 </script></div></div></figure>
 <p>It’s hard to overstate how much of a lifesaver that is when you’re trying to load a model as fast as possible, as it’s the narrowest bottleneck for your iteration speed.</p>
-<div class="crumbs"><p>Pre-allocating GPU memory removes malloc spikes (e.g., 7× for 8B, 6× for 32B in the referenced PR). <strong>Next:</strong> serving benefits directly from consistent interfaces and modularity.</p></div>
 <h3 id="transformers-serve-and-continuous-batching"><a href="#transformers-serve-and-continuous-batching">Transformers-serve and continuous batching</a></h3>
 <p>Having all these models readily available and sharing the same interface allowed us to implement transformers-serve, a CLI tool to expose models through a standard OpenAI http API.</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="bash"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">transformers</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> serve</span></span>
@@ -5087,9 +5113,9 @@ return Plotly;
 <strong>Next:</strong> what changes in v5 without breaking the promise of visible semantics.</p></div>
 <h2 id="what-is-coming-next"><a href="#what-is-coming-next">What is coming next</a></h2>
 <p>The next major version of <code>transformers</code> is just around the corner (and will have another blog post to its name when it comes out). When v5 is released, we aim to keep <a href="#backwards-compatibility">backwards compatibility</a> as solid as possible. The changes we make now are in service of that goal.</p>
-<p>We will lean further into a modular toolbox, not a framework. You should not be forced to rewrite modeling code. It’s better when a model can inherit from <code>PreTrainedModel</code> and opt into Tensor Parallel, <code>from_pretrained</code>, sharding, <code>push_to_hub</code>, loss plumbing, and external stacks like PEFT/TRL/SGLang/vLLM.</p> </main> </section> <footer class="footer"> <div class="footer-inner"> <section class="citation-block"> <h3>Citation</h3> <p>For attribution, cite this work as</p> <pre class="citation short">Pablo Montalvo (2025). &quot;Maintain the unmaintainable: 1M python loc, 400+ models&quot;.</pre> <p>BibTeX citation</p> <pre class="citation long">@misc{montalvo2025_maintain_the_unmaintaina,
   title={Maintain the unmaintainable: 1M python loc, 400+ models},
-  author={Pablo Montalvo},
   year={2025},
 }</pre> </section>   <section class="references-block">  </section> </div> </footer> <script>

 <!DOCTYPE html><html lang="en" data-theme="light" data-toc-auto-collapse="1"> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Maintain the unmaintainable:
 1M python loc, 400+ models</title><meta name="description" content="A peek into software engineering for the transformers library"><link rel="canonical" href="http://localhost:4321/"><meta property="og:type" content="article"><meta property="og:title" content="Maintain the unmaintainable:
+1M python loc, 400+ models"><meta property="og:description" content="A peek into software engineering for the transformers library"><meta property="og:url" content="http://localhost:4321/"><meta property="og:image" content="/thumb.auto.jpg"><meta property="article:published_time" content="October 2, 2025"><meta property="article:author" content="Pablo Montalvo"><meta property="article:author" content="Lysandre Debut"><meta property="article:author" content="Pedro Cuenca"><meta property="article:author" content="Yoni Gozlan"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="Maintain the unmaintainable:
+1M python loc, 400+ models"><meta name="twitter:description" content="A peek into software engineering for the transformers library"><meta name="twitter:image" content="/thumb.auto.jpg"><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","headline":"Maintain the unmaintainable:\n1M python loc, 400+ models","description":"A peek into software engineering for the transformers library","datePublished":"October 2, 2025","author":[{"@type":"Person","name":"Pablo Montalvo"},{"@type":"Person","name":"Lysandre Debut"},{"@type":"Person","name":"Pedro Cuenca"},{"@type":"Person","name":"Yoni Gozlan"}],"keywords":"transformers, engineering, design-philosophy","mainEntityOfPage":"http://localhost:4321/","image":["/thumb.auto.jpg"]}</script><script>
       (() => {
         try {
           const saved = localStorage.getItem("theme");
           document.documentElement.setAttribute("data-theme", theme);
         } catch {}
       })();
+    </script><script type="module" src="/scripts/color-palettes.js"></script><!-- TO MANAGE PROPERLY --><script src="https://cdn.plot.ly/plotly-3.0.0.min.js" charset="utf-8"></script><link rel="stylesheet" href="/_astro/index.7hgRH84_.css"><script type="module" src="/_astro/hoisted.DK-CdsVg.js"></script>
+<script type="module" src="/_astro/page.CH0W_C1Z.js"></script></head> <body> <button id="theme-toggle" aria-label="Toggle color theme" data-astro-cid-x3pjskd3> <svg class="icon light" width="20" height="20" viewBox="0 0 24 24" aria-hidden="true" focusable="false" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" data-astro-cid-x3pjskd3> <circle cx="12" cy="12" r="5" data-astro-cid-x3pjskd3></circle> <line x1="12" y1="1" x2="12" y2="4" data-astro-cid-x3pjskd3></line> <line x1="12" y1="20" x2="12" y2="23" data-astro-cid-x3pjskd3></line> <line x1="1" y1="12" x2="4" y2="12" data-astro-cid-x3pjskd3></line> <line x1="20" y1="12" x2="23" y2="12" data-astro-cid-x3pjskd3></line> <line x1="4.22" y1="4.22" x2="6.34" y2="6.34" data-astro-cid-x3pjskd3></line> <line x1="17.66" y1="17.66" x2="19.78" y2="19.78" data-astro-cid-x3pjskd3></line> <line x1="4.22" y1="19.78" x2="6.34" y2="17.66" data-astro-cid-x3pjskd3></line> <line x1="17.66" y1="6.34" x2="19.78" y2="4.22" data-astro-cid-x3pjskd3></line> </svg> <svg class="icon dark" width="20" height="20" viewBox="0 0 24 24" aria-hidden="true" focusable="false" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" data-astro-cid-x3pjskd3> <path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z" data-astro-cid-x3pjskd3></path> </svg>  </button>  <section class="hero" data-astro-cid-bbe6dxrz> <h1 class="hero-title" data-astro-cid-bbe6dxrz>Maintain the unmaintainable:<br/>1M python loc, 400+ models</h1> <div class="hero-banner" data-astro-cid-bbe6dxrz> <figure class="html-embed"><div class="html-embed__card is-frameless"><div id="frag-xl9xb41cgvg"><style>
 @import url('https://fonts.googleapis.com/css2?family=Inter:wght@500;600&display=swap');
 .banner-container {
   .attr('dy','-1.1em')
   .text(d => shortId(d.id));
+// Pin the llama node at center
+const llamaNode = graph.nodes.find(d => d.id === 'llama');
+if (llamaNode) {
+  llamaNode.fx = W / 2;
+  llamaNode.fy = H / 2;
+}
 // Forces tuned for wide, short aspect
 const sim = d3.forceSimulation(graph.nodes)
   .force('link', d3.forceLink(graph.links).id(d => d.id).distance(150).strength(0.4))
 // Fit on first paint (no zoom UI for banner)
 window.addEventListener('resize', () => location.reload());
 </script>
+</div></div></figure> <p class="hero-desc" data-astro-cid-bbe6dxrz>A peek into software engineering for the transformers library</p> </div> </section> <header class="meta" aria-label="Article meta information" data-astro-cid-bbe6dxrz> <div class="meta-container" data-astro-cid-bbe6dxrz> <div class="meta-container-cell" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Authors</h3> <ul class="authors" data-astro-cid-bbe6dxrz> <li data-astro-cid-bbe6dxrz> <a href="https://huggingface.co/Molbap" data-astro-cid-bbe6dxrz>Pablo Montalvo</a>  ,  </li><li data-astro-cid-bbe6dxrz> <a href="https://huggingface.co/Lysandre" data-astro-cid-bbe6dxrz>Lysandre Debut</a>  ,  </li><li data-astro-cid-bbe6dxrz> <a href="https://huggingface.co/pcuenq" data-astro-cid-bbe6dxrz>Pedro Cuenca</a>  ,  </li><li data-astro-cid-bbe6dxrz> <a href="https://huggingface.co/yonigozlan" data-astro-cid-bbe6dxrz>Yoni Gozlan</a>   </li> </ul> </div> <div class="meta-container-cell meta-container-cell--affiliations" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Affiliation</h3> <p data-astro-cid-bbe6dxrz> <a href="https://huggingface.co" target="_blank" rel="noopener noreferrer" data-astro-cid-bbe6dxrz> Hugging Face </a> </p> </div>  <div class="meta-container-cell meta-container-cell--published" data-astro-cid-bbe6dxrz> <h3 data-astro-cid-bbe6dxrz>Published</h3> <p data-astro-cid-bbe6dxrz>October 2, 2025</p> </div> <!-- {doi && (
       <div class="meta-container-cell">
         <h3>DOI</h3>
         <p><a href={`https://doi.org/${doi}`} target="_blank" rel="noopener noreferrer">{doi}</a></p>
       </div>
+    )} --> </div> </header>  <section class="content-grid"> <nav class="table-of-contents" aria-label="Table of Contents" data-auto-collapse="1"> <div class="title">Table of Contents</div> <div id="article-toc-placeholder"></div> </nav> <details class="table-of-contents-mobile"> <summary>Table of Contents</summary> <div id="article-toc-mobile-placeholder"></div> </details> <script>
   // Build TOC from article headings (h2/h3/h4) and render into the sticky aside
   const buildTOC = () => {
     const holder = document.getElementById('article-toc-placeholder');
 <p>How do you keep such a ship afloat, made of so many moving, unrelated parts, contributed to by a buzzing hivemind? Especially as the pace of ML research accelerates? We receive constant feedback on everything from function signatures with hundreds of arguments to duplicated code and optimization concerns, and we listen to all of it, or try to. The library’s usage keeps on growing, and we are a small team of maintainers and contributors, backed by hundreds of open-source community members.
 We continue to support all new models and expect to do so for the foreseeable future.</p>
 <p>This post dissects the design philosophy that makes this possible. It’s the result of an evolution from our older principles, detailed on our previous <a href="https://huggingface.co/docs/transformers/en/philosophy">philosophy</a> page, as well as its accompanying <a href="https://huggingface.co/blog/transformers-design-philosophy">blog post from 2022</a>. More recently (and we strongly recommend the read) we published a blog post about <a href="https://huggingface.co/blog/faster-transformers">recent upgrades to transformers</a>, focusing on what makes the library faster today. All of these developments are only made possible thanks to these principles.</p>
+<p>We formalize and articulate the “tenets” that have been guiding our development, demonstrate how they are implemented in code, and show the measurable impact they have on the library’s sustainability and growth.</p>
+<p>For any OSS maintainer, power user, or contributor, this is the map to understanding, using, and building upon <code>transformers</code>, but not only: any project of comparable size will require you to make deep choices, not only on design and choice of abstraction, but on the very mindset of the software you are building. These tenets may or may not be applicable to your project, but they provide a glimpse on how we work that could be helpful or inspirational.</p>
+<p>Conventions used throughout this post:</p>
 <p><a href="#source-of-truth">Tenets exemplified</a> will have their summary available on hover.</p>
 <p><a href="https://huggingface.co/blog/welcome-openai-gpt-oss">External links</a> to articles will help you solidify your knowledge.</p>
+<p><a href="#generated-modeling">Several interactive visualisations</a> are available as you go - scroll, zoom, drag away to explore them.</p>
+<div class="crumbs"><ul>
+<li>Breadcrumb boxes summarize what you just learned, connect it to the tenets, and point to what’s coming <strong>Next</strong>. Think of them as narrative signposts to help you keep track.</li>
+</ul></div>
+<p>We will get started by enumerating the tenets. Then we’ll look at concrete examples that show how they shape our decision-making. These examples are necessarily detailed, and sometimes complex, because they illustrate the challenges to maintain and grow a large codebase that caters to multiple collectives, has millions of users, hundreds of contributors, and always strives for simplicity and consistency.</p>
 <h2 id="the-core-tenets-of-transformers"><a href="#the-core-tenets-of-transformers">The core tenets of transformers</a></h2>
 <p>We summarize the foundations on which we’ve built everything, and write the “tenets” of the library.  They behave like <em>software interfaces</em>, hence it is crucial that they are explicitly written down. However opinionated they are, they have evolved over time.</p>
+<p>These principles were not decided in a vacuum. The library <em>evolved</em> towards them, and once they <em>emerged</em>, they were recognized as critical.</p>
+<div class="tenet-list"><ol><li class="tenet"><a id="source-of-truth"></a><strong>Source of Truth</strong><p>We aim to be a <a href="https://huggingface.co/blog/transformers-model-definition">source of truth for all model definitions</a>. This is more of a goal than a tenet, but it strongly guides our decisions. Model implementations should be reliable, reproducible, and faithful to the original implementations. If we are successful, they should become reference baselines for the ecosystem, so they’ll be easily adopted by downstream libraries and projects. It’s much easier for a project to <em>always</em> refer to the transformers implementation, than to learn a different research codebase every time a new architecture is released.</p><em>This overarching guideline ensures quality and reproducibility across all models in the library, and aspires to make the community work easier.</em></li><li class="tenet"><a id="one-model-one-file"></a><strong>One Model, One File</strong><p>All inference and training core logic has to be visible, top‑to‑bottom, to maximize each model’s hackability.</p><em>Every model should be completely understandable and hackable by reading a single file from top to bottom.</em></li><li class="tenet"><a id="code-is-product"></a><strong>Code is Product</strong><p>Optimize for reading, diffing, and tweaking, our users are power users. Variables can be explicit, full words, even several words, readability is primordial.</p><em>Code quality matters as much as functionality - optimize for human readers, not just computers.</em></li><li class="tenet"><a id="standardize-dont-abstract"></a><strong>Standardize, Don’t Abstract</strong><p>If it’s model behavior, keep it in the file; abstractions only for generic infra.</p><em>Model-specific logic belongs in the model file, not hidden behind abstractions.</em></li><li class="tenet"><a id="do-repeat-yourself"></a><strong>DRY* (DO Repeat Yourself)</strong><p>Copy when it helps users; keep successors in sync without centralizing behavior.</p><p><strong>Amendment:</strong> With the introduction and global adoption of <a href="#modular">modular</a> transformers, we do not repeat any logic in the modular files, but end user files remain faithful to the original tenet.</p><em>Strategic duplication can improve readability and maintainability when done thoughtfully.</em></li><li class="tenet"><a id="minimal-user-api"></a><strong>Minimal User API</strong><p>Config, model, preprocessing; from_pretrained, save_pretrained, push_to_hub. We want the least amount of codepaths. Reading should be obvious, configurations should be obvious.</p><em>Keep the public interface simple and predictable, users should know what to expect.</em></li><li class="tenet"><a id="backwards-compatibility"></a><strong>Backwards Compatibility</strong><p>Evolve by additive standardization, never break public APIs.</p><p>Any artifact that was once on the hub and loadable with transformers should be usable indefinitely with the same interface. Further, public methods should not change to avoid breaking dependencies. If we do deprecate something, it’s with very long cycles beforehand.</p><em>Once something is public, it stays public, evolution through addition, not breaking changes.</em></li><li class="tenet"><a id="consistent-public-surface"></a><strong>Consistent Public Surface</strong><p>Same argument names, same outputs, hidden states and attentions exposed, enforced by tests. This is a goal as well as a tenet.</p><em>All models should feel familiar - consistent interfaces reduce cognitive load.</em></li></ol></div>
+<p>When a PR is merged, it is because the contribution is worthwhile, and because the <code>transformers</code> team finds the design of the contribution to be aligned with the tenets.</p>
+<p>Does all the code in the library strictly follow these tenets? No. The library is a gigantic house with connected nooks, corridors, crannies everywhere, built by thousands of different workers. We <em>try</em> to make it so all the code added is compliant, because if we fail and merge it, we cannot change it lest we break <a href="#backwards-compatibility">backwards compatibility</a>.</p>
+<p>To see what constitutes adherence to the tenets, let’s take the example of code repetition.</p>
+<p>The following function, essential to the implementation of <a href="https://huggingface.co/papers/2104.09864">Rotary Positional Embeddings</a> can be found in more than 70 <code>modeling_&lt;file&gt;.py</code> across <code>src/transformers/models/.</code>  Why keep it? Because we want all the model logic to be <a href="#one-model-one-file">contained in the modeling file</a>. In order to do that, we <a href="#do-repeat-yourself">do repeat ourselves</a>.</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="python"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> rotate_half</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(x):</span></span>
 <span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF">    &quot;&quot;&quot;Rotates half the hidden dims of the input.&quot;&quot;&quot;</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">    x1 </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> x[</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">...</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, : x.shape[</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">] </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">//</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 2</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">]</span></span>
 <span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">    x2 </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> x[</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">...</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">, x.shape[</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">] </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">//</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF"> 2</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> :]</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">    return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> torch.cat((</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">-</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">x2, x1), </span><span style="--shiki-light:#E36209;--shiki-dark:#FFAB70">dim</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">=-</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">1</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">)</span></span>
 <span class="line"></span></code></pre></div>
+<p>We want all models to have self-contained modeling code. Every core functionality <em>must</em> be in the modeling code, every non-core functionality <em>can</em> be outside of it.</p>
+<p>This comes at a great cost. For years, we have used what we call the <code>#Copied from...</code> mechanism: we added comments of a specific format documenting that some code was copied from another model, saving time both for the reviewers and for the CI: we had tooling to ensure that the copied blocks remained in sync.</p>
+<p>But the LOC count kept creeping up. Each new model copied over hundreds of lines that we considered largely boilerplate, yet, we could not remove them.</p>
+<p>We needed to separate two principles that were so far intertwined, <a href="#do-repeat-yourself">repetition</a> and <a href="#one-model-one-file">hackability</a>.</p>
+<p>What was the solution to this? Let’s talk about modular transformers.</p>
+<div class="crumbs"><p><strong>TL;DR:</strong> Read the code in one place (<a href="#one-model-one-file">One Model, One File</a>). Keep semantics local (<a href="#standardize-dont-abstract">Standardize, Don’t Abstract</a>). Allow strategic duplication for end users (<a href="#do-repeat-yourself">DRY*</a>). Keep the public surface minimal and stable (<a href="#minimal-user-api">Minimal API</a>, <a href="#backwards-compatibility">Backwards Compatibility</a>, <a href="#consistent-public-surface">Consistent Surface</a>).</p><p><strong>Next:</strong> how modular transformers honor these while removing boilerplate.</p></div>
 <h2 id="-modular-transformers"><a href="#-modular-transformers"><a id="modular"></a> Modular transformers</a></h2>
+<p>Transformers is an opinionated library. The previous <a href="https://huggingface.co/docs/transformers/en/philosophy">philosophy</a> page, and the <a href="https://huggingface.co/blog/transformers-design-philosophy">blog post</a> were already pointing at the drawbacks mentioned just above, which have been iteratively addressed. <a href="https://huggingface.co/docs/transformers/en/modular_transformers"><code>modular</code> transformers was introduced</a> to allow a form of inheritance without breaking <a href="#one-model-one-file">One model, One file</a>.</p>
+<p>We amended the principle of <a href="#do-repeat-yourself">DRY*</a> by progressively removing all pieces of code that were “copied from” another file.</p>
+<p>It works as follows. In order to contribute a model, let us take GLM for instance, we define a <code>modular_</code> file that can inherit from <em>any function across all other modeling, configuration and processor files</em> already existing in the libary.
+The modular file can use inheritance across models: and then, it will be unravelled into a fully functional modeling file.</p>
 <summary id="generated-modeling">Auto-generated modeling code</summary>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-md3mnd0y02"><div class="code-compare" style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin: 1.5rem 0;">
     <div class="code-column" style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
         <div class="code-header" style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
             modular_glm.py
     <strong>Left:</strong> Clean modular definition with inheritance.
     <strong>Right:</strong> Auto-expanded version with all inherited functionality visible.
 </p></div></div></figure>
+<p>As you can see, we can define a new model as a <em>modular</em> combination of fragments taken from others.</p>
 <p>You might think “well that’s just how inheritance works”. The crucial difference is that we do <em>visibly</em> what is essentially the <em>compiler</em>’s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it <a href="#one-model-one-file">all in one piece</a>.</p>
+<p>You can see below the difference between <code>GlmAttention</code> and <code>LlamaAttention</code>, with the latter having been copied with minimal changes.</p>
+<p><img src="/images/transformers/llama_glm_attn.png" alt="Llama vs GLM"/></p>
 <p>What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.</p>
+<p>When <code>AutoModel.from_pretrained(...)</code> is called, it is indeed the modeling (right side) that is ran, and all the tests run on the modeling code.</p>
+<p>More importantly, the auto-generated modeling file is what users <em>read</em> to understand the code, what they step through in their debuggers and what they hack for their needs.</p>
 <p>What does that give us?</p>
 <div class="crumbs"><p><strong>TL;DR:</strong> A small <code>modular_*.py</code> declares reuse; the expanded modeling file stays visible (<a href="#one-model-one-file">One Model, One File tenet preserved</a>). Reviewers and contributors maintain the shard, not the repetition.</p><p><strong>Next:</strong> the measurable effect on effective LOC and maintenance cost.</p></div>
 <h3 id="a-maintainable-control-surface"><a href="#a-maintainable-control-surface">A maintainable control surface</a></h3>
 However, if a model has a modular_<em>.py and a corresponding automatically generated modeling_</em>/.py, we only count the LOC under the modular file. The modeling code has no maintenance cost as it is strictly dependent on the modular file.</p>
 <p>That gives an “effective LOC” curve: the 𝗺𝗮𝗶𝗻𝘁𝗲𝗻𝗮𝗻𝗰𝗲 𝘀𝘂𝗿𝗳𝗮𝗰𝗲.</p>
 <p>Measured on git history, raw <code>modeling_*.py</code> grew at ~362 LOC/day before modular; counting only modular shards yields ~25 LOC/day after — about <strong>15× lower</strong>. The effective curve (blue line below) represents the <strong>maintenance surface</strong> today: what maintainers actually read and review.</p>
+<p>Less code to hand-maintain means fewer places to break. Naturally LOC is not a direct measure of complexity, but they correlate in review effort and change risk.</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-xn06vuxqbr"><iframe
 src="https://molbap-loc-1.hf.space"
 style="width:100%; height:900px; border:0"
 allow="clipboard-read; clipboard-write; fullscreen"
 referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
+<p>The blue line (effective) is the sum of the red + green, whereas the yellow would have been the progression without modular. We can see that the maintenance surface is essentially constant (in LOC) since the implementation of <code>modular</code>.
+If you zoom in, you’ll notice there’s a sharp drop near the end, it’s essentially due to us <a href="https://github.com/huggingface/transformers/commit/4df2529d79d75f44e70396df5888a32ffa02d61e#diff-60849db3e9922197854ef1cac92bf4aba08b5d7fd3fe6f3c16a3511e29e0eacc">removing support for Jax and TensorFlow</a> library-wide.</p>
 <p>But this was not the only effort that allowed us to reduce maintenance load.</p>
 <p>We recently underwent a deep refactor of the attention implementation. You’ve likely heard about <a href="https://huggingface.co/docs/text-generation-inference/en/conceptual/flash_attention">flash attention</a> and its several variants.</p>
 <p>The <em>attention computation</em> itself happens at a <em>lower</em> level of abstraction than the model itself.</p>
 <div class="crumbs"><p>Attention semantics remain in <code>eager_attention_forward</code>; faster backends are opt-in via config. We inform via types/annotations rather than enforce rigid kwargs, preserving integrations.</p><p><strong>Next:</strong> parallel partitioning is declared as a plan, not through model surgery.</p></div>
 <h3 id="-configurable-tensor-parallelism"><a href="#-configurable-tensor-parallelism"><a id="simpler-tensor-parallelism"></a> Configurable Tensor Parallelism</a></h3>
 <p>If you’re not familiar with the different flavours of parallelism, I recommend checking out <a href="https://huggingface.co/blog/accelerate-nd-parallel">this blog post</a> first, and of course a full <a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook">dive into the ultra-scale playbook</a> is always recommended.</p>
+<p>The essential part is that, as <a href="https://huggingface.co/docs/transformers/v4.56.2/perf_train_gpu_many#tensor-parallelism">the documentation states</a>, when tensors get too large to fit on a single GPU, they are sliced along a particular dimension and every slice is sent to a different GPU.</p>
 <p>Why does it matter?</p>
+<p>Because we want to avoid code modifications that are unrelated to the model.</p>
+<p>We choose to place the level of abstraction higher than the device placement: a matrix multiplication - a <code>nn.Linear</code> layer - should be always expressed in the same way, regardless of how it is placed.</p>
+<p>Hence, we want to touch the modeling code <a href="#minimal-user-api">minimally</a>, and only modify it when <em>architectural changes</em> are involved – not depending on the way you run it. For tensor parallelism, we simply specify a <code>tp_plan</code>:</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-8bjtx56ll89"><pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj":   "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    # Runtime
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    model_id = "your/model-or-local-checkpoint"
+    model = AutoModelForCausalLM.from_pretrained( # <-- will automatically map to the plan defined above
+        model_id,
+        dtype=torch.bfloat16,
+    )
+    tok = AutoTokenizer.from_pretrained(model_id)
+    inputs = tok("Hello", return_tensors="pt").to(model.device)
+    out = model(**inputs)</code></pre></div></div></figure>
+<p>The plan is written once, saved as part of the config and passed to <code>.from_pretrained()</code>. It maps module name patterns to partitioning strategies. Strategies are resolved by the internal <code>ParallelInterface</code>, which wires to sharding implementations <code>ColwiseParallel</code>, <code>RowwiseParallel</code>, packed variants, and so on.</p>
+<p>The alternative would be to modify classes depending on supported types of parallelism.</p>
+<p>The <code>tp_plan</code> solution allows users to run the same model on a single GPU, or distribute it using multiple processes per node, e.g. 4 GPUs:</p>
 <p><code>torchrun --nproc-per-node 4 demo.py</code></p>
+<p>Semantics stay in the model (a Linear stays a Linear), parallelization is orthogonal and declared via strings: “colwise” splits columns of weights/bias across ranks; “rowwise” splits rows; packed variants shard fused weights; The mapping keys accept glob patterns like <code>layers.*.mlp.down_proj</code> to target repeated submodules.</p>
+<div class="crumbs"><p>Parallelization is specified in the configuration (<code>tp_plan</code>), not through edits to <code>Linear</code>s. Glob patterns target repeated blocks; modeling semantics stay intact.</p><p><strong>Next:</strong> per-layer attention/caching schedules declared in config, not hardcoded.</p></div>
 <h3 id="-layers-attentions-and-caches"><a href="#-layers-attentions-and-caches"><a id="layers-attentions-caches"></a> Layers, attentions and caches</a></h3>
 <p>Following the same logic, the <em>nature</em> of attention and per-layer caching should not be hardcoded. We should be able to specify in the configuration how each layer is implemented. Thus, we define a mapping like:</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="python"><code><span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">ALLOWED_LAYER_TYPES</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583"> =</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> (</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">class</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0"> GlmRMSNorm</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">(</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">nn</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">.</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">Module</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8">):</span></span>
 <span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF">    ...</span></span>
 <span class="line"></span></code></pre></div>
+<p>This also opens another contribution path: GPU specialists can contribute optimized kernels to the <a href="https://huggingface.co/kernels-community">Kernels Hub</a>, and have them immediately available to use in <code>transformers</code> and other libraries. You can check the <a href="https://huggingface.co/blog/hello-hf-kernels">kernel community blog post</a> to learn more about it!</p>
 <p>Even more resources have been added, like the formidable <a href="https://github.com/huggingface/kernel-builder">kernel builder</a> with its connected resources to <a href="https://github.com/huggingface/kernel-builder/blob/main/docs/writing-kernels.md">help you build kernels with it</a> and <a href="https://github.com/huggingface/kernel-builder/blob/main/docs/nix.md">with nix</a>.</p>
+<div class="crumbs"><p>Models define semantics; kernels define how to run them faster. Use decorations to borrow community forwards while keeping a consistent public surface.</p><p><strong>Next:</strong> what modularity looks like across the repo.</p></div>
+<h2 id="a-modular-state"><a href="#a-modular-state">A Modular State</a></h2>
+<p>With <code>modular</code> transformers, we have a form of inheritance in our codebase. Some models become standards, and model contributors are given the opportunity to <em>define standards</em>. Pushing the boundaries of scientific knowledge can translate into the boundaries of engineering if this effort is made, and we’re striving for it.
 It’s hard to conceptualize very large libraries and how their components interact with each other, regardless of your cognitive abilities for abstractions.
 So I wanted to take a look at the current <strong>state of modularity</strong> across the repository. How many models are defined using components of others?</p>
 <p>To get this graph, I used the heuristic of modular inheritance.</p>
 <li>In this <code>modular</code> file, what models, configurations and processings are imported?</li>
 <li>Recurse through the model list that way.</li>
 </ol>
+<p>So what do we see?</p>
+<p>(Graph reading guide: nodes are models; edges are modular imports).</p>
+<p>Check out the <a href="https://huggingface.co/spaces/Molbap/transformers-modular-refactor">full viewer here</a> (tab “dependency graph”, hit “build graph”) for better manipulation and exploration.</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-kf45zromzyi"><iframe
 src="https://molbap-dependencies-1.hf.space"
 style="width:100%; height:680px; border:0"
 allow="clipboard-read; clipboard-write; fullscreen"
 referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
+<p>Le’ts walk through some sections of this graph together.</p>
+<p>Llama is a basis and an influence for many models, and it shows.</p>
+<p><img src="/images/transformers/llama_center.png" alt="Llama in the center"/></p>
+<p>Radically different architectures such as mamba have spawned their own dependency subgraph.</p>
+<p>Audio models form sparser archipelagos, see for instance wav2vec2 which is a significant basis.</p>
+<p><img src="/images/transformers/cluster_wave2vec2.png" alt="Wav2vec2 influence"/></p>
 <p>In the case of VLMs, there’s far too many vision-based architectures that are not yet defined as modulars of other existing archs. In other words, there is no strong reference point in terms of software for vision models.
+)</p>
+<p>As you can see, there is a small DETR island:
+<img src="/images/transformers/detr_island.png" alt="DETR archipelago"/></p>
+<p>There is also a little llava pocket, and so on, but it’s not comparable to the centrality observed for llama.</p>
+<p>Another problem is, this visualization only shows <code>modular</code> models. Several models still do NOT have a modular file. If we zoom out significantly, we can see them,  the red nodes are models that do not have a modular file yet.</p>
+<p><img src="/images/transformers/big_picture_zoomout.png" alt="Red nodes"/></p>
+<p>Hence the next question, and how do we identify modularisable models?</p>
+<div class="crumbs"><p>Llama-lineage is a hub; several VLMs remain islands — engineering opportunity for shared parents.
+<strong>Next:</strong> timeline + similarity signals to spot modularisable candidates.</p></div>
 <h3 id="many-models-but-not-enough-yet-are-alike"><a href="#many-models-but-not-enough-yet-are-alike">Many models, but not enough yet, are alike</a></h3>
+<p>I looked into Jaccard similarity, which we use to measure set differences, to find similarities across models. I know that code is more than a set of characters stringed together. We also tried code-embedding models that ranked candidates better in practice, but for this post we stick to the deterministic Jaccard index.</p>
+<p>It is interesting, for our comparison, to look at <em>when</em> we deployed the modular logic and what was its rippling effect on the library. You can check the <a href="https://huggingface.co/spaces/Molbap/transformers-modular-refactor">larger space</a> to play around, but the gist is: adding modular allowed to connect more and more models to solid reference points.</p>
 <p>Yet, we still have a lot of gaps to fill.</p>
+<p>Zoom out below - it’s full of models. You can click on a node to see its connections better, or use the text box to search for a model. You can use the <a href="https://huggingface.co/spaces/Molbap/transformers-modular-refactor">full viewer</a> (tab “timeline”, hit “build timeline”) for better exploration.</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-oii2s57xyo">    <iframe
   src="https://molbap-timeline-1.hf.space"
   style="width:100%; height:680px; border:0"
   allow="clipboard-read; clipboard-write; fullscreen"
   referrerpolicy="no-referrer-when-downgrade"
 ></iframe></div></div></figure>
+<p>Let’s look at a few highly connected models. Let’s start by the foundational work of <a href="https://arxiv.org/abs/2304.08485">Llava</a>.</p>
+<p><img src="/images/transformers/timeline_llava.png" alt="DETR archipelago"/></p>
+<p>You see that <code>llava_video</code> is a red node, connected by a red edge to <code>llava</code>: it’s a candidate, something that we can <em>likely</em> remodularize, <a href="#backwards-compatibility">not touching the actual model</a> but being much more readable with <a href="#do-repeat-yourself">DRY*</a>.</p>
 <div class="crumbs"><p>Similarity metrics (Jaccard index or embeddings) surfaces likely parents; the timeline shows consolidation after modular landed. Red nodes/edges = candidates (e.g., <code>llava_video</code> → <code>llava</code>) for refactors that preserve behavior. <strong>Next:</strong> concrete VLM choices that avoid leaky abstractions.</p></div>
 <h3 id="vlm-improvements-avoiding-abstraction"><a href="#vlm-improvements-avoiding-abstraction">VLM improvements, avoiding abstraction</a></h3>
 <p>We don’t yet have a cookbook for common VLM patterns (image token scatter, multi‑tower encoders, cross‑attention bridges). This is one of the main improvement points where we can work.</p>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583">        return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8"> special_image_mask, special_video_mask</span></span>
 <span class="line"></span></code></pre></div>
 <p>But this is <em>within</em> the modeling file, not in the <code>PreTrainedModel</code> base class. It will not move away from it, because it’d break the <a href="#one-model-one-file">self-contained logic</a> of the model.</p>
+<div class="crumbs"><p>Keep VLM embedding mix in the modeling file (semantics), standardize safe helpers (e.g., placeholder masking), don’t migrate behavior to <code>PreTrainedModel</code>.
+<strong>Next:</strong> pipeline-level wins that came from PyTorch-first choices (fast processors).</p></div>
 <h3 id="on-image-processing-and-processors"><a href="#on-image-processing-and-processors">On image processing and processors</a></h3>
+<p>Deciding to become a <code>torch</code>-first library meant relieving a tremendous amount of support for <code>jax </code> and <code>TensorFlow</code>, and it also meant that we could be more lenient into the amount of torch-dependent utilities that we were able to accept. One of these is the <em>fast processing</em> of images. Where inputs were once minimally assumed to be ndarrays, enforcing native <code>torch</code> and <code>torchvision</code> inputs allowed us to massively improve processing speed for each model.</p>
+<p>The gains in performance are immense, up to 20x speedup for most models when using compiled torchvision ops. Furthermore, it allows to run the whole pipeline solely on GPU.</p>
 <p><img src="/images/transformers/fast_image_processors.png" alt="Fast Image Processors Performance"/>
 <p class="figure-legend">Thanks <a href="https://huggingface.co/yonigozlan">Yoni Gozlan</a> for the great work!</p></p>
+<div class="crumbs"><p>PyTorch-first lets processors assume torch/torchvision and run the whole pipeline on GPU; big per-model speedups.</p><p><strong>Next:</strong> how this lowers friction for contributors and downstream users.</p></div>
 <h2 id="reduce-barrier-to-entrycontribution"><a href="#reduce-barrier-to-entrycontribution">Reduce barrier to entry/contribution</a></h2>
 <p>This is an overall objective: there’s no <code>transformers</code> without its community.</p>
 <p>Having a framework means forcing users into it. It restrains flexibility and creativity, which are the fertile soil for new ideas to grow.</p>
 <strong>Next:</strong> power tools enabled by a consistent API.</p></div>
 <h3 id="-models-popularity"><a href="#-models-popularity"><a id="encoders-ftw"></a> Models popularity</a></h3>
 <p>Talking about dependencies, we can take a look at the number of downloads as a measure of popularity. One thing we see is the prominence of encoders, despite the apparent prevalence of decoder LLMs. The reason is that encoders are used to generate embeddings, which have multiple downstream uses. Just check out <a href="https://huggingface.co/blog/embeddinggemma">EmbeddingGemma</a> for a modern recap. Hence, it is vital to keep the encoders portion of the library viable, usable, fine-tune-able.</p>
+<div><figure class="html-embed"><div class="html-embed__card"><div id="frag-ejk5kk4wtm"><html>
 <head><meta charset="utf-8" /></head>
 <body>
     <div>                        <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
 <p>All models have the same API for attention computation, thanks to <a href="#external-attention-classes">the externalisation of attention classes</a>.</p>
 <p>This uniformity allows us to build cool tools to visualize the inner workings of the attention mechanism.</p>
 <p>One particular piece of machinery is the <code>attention mask</code>. Here you see the famous bidirectional attention pattern for the whole prefix (text + image) in PaliGemma and all Gemma2+ models, contrasting with the usual “causal-only” models.</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-1ihz7she8ee"><!-- Minimal HTML fragment: terminal-style ASCII attention masks -->
 <div style="max-width: 940px; margin: 16px 0; border:1px solid #2a2f3a; border-radius:8px; background:#0b0f19; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', monospace; color:#e5e7eb;">
     <div style="display:flex; align-items:center; gap:8px; padding:8px 10px; border-bottom:1px solid #1f2430; background:#111827; border-top-left-radius:8px; border-top-right-radius:8px;">
       <span style="width:10px; height:10px; background:#ef4444; border-radius:50%; display:inline-block;"></span>
     </div>
   </div>
   </div></div></figure>
+<div class="crumbs"><p>Uniform attention APIs enable cross-model diagnostics (e.g., PaliGemma prefix bidirectionality vs causal).</p><p><strong>Next:</strong> whole-model tracing for ports and regressions.</p></div>
 <h3 id="logging-entire-model-activations"><a href="#logging-entire-model-activations">Logging entire model activations</a></h3>
+<p>Because everything is PyTorch, we can easily <a href="https://huggingface.co/docs/transformers/internal/model_debugging_utils">debug any model</a> when we want to add it to transformers. We now have a power-user tool for porting or adding models, that wraps a forward pass, intercepts every submodule call, and logs shapes, dtypes, and sample statistics of inputs/outputs to nested JSON.</p>
+<p>It just works with PyTorch models and is especially useful when aligning outputs with a reference implementation, to match our <a href="#source-of-truth">Source of Truth guideline</a>.</p>
 <p><img src="/images/transformers/model_debugger.png" alt="Model debugger interface"/></p>
+<div class="crumbs"><p>Forward interception and nested JSON logging align ports to reference implementations, reinforcing “Source of Truth.” <strong>Next:</strong> CUDA warmup reduces load-time without touching modeling semantics.</p></div>
 <h3 id="cooking-faster-cuda-warmups"><a href="#cooking-faster-cuda-warmups">Cooking faster CUDA warmups</a></h3>
+<p>Having a clean <em>external</em> API allows us to work on the <a href="#code-is-product">true inner workings of transformers</a>. One of a few recent additions was the <em>CUDA warmup</em> via <code>caching_allocator_warmup</code>, which dramatically improved loading times by pre-allocating GPU memory to avoid malloc bottlenecks during model loading. It can achieve a 7x speedup factor for an 8B model, or 6x for a 32B one, as you can check in <a href="https://github.com/huggingface/transformers/pull/36380">the PR</a>!</p>
+<figure class="html-embed"><div class="html-embed__card"><div id="frag-20u85e0tulx"><style>
 /* 1) Scope tokens to the widget */
 .warmup-demo{
   --page-bg:#ffffff;
     }
 </script></div></div></figure>
 <p>It’s hard to overstate how much of a lifesaver that is when you’re trying to load a model as fast as possible, as it’s the narrowest bottleneck for your iteration speed.</p>
+<div class="crumbs"><p>Pre-allocating GPU memory removes malloc spikes (e.g., 7× for 8B, 6× for 32B in the referenced PR).</p><p><strong>Next:</strong> consistent interfaces allow transformers-serve.</p></div>
 <h3 id="transformers-serve-and-continuous-batching"><a href="#transformers-serve-and-continuous-batching">Transformers-serve and continuous batching</a></h3>
 <p>Having all these models readily available and sharing the same interface allowed us to implement transformers-serve, a CLI tool to expose models through a standard OpenAI http API.</p>
 <div class="code-card"><button class="code-copy button--ghost" type="button" aria-label="Copy code"><svg viewBox="0 0 24 24" aria-hidden="true" focusable="false"><path d="M16 1H4c-1.1 0-2 .9-2 2v12h2V3h12V1zm3 4H8c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h11c1.1 0 2-.9 2-2V7c0-1.1-.9-2-2-2zm0 16H8V7h11v14z"></path></svg></button><pre class="astro-code astro-code-themes github-light github-dark" style="--shiki-light:#24292e;--shiki-dark:#e1e4e8;--shiki-light-bg:#fff;--shiki-dark-bg:#24292e;overflow-x:auto" tabindex="0" data-language="bash"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0">transformers</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF"> serve</span></span>
 <strong>Next:</strong> what changes in v5 without breaking the promise of visible semantics.</p></div>
 <h2 id="what-is-coming-next"><a href="#what-is-coming-next">What is coming next</a></h2>
 <p>The next major version of <code>transformers</code> is just around the corner (and will have another blog post to its name when it comes out). When v5 is released, we aim to keep <a href="#backwards-compatibility">backwards compatibility</a> as solid as possible. The changes we make now are in service of that goal.</p>
+<p>We will lean further into a modular toolbox, not a framework. You should not be forced to rewrite modeling code. It’s better when a model can inherit from <code>PreTrainedModel</code> and opt into Tensor Parallel, <code>from_pretrained</code>, sharding, <code>push_to_hub</code>, loss plumbing, and external stacks like PEFT/TRL/SGLang/vLLM.</p> </main> </section> <footer class="footer"> <div class="footer-inner"> <section class="citation-block"> <h3>Citation</h3> <p>For attribution, cite this work as</p> <pre class="citation short">Pablo Montalvo, Lysandre Debut, Pedro Cuenca, Yoni Gozlan (2025). &quot;Maintain the unmaintainable: 1M python loc, 400+ models&quot;.</pre> <p>BibTeX citation</p> <pre class="citation long">@misc{montalvo2025_maintain_the_unmaintaina,
   title={Maintain the unmaintainable: 1M python loc, 400+ models},
+  author={Pablo Montalvo and Lysandre Debut and Pedro Cuenca and Yoni Gozlan},
   year={2025},
 }</pre> </section>   <section class="references-block">  </section> </div> </footer> <script>

app/dist/index.html.gz CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ed0105da49dc4bd2501866a3127d0d5d87cece9f044a8d3acf75e7007e611f9f
-size 1488729

 version https://git-lfs.github.com/spec/v1
+oid sha256:6912a23ebe8db8e4eda9fcfcb65eb6cd7fa541f19286b11060733984d7a7f8ff
+size 1489962

app/dist/llama_center.png ADDED Viewed

Git LFS Details

SHA256: 3ec2caa493f919717ece1366836e156d8d05a3bf09ef4313ea502d5130a82cb0
Pointer size: 131 Bytes
Size of remote file: 406 kB

app/dist/llama_glm_attn.png ADDED Viewed

Git LFS Details

SHA256: 6b2c88d5eb3d461d791e7e280f74e54d05f01babb02ea0536b50386b7b1b1b8a
Pointer size: 131 Bytes
Size of remote file: 138 kB

app/public/images/transformers/big_picture_zoomout.png ADDED Viewed

Git LFS Details

SHA256: 6b48173ad33c50e9b1b7f674bb21948da982db04e4a927cf0cecee45bc749297
Pointer size: 131 Bytes
Size of remote file: 218 kB

app/public/images/transformers/cluster_wave2vec2.png ADDED Viewed

Git LFS Details

SHA256: ad2931607cfd522cbccddc8047ee7fd6ee3945a2d818fe72b6c6b08c58e062b3
Pointer size: 130 Bytes
Size of remote file: 55.4 kB

app/public/images/transformers/detr_island.png ADDED Viewed

Git LFS Details

SHA256: 6f6daf8ce4f8e71a0a9b3c60f2a7a18aacf1812a54337cf345b9005eaa251125
Pointer size: 130 Bytes
Size of remote file: 18.5 kB

app/public/images/transformers/llama_center.png ADDED Viewed

Git LFS Details

SHA256: 3ec2caa493f919717ece1366836e156d8d05a3bf09ef4313ea502d5130a82cb0
Pointer size: 131 Bytes
Size of remote file: 406 kB

app/public/images/transformers/llama_glm_attn.png ADDED Viewed

Git LFS Details

SHA256: 6b2c88d5eb3d461d791e7e280f74e54d05f01babb02ea0536b50386b7b1b1b8a
Pointer size: 131 Bytes
Size of remote file: 138 kB

app/public/images/transformers/timeline_llava.png ADDED Viewed

Git LFS Details

SHA256: 2bd0469fa24737bf309c1225005b62d7f0a9d722df0e56a18c578a1327cf94fc
Pointer size: 131 Bytes
Size of remote file: 621 kB

app/public/llama_center.png ADDED Viewed

Git LFS Details

SHA256: 3ec2caa493f919717ece1366836e156d8d05a3bf09ef4313ea502d5130a82cb0
Pointer size: 131 Bytes
Size of remote file: 406 kB

app/public/llama_glm_attn.png ADDED Viewed

Git LFS Details

SHA256: 6b2c88d5eb3d461d791e7e280f74e54d05f01babb02ea0536b50386b7b1b1b8a
Pointer size: 131 Bytes
Size of remote file: 138 kB

app/src/components/Hero.astro CHANGED Viewed

@@ -183,19 +183,6 @@ const pdfFilename = `${slugify(pdfBase)}.pdf`;
         <p><a href={`https://doi.org/${doi}`} target="_blank" rel="noopener noreferrer">{doi}</a></p>
       </div>
     )} -->
-    <div class="meta-container-cell meta-container-cell--pdf">
-      <h3>PDF</h3>
-      <p>
-        <a
-          class="button"
-          href={`/${pdfFilename}`}
-          download={pdfFilename}
-          aria-label={`Download PDF ${pdfFilename}`}
-        >
-          Download PDF
-        </a>
-      </p>
-    </div>
   </div>
 </header>

         <p><a href={`https://doi.org/${doi}`} target="_blank" rel="noopener noreferrer">{doi}</a></p>
       </div>
     )} -->
   </div>
 </header>

app/src/content/article.mdx CHANGED Viewed

@@ -6,6 +6,15 @@ authors:
   - name: "Pablo Montalvo"
     url: "https://huggingface.co/Molbap"
     affiliations: [1]
 affiliations:
   - name: "Hugging Face"
     url: "https://huggingface.co"
@@ -29,34 +38,38 @@ We continue to support all new models and expect to do so for the foreseeable fu
 This post dissects the design philosophy that makes this possible. It's the result of an evolution from our older principles, detailed on our previous [philosophy](https://huggingface.co/docs/transformers/en/philosophy) page, as well as its accompanying [blog post from 2022](https://huggingface.co/blog/transformers-design-philosophy). More recently (and we strongly recommend the read) we published a blog post about [recent upgrades to transformers](https://huggingface.co/blog/faster-transformers), focusing on what makes the library faster today. All of these developments are only made possible thanks to these principles.
-We codify the "tenets" that guide our development, demonstrate how they are implemented in code, and show the measurable impact they have on the library's sustainability and growth.
-For any OSS maintainer, power user, or contributor, this is the map to understanding, using, and building upon `transformers`, but not only: any project of comparable size will require you to make deep choices, not only on design and choice of abstraction, but on the very mindset of the software you are building.
 [Tenets exemplified](#source-of-truth) will have their summary available on hover.
 [External links](https://huggingface.co/blog/welcome-openai-gpt-oss) to articles will help you solidify your knowledge.
-[Several interactive visualisations](#generated-modeling) are available as you go - scroll, zoom, drag away.
 <div class="crumbs">
-Throughout this post, you'll find breadcrumb boxes like this one. They summarize what you just learned, connect it to the tenets, and point to what's coming <strong>Next</strong>. Think of them as narrative signposts to help you keep track.
 </div>
 ## The core tenets of transformers
 We summarize the foundations on which we've built everything, and write the "tenets" of the library.  They behave like _software interfaces_, hence it is crucial that they are explicitly written down. However opinionated they are, they have evolved over time.
-Note that the library _evolved_ towards these principles, and that they _emerged_ from decisions taken, and once emerged they were recognized as critical.
 <div class="tenet-list">
 <ol>
 <li class="tenet">
 <a id="source-of-truth"></a>
 <strong>Source of Truth</strong>
-<p>We aim to be a [source of truth for all model definitions](#https://huggingface.co/blog/transformers-model-definition). This is not a tenet, but something that still guides our decisions. Model implementations should be reliable, reproducible, and faithful to the original performances.</p>
-<em>This overarching guideline ensures quality and reproducibility across all models in the library.</em>
 </li>
 <li class="tenet">
@@ -94,24 +107,26 @@ Note that the library _evolved_ towards these principles, and that they _emerged
 <a id="backwards-compatibility"></a>
 <strong>Backwards Compatibility</strong>
 <p>Evolve by additive standardization, never break public APIs.</p>
-<p>Any artifact that was once on the hub and loadable with transformers should be usable indefinitely with the same interface. Further, public methods should not change to avoid breaking dependencies.</p>
 <em>Once something is public, it stays public, evolution through addition, not breaking changes.</em>
 </li>
 <li class="tenet">
 <a id="consistent-public-surface"></a>
 <strong>Consistent Public Surface</strong>
-<p>Same argument names, same outputs, hidden states and attentions exposed, enforced by tests. This is a goal we have as well as a tenet.</p>
 <em>All models should feel familiar - consistent interfaces reduce cognitive load.</em>
 </li>
 </ol>
 </div>
-When a PR is merged, it is because the contribution is worthwhile, and that the  `transformers` team finds the design of the contribution to be aligned with what is above.
-Does all the code in the library follow strictly these tenets? No. The library is a gigantic house with connected nooks, corridors, crannies everywhere built by thousands of different workers. We _try_ to make it so all the code added is compliant, because if we fail and merge it, we cannot change it lest we break [backwards compatibility](#backwards-compatibility).
-For instance, one function essential to the implementation of [Rotary Positional Embeddings](https://huggingface.co/papers/2104.09864) is identical in 70 `modeling_<file>.py` across `src/transformers/models/.`  Why keep it? Because we want all the model logic to be [contained in the modeling file](#one-model-one-file). In order to do that, we [do repeat ourselves](#do-repeat-yourself).
 ```python
 def rotate_half(x):
@@ -121,43 +136,51 @@ def rotate_half(x):
     return torch.cat((-x2, x1), dim=-1)
 ```
-You can use a simple regex to look at all methods of a given name across your codebase and look at their differences and similarities, that's what I did (+ a hash to avoid quadraticity).
-We want all models to have self-contained modeling code.
-Every core functionality _must_ be in the modeling code, every non-core functionality _can_ be outside of it.
-This comes as a great cost. Enter the `#Copied from...` mechanism: for a long time, these comments were indicating that some code was copied from another model, saving time both for the reviewers and for the CI. But the LOC count kept creeping up. Each new model copied over hundreds of lines that we considered largely boilerplate, yet, we could not remove them.
-We needed to separate both principles that were so far intertwined, [repetition](#do-repeat-yourself) and [hackability](#one-model-one-file).
-What was the solution to this?
 <div class="crumbs">
-Read the code in one place (<a href="#one-model-one-file">One Model, One File</a>). Keep semantics local (<a href="#standardize-dont-abstract">Standardize, Don't Abstract</a>). Allow strategic duplication for end users (<a href="#do-repeat-yourself">DRY*</a>). Keep the public surface minimal and stable (<a href="#minimal-user-api">Minimal API</a>, <a href="#backwards-compatibility">Backwards Compatibility</a>, <a href="#consistent-public-surface">Consistent Surface</a>). <strong>Next:</strong> how modular transformers honor these while removing boilerplate.
 </div>
 ## <a id="modular"></a> Modular transformers
-Transformers is an opinionated library. The previous [philosophy](https://huggingface.co/docs/transformers/en/philosophy) page, and the [blog post](https://huggingface.co/blog/transformers-design-philosophy) were already pointing at the drawbacks mentioned just above, which have been iteratively addressed. [`modular` transformers were introduced](https://huggingface.co/docs/transformers/en/modular_transformers), allowing a form of inheritance without breaking [One model, One file](#one-model-one-file).
-We amended the principle of [DRY*](#do-repeat-yourself) by removing progressively all pieces of code that were "copied from" another file.
-It works as follows. In order to contribute a model, say for instance  define a `modular_` file that can inherit from _any function across all other modeling, configuration and processor files_.
-This modular file can use inheritance across models: and then, it will be unravelled into a fully functional modeling file.
 <summary id="generated-modeling">Auto-generated modeling code</summary>
 <HtmlEmbed src="transformers/glm-compare.html" />
-As you can see, we can now define any model as a _modular_ of another.
 You might think "well that's just how inheritance works". The crucial difference is that we do _visibly_ what is essentially the _compiler_'s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it [all in one piece](#one-model-one-file).
 What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.
-When `AutoModel.from_pretrained(...)` is called, it is indeed the modeling (right side) that is ran, and all the tests are run on the modeling code.
 What does that give us?
@@ -177,10 +200,11 @@ That gives an "effective LOC" curve: the 𝗺𝗮𝗶𝗻𝘁𝗲𝗻𝗮𝗻
 Measured on git history, raw `modeling_*.py` grew at ~362 LOC/day before modular; counting only modular shards yields ~25 LOC/day after — about **15× lower**. The effective curve (blue line below) represents the **maintenance surface** today: what maintainers actually read and review.
-Less code to hand-maintain means fewer places to break. LOC is not a direct measure of complexity, but they correlate in review effort and change risk.
 <HtmlEmbed src="transformers/loc-growth.html" />
 If you zoom in, you'll notice there's a sharp drop near the end, it's essentially due to us [removing support for Jax and TensorFlow](https://github.com/huggingface/transformers/commit/4df2529d79d75f44e70396df5888a32ffa02d61e#diff-60849db3e9922197854ef1cac92bf4aba08b5d7fd3fe6f3c16a3511e29e0eacc) library-wide.
 But this was not the only effort that allowed us to reduce maintenance load.
@@ -238,30 +262,32 @@ Attention semantics remain in <code>eager_attention_forward</code>; faster backe
 If you're not familiar with the different flavours of parallelism, I recommend checking out [this blog post](https://huggingface.co/blog/accelerate-nd-parallel) first, and of course a full [dive into the ultra-scale playbook](https://huggingface.co/spaces/nanotron/ultrascale-playbook) is always recommended.
-The essential part is that, as [the documentation states](https://huggingface.co/docs/transformers/v4.56.2/perf_train_gpu_many#tensor-parallelism) when tensors get too large to fit on a single GPU, they are sliced along a particular dimension and every slice is sent to a different GPU.
 Why does it matter?
 Because we want to avoid code modifications that are unrelated to the model.
-We choose to place the level of abstraction higher than the device placement: a matrix multiplication - a `nn.Linear` layer - should be always expressed in the same way, regardless of how it is placed.
-Hence, we want to touch [minimally](#minimal-user-api) to the modeling code, and only modify it when _architectural changes_ are involved. For instance, for tensor parallelism, we instead now specify a simple `tp_plan`.
-The alternative would be to modify parent classes specific to their
-It is written once in the config and passed to `.from_pretrained()`. The plan maps module name patterns to partitioning strategies. Strategies are resolved by the internal `ParallelInterface`, which wires to sharding implementations `ColwiseParallel`, `RowwiseParallel`, packed variants, and so on.
 <HtmlEmbed src="transformers/tp-plan.html" />
-Which allows a user to run with multiple processes per node, e.g. 4 GPUs:
 `torchrun --nproc-per-node 4 demo.py`
-Semantics stay in the model (a Linear stays a Linear), distribution is orthogonal and declared via strings: "colwise" splits columns of weights/bias across ranks; "rowwise" splits rows; packed variants shard fused weights; The mapping keys accept glob patterns like `layers.*.mlp.down_proj` to target repeated submodules.
 <div class="crumbs">
-Sharding is configuration (<code>tp_plan</code>), not edits to <code>Linear</code>s. Glob patterns target repeated blocks; modeling semantics stay intact. <strong>Next:</strong> per-layer attention/caching schedules declared in config, not hardcoded.
 </div>
 ### <a id="layers-attentions-caches"></a> Layers, attentions and caches
@@ -310,18 +336,20 @@ class GlmRMSNorm(nn.Module):
     ...
 ```
-This also opens another contribution path: GPU specialists can contribute optimized kernels to the kernel hub, and have them usable in `transformers`. You can check on the [kernel community blog post](https://huggingface.co/blog/hello-hf-kernels) to learn more about it!
 Even more resources have been added, like the formidable [kernel builder](https://github.com/huggingface/kernel-builder) with its connected resources to [help you build kernels with it](https://github.com/huggingface/kernel-builder/blob/main/docs/writing-kernels.md) and [with nix](https://github.com/huggingface/kernel-builder/blob/main/docs/nix.md).
 <div class="crumbs">
-Models define semantics; kernels define how to run them faster. Use annotations to borrow community forwards while keeping a consistent public surface. <strong>Next:</strong> what modularity looks like across the repo.
 </div>
-## Modular developments
-Now, we have a form of inheritance in our codebase. Some models become standards, and model contributors are given the opportunity to _define standards_. Pushing the boundaries of scientific knowledge can translate into the boundaries of engineering if this effort is made, and we're striving for it.
 It's hard to conceptualize very large libraries and how their components interact with each other, regardless of your cognitive abilities for abstractions.
 So I wanted to take a look at the current **state of modularity** across the repository. How many models are defined using components of others?
@@ -330,37 +358,64 @@ To get this graph, I used the heuristic of modular inheritance.
 2. In this `modular` file, what models, configurations and processings are imported?
 3. Recurse through the model list that way.
-So what do we see? Llama is a basis for many models, and it shows.
-Radically different architectures such as mamba have spawned their own dependency subgraph.
 <HtmlEmbed src="transformers/dependency-graph.html" />
 In the case of VLMs, there's far too many vision-based architectures that are not yet defined as modulars of other existing archs. In other words, there is no strong reference point in terms of software for vision models.
-As you can see, there is a small DETR island, a little llava pocket, and so on, but it's not comparable to the centrality observed for llama.
-Another problem is, this visualization only shows `modular` models. Several models still do NOT have a modular file.
-How do we spot them, and how do we identify modularisable models?
 <div class="crumbs">
-Graph reading guide: nodes are models; edges are modular imports. Llama-lineage is a hub; several VLMs remain islands — engineering opportunity for shared parents. <strong>Next:</strong> timeline + similarity signals to spot candidates.
 </div>
 ### Many models, but not enough yet, are alike
-I looked into Jaccard similarity, which we use to measure set differences, to find similarities across models. I know that code is more than a set of characters stringed together. We also tried code-embedding models that ranked candidates better in practice, but for this post we stick to the deterministic Jaccard index. You can take a look at [the corresponding PR](https://github.com/huggingface/transformers/pull/41289) for the embedding method.
-It is interesting, for our comparison, to look at _when_ we deployed this modular logic and what was its rippling effect on the library. You can check the [larger space](https://huggingface.co/spaces/Molbap/transformers-modular-refactor) to play around, but the gist is: adding modular allowed to connect more and more models to solid reference points.
 Yet, we still have a lot of gaps to fill.
-Zoom out below - it's full of models. You can click on a node to see its connections better, or use the text box to search for a model.
 <HtmlEmbed src="transformers/model-timeline.html" />
-If you've checked out llava, you've seen that llava_video is a red node, connected by a red edge to llava: it's a candidate, something that we can _likely_ remodularize, [not touching the actual model](#backwards-compatibility) but being much more readable with [DRY*](#do-repeat-yourself).
 <div class="crumbs">
 Similarity metrics (Jaccard index or embeddings) surfaces likely parents; the timeline shows consolidation after modular landed. Red nodes/edges = candidates (e.g., <code>llava_video</code> → <code>llava</code>) for refactors that preserve behavior. <strong>Next:</strong> concrete VLM choices that avoid leaky abstractions.
@@ -435,21 +490,24 @@ The following [Pull request to standardize placeholder masking](https://github.c
 But this is _within_ the modeling file, not in the `PreTrainedModel` base class. It will not move away from it, because it'd break the [self-contained logic](#one-model-one-file) of the model.
 <div class="crumbs">
-Keep VLM embedding mix in the modeling file (semantics), standardize safe helpers (e.g., placeholder masking), don't migrate behavior to <code>PreTrainedModel</code>. <strong>Next:</strong> pipeline-level wins that came from PyTorch-first choices (fast processors).
 </div>
 ### On image processing and processors
-Choosing to be a `torch`-first software meant relieving a tremendous amount of support from `jax ` and `TensorFlow` , and it also meant that we could be more lenient into the amount of torch-dependent utilities that we were able to add. One of these is the _fast processing_ of images. Where they were before assumed to be minimal ndarrays, making stronger assumptions and enforcing `torch` and `torchvision` native inputs allowed up to speed up massively the processing time for each model.
-The gains in performance are immense, up to 20x speed for most models when compiled torchvision ops. Further, it allows to have the whole pipeline solely on GPU.
 ![Fast Image Processors Performance](/images/transformers/fast_image_processors.png)
 <p class="figure-legend">Thanks <a href="https://huggingface.co/yonigozlan">Yoni Gozlan</a> for the great work!</p>
 <div class="crumbs">
-Torch-first lets processors assume torch/torchvision and run the whole pipeline on GPU; big per-model speedups. <strong>Next:</strong> how this lowers friction for contributors and downstream users.
 </div>
@@ -506,38 +564,42 @@ One particular piece of machinery is the `attention mask`. Here you see the famo
 <div class="crumbs">
 Uniform attention APIs enable cross-model diagnostics (e.g., PaliGemma prefix bidirectionality vs causal).
 <strong>Next:</strong> whole-model tracing for ports and regressions.
 </div>
 ### Logging entire model activations
-Further, because it is all PyTorch (and it is even more now that we support only PyTorch), we can easily [debug any model](https://huggingface.co/docs/transformers/internal/model_debugging_utils) when we want to add it to transformers. We now have a power-user tool for porting or adding models, that wraps a forward pass, intercepts every submodule call, and logs shapes, dtypes, and sample statistics of inputs/outputs to nested JSON.
-It just works with PyTorch models and is especially useful when aligning outputs with a reference implementation, aligned with our [core guideline](#source-of-truth).
 ![Model debugger interface](/images/transformers/model_debugger.png)
 <div class="crumbs">
-Forward interception and nested JSON logging align ports to reference implementations, reinforcing "Source of Truth." <strong>Next:</strong> CUDA warmup reduces load-time stalls without touching modeling semantics.
 </div>
 ### Cooking faster CUDA warmups
-Having a clean _external_ API allows us to work on the [true inner workings of transformers](#code-is-product). One of the few recent additions was the _CUDA warmup_ via `caching_allocator_warmup` which improved massively the loading footprint by pre-allocating GPU memory to avoid malloc bottlenecks during model loading, achieving a 7x factor for an 8B model, 6x for a 32B, you can check out [the source](https://github.com/huggingface/transformers/pull/36380)!
 <HtmlEmbed src="transformers/warmup_demo.html" />
 It's hard to overstate how much of a lifesaver that is when you're trying to load a model as fast as possible, as it's the narrowest bottleneck for your iteration speed.
 <div class="crumbs">
-Pre-allocating GPU memory removes malloc spikes (e.g., 7× for 8B, 6× for 32B in the referenced PR). <strong>Next:</strong> serving benefits directly from consistent interfaces and modularity.
 </div>
 ### Transformers-serve and continuous batching
 Having all these models readily available and sharing the same interface allowed us to implement transformers-serve, a CLI tool to expose models through a standard OpenAI http API.

   - name: "Pablo Montalvo"
     url: "https://huggingface.co/Molbap"
     affiliations: [1]
+  - name: "Lysandre Debut"
+    url: "https://huggingface.co/Lysandre"
+    affiliations: [1]
+  - name: "Pedro Cuenca"
+    url: "https://huggingface.co/pcuenq"
+    affiliations: [1]
+  - name: "Yoni Gozlan"
+    url: "https://huggingface.co/yonigozlan"
+    affiliations: [1]
 affiliations:
   - name: "Hugging Face"
     url: "https://huggingface.co"
 This post dissects the design philosophy that makes this possible. It's the result of an evolution from our older principles, detailed on our previous [philosophy](https://huggingface.co/docs/transformers/en/philosophy) page, as well as its accompanying [blog post from 2022](https://huggingface.co/blog/transformers-design-philosophy). More recently (and we strongly recommend the read) we published a blog post about [recent upgrades to transformers](https://huggingface.co/blog/faster-transformers), focusing on what makes the library faster today. All of these developments are only made possible thanks to these principles.
+We formalize and articulate the "tenets" that have been guiding our development, demonstrate how they are implemented in code, and show the measurable impact they have on the library's sustainability and growth.
+For any OSS maintainer, power user, or contributor, this is the map to understanding, using, and building upon `transformers`, but not only: any project of comparable size will require you to make deep choices, not only on design and choice of abstraction, but on the very mindset of the software you are building. These tenets may or may not be applicable to your project, but they provide a glimpse on how we work that could be helpful or inspirational.
+Conventions used throughout this post:
 [Tenets exemplified](#source-of-truth) will have their summary available on hover.
 [External links](https://huggingface.co/blog/welcome-openai-gpt-oss) to articles will help you solidify your knowledge.
+[Several interactive visualisations](#generated-modeling) are available as you go - scroll, zoom, drag away to explore them.
 <div class="crumbs">
+* Breadcrumb boxes summarize what you just learned, connect it to the tenets, and point to what's coming <strong>Next</strong>. Think of them as narrative signposts to help you keep track.
 </div>
+We will get started by enumerating the tenets. Then we'll look at concrete examples that show how they shape our decision-making. These examples are necessarily detailed, and sometimes complex, because they illustrate the challenges to maintain and grow a large codebase that caters to multiple collectives, has millions of users, hundreds of contributors, and always strives for simplicity and consistency.
 ## The core tenets of transformers
 We summarize the foundations on which we've built everything, and write the "tenets" of the library.  They behave like _software interfaces_, hence it is crucial that they are explicitly written down. However opinionated they are, they have evolved over time.
+These principles were not decided in a vacuum. The library _evolved_ towards them, and once they _emerged_, they were recognized as critical.
 <div class="tenet-list">
 <ol>
 <li class="tenet">
 <a id="source-of-truth"></a>
 <strong>Source of Truth</strong>
+<p>We aim to be a [source of truth for all model definitions](https://huggingface.co/blog/transformers-model-definition). This is more of a goal than a tenet, but it strongly guides our decisions. Model implementations should be reliable, reproducible, and faithful to the original implementations. If we are successful, they should become reference baselines for the ecosystem, so they'll be easily adopted by downstream libraries and projects. It's much easier for a project to _always_ refer to the transformers implementation, than to learn a different research codebase every time a new architecture is released.</p>
+<em>This overarching guideline ensures quality and reproducibility across all models in the library, and aspires to make the community work easier.</em>
 </li>
 <li class="tenet">
 <a id="backwards-compatibility"></a>
 <strong>Backwards Compatibility</strong>
 <p>Evolve by additive standardization, never break public APIs.</p>
+<p>Any artifact that was once on the hub and loadable with transformers should be usable indefinitely with the same interface. Further, public methods should not change to avoid breaking dependencies. If we do deprecate something, it's with very long cycles beforehand.</p>
 <em>Once something is public, it stays public, evolution through addition, not breaking changes.</em>
 </li>
 <li class="tenet">
 <a id="consistent-public-surface"></a>
 <strong>Consistent Public Surface</strong>
+<p>Same argument names, same outputs, hidden states and attentions exposed, enforced by tests. This is a goal as well as a tenet.</p>
 <em>All models should feel familiar - consistent interfaces reduce cognitive load.</em>
 </li>
 </ol>
 </div>
+When a PR is merged, it is because the contribution is worthwhile, and because the `transformers` team finds the design of the contribution to be aligned with the tenets.
+Does all the code in the library strictly follow these tenets? No. The library is a gigantic house with connected nooks, corridors, crannies everywhere, built by thousands of different workers. We _try_ to make it so all the code added is compliant, because if we fail and merge it, we cannot change it lest we break [backwards compatibility](#backwards-compatibility).
+To see what constitutes adherence to the tenets, let's take the example of code repetition.
+The following function, essential to the implementation of [Rotary Positional Embeddings](https://huggingface.co/papers/2104.09864) can be found in more than 70 `modeling_<file>.py` across `src/transformers/models/.`  Why keep it? Because we want all the model logic to be [contained in the modeling file](#one-model-one-file). In order to do that, we [do repeat ourselves](#do-repeat-yourself).
 ```python
 def rotate_half(x):
     return torch.cat((-x2, x1), dim=-1)
 ```
+We want all models to have self-contained modeling code. Every core functionality _must_ be in the modeling code, every non-core functionality _can_ be outside of it.
+This comes at a great cost. For years, we have used what we call the `#Copied from...` mechanism: we added comments of a specific format documenting that some code was copied from another model, saving time both for the reviewers and for the CI: we had tooling to ensure that the copied blocks remained in sync.
+But the LOC count kept creeping up. Each new model copied over hundreds of lines that we considered largely boilerplate, yet, we could not remove them.
+We needed to separate two principles that were so far intertwined, [repetition](#do-repeat-yourself) and [hackability](#one-model-one-file).
+What was the solution to this? Let's talk about modular transformers.
 <div class="crumbs">
+<strong>TL;DR:</strong> Read the code in one place (<a href="#one-model-one-file">One Model, One File</a>). Keep semantics local (<a href="#standardize-dont-abstract">Standardize, Don't Abstract</a>). Allow strategic duplication for end users (<a href="#do-repeat-yourself">DRY*</a>). Keep the public surface minimal and stable (<a href="#minimal-user-api">Minimal API</a>, <a href="#backwards-compatibility">Backwards Compatibility</a>, <a href="#consistent-public-surface">Consistent Surface</a>).
+<strong>Next:</strong> how modular transformers honor these while removing boilerplate.
 </div>
 ## <a id="modular"></a> Modular transformers
+Transformers is an opinionated library. The previous [philosophy](https://huggingface.co/docs/transformers/en/philosophy) page, and the [blog post](https://huggingface.co/blog/transformers-design-philosophy) were already pointing at the drawbacks mentioned just above, which have been iteratively addressed. [`modular` transformers was introduced](https://huggingface.co/docs/transformers/en/modular_transformers) to allow a form of inheritance without breaking [One model, One file](#one-model-one-file).
+We amended the principle of [DRY*](#do-repeat-yourself) by progressively removing all pieces of code that were "copied from" another file.
+It works as follows. In order to contribute a model, let us take GLM for instance, we define a `modular_` file that can inherit from _any function across all other modeling, configuration and processor files_ already existing in the libary.
+The modular file can use inheritance across models: and then, it will be unravelled into a fully functional modeling file.
 <summary id="generated-modeling">Auto-generated modeling code</summary>
 <HtmlEmbed src="transformers/glm-compare.html" />
+As you can see, we can define a new model as a _modular_ combination of fragments taken from others.
 You might think "well that's just how inheritance works". The crucial difference is that we do _visibly_ what is essentially the _compiler_'s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it [all in one piece](#one-model-one-file).
+You can see below the difference between `GlmAttention` and `LlamaAttention`, with the latter having been copied with minimal changes.
+![Llama vs GLM](/images/transformers/llama_glm_attn.png)
 What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.
+When `AutoModel.from_pretrained(...)` is called, it is indeed the modeling (right side) that is ran, and all the tests run on the modeling code.
+More importantly, the auto-generated modeling file is what users _read_ to understand the code, what they step through in their debuggers and what they hack for their needs.
 What does that give us?
 Measured on git history, raw `modeling_*.py` grew at ~362 LOC/day before modular; counting only modular shards yields ~25 LOC/day after — about **15× lower**. The effective curve (blue line below) represents the **maintenance surface** today: what maintainers actually read and review.
+Less code to hand-maintain means fewer places to break. Naturally LOC is not a direct measure of complexity, but they correlate in review effort and change risk.
 <HtmlEmbed src="transformers/loc-growth.html" />
+The blue line (effective) is the sum of the red + green, whereas the yellow would have been the progression without modular. We can see that the maintenance surface is essentially constant (in LOC) since the implementation of `modular`.
 If you zoom in, you'll notice there's a sharp drop near the end, it's essentially due to us [removing support for Jax and TensorFlow](https://github.com/huggingface/transformers/commit/4df2529d79d75f44e70396df5888a32ffa02d61e#diff-60849db3e9922197854ef1cac92bf4aba08b5d7fd3fe6f3c16a3511e29e0eacc) library-wide.
 But this was not the only effort that allowed us to reduce maintenance load.
 If you're not familiar with the different flavours of parallelism, I recommend checking out [this blog post](https://huggingface.co/blog/accelerate-nd-parallel) first, and of course a full [dive into the ultra-scale playbook](https://huggingface.co/spaces/nanotron/ultrascale-playbook) is always recommended.
+The essential part is that, as [the documentation states](https://huggingface.co/docs/transformers/v4.56.2/perf_train_gpu_many#tensor-parallelism), when tensors get too large to fit on a single GPU, they are sliced along a particular dimension and every slice is sent to a different GPU.
 Why does it matter?
 Because we want to avoid code modifications that are unrelated to the model.
+We choose to place the level of abstraction higher than the device placement: a matrix multiplication - a `nn.Linear` layer - should be always expressed in the same way, regardless of how it is placed.
+Hence, we want to touch the modeling code [minimally](#minimal-user-api), and only modify it when _architectural changes_ are involved – not depending on the way you run it. For tensor parallelism, we simply specify a `tp_plan`:
 <HtmlEmbed src="transformers/tp-plan.html" />
+The plan is written once, saved as part of the config and passed to `.from_pretrained()`. It maps module name patterns to partitioning strategies. Strategies are resolved by the internal `ParallelInterface`, which wires to sharding implementations `ColwiseParallel`, `RowwiseParallel`, packed variants, and so on.
+The alternative would be to modify classes depending on supported types of parallelism.
+The `tp_plan` solution allows users to run the same model on a single GPU, or distribute it using multiple processes per node, e.g. 4 GPUs:
 `torchrun --nproc-per-node 4 demo.py`
+Semantics stay in the model (a Linear stays a Linear), parallelization is orthogonal and declared via strings: "colwise" splits columns of weights/bias across ranks; "rowwise" splits rows; packed variants shard fused weights; The mapping keys accept glob patterns like `layers.*.mlp.down_proj` to target repeated submodules.
 <div class="crumbs">
+Parallelization is specified in the configuration (<code>tp_plan</code>), not through edits to <code>Linear</code>s. Glob patterns target repeated blocks; modeling semantics stay intact.
+<strong>Next:</strong> per-layer attention/caching schedules declared in config, not hardcoded.
 </div>
 ### <a id="layers-attentions-caches"></a> Layers, attentions and caches
     ...
 ```
+This also opens another contribution path: GPU specialists can contribute optimized kernels to the [Kernels Hub](https://huggingface.co/kernels-community), and have them immediately available to use in `transformers` and other libraries. You can check the [kernel community blog post](https://huggingface.co/blog/hello-hf-kernels) to learn more about it!
 Even more resources have been added, like the formidable [kernel builder](https://github.com/huggingface/kernel-builder) with its connected resources to [help you build kernels with it](https://github.com/huggingface/kernel-builder/blob/main/docs/writing-kernels.md) and [with nix](https://github.com/huggingface/kernel-builder/blob/main/docs/nix.md).
 <div class="crumbs">
+Models define semantics; kernels define how to run them faster. Use decorations to borrow community forwards while keeping a consistent public surface.
+<strong>Next:</strong> what modularity looks like across the repo.
 </div>
+## A Modular State
+With `modular` transformers, we have a form of inheritance in our codebase. Some models become standards, and model contributors are given the opportunity to _define standards_. Pushing the boundaries of scientific knowledge can translate into the boundaries of engineering if this effort is made, and we're striving for it.
 It's hard to conceptualize very large libraries and how their components interact with each other, regardless of your cognitive abilities for abstractions.
 So I wanted to take a look at the current **state of modularity** across the repository. How many models are defined using components of others?
 2. In this `modular` file, what models, configurations and processings are imported?
 3. Recurse through the model list that way.
+So what do we see?
+(Graph reading guide: nodes are models; edges are modular imports).
+Check out the [full viewer here](https://huggingface.co/spaces/Molbap/transformers-modular-refactor) (tab "dependency graph", hit "build graph") for better manipulation and exploration.
 <HtmlEmbed src="transformers/dependency-graph.html" />
+Le'ts walk through some sections of this graph together.
+Llama is a basis and an influence for many models, and it shows.
+![Llama in the center](/images/transformers/llama_center.png)
+Radically different architectures such as mamba have spawned their own dependency subgraph.
+Audio models form sparser archipelagos, see for instance wav2vec2 which is a significant basis.
+![Wav2vec2 influence](/images/transformers/cluster_wave2vec2.png)
 In the case of VLMs, there's far too many vision-based architectures that are not yet defined as modulars of other existing archs. In other words, there is no strong reference point in terms of software for vision models.
+)
+As you can see, there is a small DETR island:
+![DETR archipelago](/images/transformers/detr_island.png)
+There is also a little llava pocket, and so on, but it's not comparable to the centrality observed for llama.
+Another problem is, this visualization only shows `modular` models. Several models still do NOT have a modular file. If we zoom out significantly, we can see them,  the red nodes are models that do not have a modular file yet.
+![Red nodes](/images/transformers/big_picture_zoomout.png)
+Hence the next question, and how do we identify modularisable models?
 <div class="crumbs">
+Llama-lineage is a hub; several VLMs remain islands — engineering opportunity for shared parents.
+<strong>Next:</strong> timeline + similarity signals to spot modularisable candidates.
 </div>
 ### Many models, but not enough yet, are alike
+I looked into Jaccard similarity, which we use to measure set differences, to find similarities across models. I know that code is more than a set of characters stringed together. We also tried code-embedding models that ranked candidates better in practice, but for this post we stick to the deterministic Jaccard index.
+It is interesting, for our comparison, to look at _when_ we deployed the modular logic and what was its rippling effect on the library. You can check the [larger space](https://huggingface.co/spaces/Molbap/transformers-modular-refactor) to play around, but the gist is: adding modular allowed to connect more and more models to solid reference points.
 Yet, we still have a lot of gaps to fill.
+Zoom out below - it's full of models. You can click on a node to see its connections better, or use the text box to search for a model. You can use the [full viewer](https://huggingface.co/spaces/Molbap/transformers-modular-refactor) (tab "timeline", hit "build timeline") for better exploration.
 <HtmlEmbed src="transformers/model-timeline.html" />
+Let's look at a few highly connected models. Let's start by the foundational work of [Llava](https://arxiv.org/abs/2304.08485).
+![DETR archipelago](/images/transformers/timeline_llava.png)
+You see that `llava_video` is a red node, connected by a red edge to `llava`: it's a candidate, something that we can _likely_ remodularize, [not touching the actual model](#backwards-compatibility) but being much more readable with [DRY*](#do-repeat-yourself).
 <div class="crumbs">
 Similarity metrics (Jaccard index or embeddings) surfaces likely parents; the timeline shows consolidation after modular landed. Red nodes/edges = candidates (e.g., <code>llava_video</code> → <code>llava</code>) for refactors that preserve behavior. <strong>Next:</strong> concrete VLM choices that avoid leaky abstractions.
 But this is _within_ the modeling file, not in the `PreTrainedModel` base class. It will not move away from it, because it'd break the [self-contained logic](#one-model-one-file) of the model.
 <div class="crumbs">
+Keep VLM embedding mix in the modeling file (semantics), standardize safe helpers (e.g., placeholder masking), don't migrate behavior to <code>PreTrainedModel</code>.
+<strong>Next:</strong> pipeline-level wins that came from PyTorch-first choices (fast processors).
 </div>
 ### On image processing and processors
+Deciding to become a `torch`-first library meant relieving a tremendous amount of support for `jax ` and `TensorFlow`, and it also meant that we could be more lenient into the amount of torch-dependent utilities that we were able to accept. One of these is the _fast processing_ of images. Where inputs were once minimally assumed to be ndarrays, enforcing native `torch` and `torchvision` inputs allowed us to massively improve processing speed for each model.
+The gains in performance are immense, up to 20x speedup for most models when using compiled torchvision ops. Furthermore, it allows to run the whole pipeline solely on GPU.
 ![Fast Image Processors Performance](/images/transformers/fast_image_processors.png)
 <p class="figure-legend">Thanks <a href="https://huggingface.co/yonigozlan">Yoni Gozlan</a> for the great work!</p>
 <div class="crumbs">
+PyTorch-first lets processors assume torch/torchvision and run the whole pipeline on GPU; big per-model speedups.
+<strong>Next:</strong> how this lowers friction for contributors and downstream users.
 </div>
 <div class="crumbs">
 Uniform attention APIs enable cross-model diagnostics (e.g., PaliGemma prefix bidirectionality vs causal).
 <strong>Next:</strong> whole-model tracing for ports and regressions.
 </div>
 ### Logging entire model activations
+Because everything is PyTorch, we can easily [debug any model](https://huggingface.co/docs/transformers/internal/model_debugging_utils) when we want to add it to transformers. We now have a power-user tool for porting or adding models, that wraps a forward pass, intercepts every submodule call, and logs shapes, dtypes, and sample statistics of inputs/outputs to nested JSON.
+It just works with PyTorch models and is especially useful when aligning outputs with a reference implementation, to match our [Source of Truth guideline](#source-of-truth).
 ![Model debugger interface](/images/transformers/model_debugger.png)
 <div class="crumbs">
+Forward interception and nested JSON logging align ports to reference implementations, reinforcing "Source of Truth." <strong>Next:</strong> CUDA warmup reduces load-time without touching modeling semantics.
 </div>
 ### Cooking faster CUDA warmups
+Having a clean _external_ API allows us to work on the [true inner workings of transformers](#code-is-product). One of a few recent additions was the _CUDA warmup_ via `caching_allocator_warmup`, which dramatically improved loading times by pre-allocating GPU memory to avoid malloc bottlenecks during model loading. It can achieve a 7x speedup factor for an 8B model, or 6x for a 32B one, as you can check in [the PR](https://github.com/huggingface/transformers/pull/36380)!
 <HtmlEmbed src="transformers/warmup_demo.html" />
 It's hard to overstate how much of a lifesaver that is when you're trying to load a model as fast as possible, as it's the narrowest bottleneck for your iteration speed.
 <div class="crumbs">
+Pre-allocating GPU memory removes malloc spikes (e.g., 7× for 8B, 6× for 32B in the referenced PR).
+<strong>Next:</strong> consistent interfaces allow transformers-serve.
 </div>
 ### Transformers-serve and continuous batching
 Having all these models readily available and sharing the same interface allowed us to implement transformers-serve, a CLI tool to expose models through a standard OpenAI http API.

app/src/content/embeds/banner.html CHANGED Viewed

@@ -122,6 +122,13 @@ node.append('text')
   .attr('dy','-1.1em')
   .text(d => shortId(d.id));
 // Forces tuned for wide, short aspect
 const sim = d3.forceSimulation(graph.nodes)
   .force('link', d3.forceLink(graph.links).id(d => d.id).distance(150).strength(0.4))

   .attr('dy','-1.1em')
   .text(d => shortId(d.id));
+// Pin the llama node at center
+const llamaNode = graph.nodes.find(d => d.id === 'llama');
+if (llamaNode) {
+  llamaNode.fx = W / 2;
+  llamaNode.fy = H / 2;
+}
 // Forces tuned for wide, short aspect
 const sim = d3.forceSimulation(graph.nodes)
   .force('link', d3.forceLink(graph.links).id(d => d.id).distance(150).strength(0.4))

app/src/content/embeds/transformers/tp-plan.html CHANGED Viewed

@@ -1,24 +1,23 @@
 <pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
-base_model_tp_plan = {
-    "layers.*.self_attn.q_proj": "colwise",
-    "layers.*.self_attn.k_proj": "colwise",
-    "layers.*.self_attn.v_proj": "colwise",
-    "layers.*.self_attn.o_proj": "rowwise",
-    "layers.*.mlp.gate_proj": "colwise",
-    "layers.*.mlp.up_proj":   "colwise",
-    "layers.*.mlp.down_proj": "rowwise",
-}
-# Runtime
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-model_id = "your/model-or-local-checkpoint"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    dtype=torch.bfloat16,
-    tp_plan=base_model_tp_plan,   # <-- plan defined above
-)
-tok = AutoTokenizer.from_pretrained(model_id)
-inputs = tok("Hello", return_tensors="pt").to(model.device)
-out = model(**inputs)</code></pre>

 <pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj":   "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    # Runtime
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    model_id = "your/model-or-local-checkpoint"
+    model = AutoModelForCausalLM.from_pretrained( # <-- will automatically map to the plan defined above
+        model_id,
+        dtype=torch.bfloat16,
+    )
+    tok = AutoTokenizer.from_pretrained(model_id)
+    inputs = tok("Hello", return_tensors="pt").to(model.device)
+    out = model(**inputs)</code></pre>

app/src/content/new_article.mdx CHANGED Viewed

@@ -163,6 +163,7 @@ As you can see, we can define a new model as a _modular_ combination of fragment
 You might think "well that's just how inheritance works". The crucial difference is that we do _visibly_ what is essentially the _compiler_'s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it [all in one piece](#one-model-one-file).
 <!-- some ideas for additional hand-holding: link to the implementation of `LlamaAttention` to show it was copied (and modified), or maybe provide a git diff view between the GlmAttention and LlamaAttention implementations -->
 What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.

 You might think "well that's just how inheritance works". The crucial difference is that we do _visibly_ what is essentially the _compiler_'s job: by unrolling the inheritances, we make visible all of the modeling code, keeping it [all in one piece](#one-model-one-file).
 <!-- some ideas for additional hand-holding: link to the implementation of `LlamaAttention` to show it was copied (and modified), or maybe provide a git diff view between the GlmAttention and LlamaAttention implementations -->
 What is the consequence? When adding a model, we do not need to go over the entire modeling file. The modular (left side above) is enough.

app/src/styles/_base.css CHANGED Viewed

@@ -109,11 +109,11 @@ html {
 }
 .content-grid main mark {
-  background-color: color-mix(in srgb, var(--primary-color, #007AFF) 10%, transparent);
-  border: 1px solid color-mix(in srgb, var(--primary-color) 18%, transparent);
   color: inherit;
-  padding: 4px 6px;
-  border-radius: 4px;
   font-weight: 500;
   box-decoration-break: clone;
   -webkit-box-decoration-break: clone;

 }
 .content-grid main mark {
+  background-color: color-mix(in srgb, var(--primary-color, #007AFF) 3%, transparent);
+  border: 1px solid color-mix(in srgb, var(--primary-color) 5%, transparent);
   color: inherit;
+  padding: 1px 3px;
+  border-radius: 2px;
   font-weight: 500;
   box-decoration-break: clone;
   -webkit-box-decoration-break: clone;

src/distill.js DELETED Viewed

The diff for this file is too large to render. See raw diff

src/fragments/attention-visualizer.html DELETED Viewed

@@ -1,45 +0,0 @@
-<!-- Minimal HTML fragment: terminal-style ASCII attention masks -->
-<div style="max-width: 940px; margin: 16px 0; border:1px solid #2a2f3a; border-radius:8px; background:#0b0f19; font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', monospace; color:#e5e7eb;">
-    <div style="display:flex; align-items:center; gap:8px; padding:8px 10px; border-bottom:1px solid #1f2430; background:#111827; border-top-left-radius:8px; border-top-right-radius:8px;">
-      <span style="width:10px; height:10px; background:#ef4444; border-radius:50%; display:inline-block;"></span>
-      <span style="width:10px; height:10px; background:#f59e0b; border-radius:50%; display:inline-block;"></span>
-      <span style="width:10px; height:10px; background:#22c55e; border-radius:50%; display:inline-block;"></span>
-      <span style="margin-left:8px; font-size:12px; color:#9ca3af;">attention-mask-visualizer</span>
-    </div>
-    <div style="padding:12px 14px; overflow:auto; font-size:12.5px; line-height:1.4;">
-      <pre style="margin:0; white-space:pre; tab-size:2;">
-  ATTN MASK — GPT-2 (causal)
-  Tokens: [The, cat, sat, on, the, mat]
-  Legend: x = can attend, . = masked (future)
-           The cat sat on  the mat
-  The       x
-  cat       x   x
-  sat       x   x   x
-  on        x   x   x   x
-  the       x   x   x   x   x
-  mat       x   x   x   x   x   x
-  ATTN MASK — PaliGemma-style (bidirectional prefix + causal suffix)
-  Prefix:  [&lt;i0&gt; &lt;i1&gt; &lt;i2&gt; &lt;i3&gt; &lt;i4&gt; What is this]
-  Suffix:  [A great duck]
-  Legend: ✓ = can attend, ✗ = cannot
-             &lt;i0&gt;&lt;i1&gt;&lt;i2&gt;&lt;i3&gt;&lt;i4&gt; What  is  this  |   A   great  duck
-  &lt;i0&gt;        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  &lt;i1&gt;        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  &lt;i2&gt;        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  &lt;i3&gt;        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  &lt;i4&gt;        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  What        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  is          ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  this        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✗     ✗      ✗
-  --------------------------------------------------------------------
-  A           ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✓     ✗      ✗
-  great       ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✓     ✓      ✗
-  duck        ✓   ✓   ✓   ✓   ✓    ✓     ✓    ✓        ✓     ✓      ✓
-      </pre>
-    </div>
-  </div>

src/fragments/d3-graph.html DELETED Viewed

@@ -1,12 +0,0 @@
-<div class="interactive-demo">
-    <div class="demo-header">
-        <h3>🔗 Model Dependency Graph</h3>
-    </div>
-    <div class="demo-content">
-        <iframe src="static/d3_dependency_graph.html" width="100%" height="600px" frameborder="0" style="border-radius: 8px; background: white;"></iframe>
-    </div>
-    <div class="demo-footer">
-        Interactive dependency graph showing real relationships between Transformers models. 🟡 Base models (HuggingFace logo), 🔵 Derived modular models. Click and drag to explore!
-    </div>
-</div>

src/fragments/dependency-graph.html DELETED Viewed

@@ -1,6 +0,0 @@
-<iframe
-src="https://molbap-dependencies-1.hf.space"
-style="width:100%; height:680px; border:0"
-allow="clipboard-read; clipboard-write; fullscreen"
-referrerpolicy="no-referrer-when-downgrade"
-></iframe>

src/fragments/glm-compare.html DELETED Viewed

@@ -1,149 +0,0 @@
-<div class="code-compare" style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin: 1.5rem 0;">
-    <div class="code-column" style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
-        <div class="code-header" style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
-            modular_glm.py
-        </div>
-        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em;"><code class="language-python">class GlmMLP(Phi3MLP):
-    pass
-class GlmAttention(LlamaAttention):
-    def __init__(self, config, layer_idx=None):
-        super().__init__(config, layer_idx)
-        self.o_proj = nn.Linear(
-            config.num_attention_heads * self.head_dim,
-            config.hidden_size,
-            bias=False
-        )
-class GlmForCausalLM(LlamaForCausalLM):
-    pass</code></pre>
-    </div>
-    <div class="code-column" style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
-        <div class="code-header" style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
-            modeling_glm.py (auto-expanded)
-        </div>
-        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em; max-height: 400px;"><code class="language-python">class GlmMLP(nn.Module):
-    def __init__(self, config):
-        super().__init__()
-        self.config = config
-        self.gate_up_proj = nn.Linear(
-            config.hidden_size,
-            2 * config.intermediate_size,
-            bias=False
-        )
-        self.down_proj = nn.Linear(
-            config.intermediate_size,
-            config.hidden_size,
-            bias=False
-        )
-        self.activation_fn = ACT2FN[config.hidden_act]
-    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
-        up_states = self.gate_up_proj(hidden_states)
-        gate, up_states = up_states.chunk(2, dim=-1)
-        up_states = up_states * self.activation_fn(gate)
-        return self.down_proj(up_states)
-class GlmAttention(nn.Module):
-    """Multi-headed attention from 'Attention Is All You Need' paper"""
-    def __init__(self, config: GlmConfig, layer_idx: Optional[int] = None):
-        super().__init__()
-        self.config = config
-        self.layer_idx = layer_idx
-        self.head_dim = getattr(
-            config, "head_dim",
-            config.hidden_size // config.num_attention_heads
-        )
-        self.num_key_value_groups = (
-            config.num_attention_heads // config.num_key_value_heads
-        )
-        self.scaling = self.head_dim**-0.5
-        self.attention_dropout = config.attention_dropout
-        self.is_causal = True
-        self.q_proj = nn.Linear(
-            config.hidden_size,
-            config.num_attention_heads * self.head_dim,
-            bias=config.attention_bias
-        )
-        self.k_proj = nn.Linear(
-            config.hidden_size,
-            config.num_key_value_heads * self.head_dim,
-            bias=config.attention_bias
-        )
-        self.v_proj = nn.Linear(
-            config.hidden_size,
-            config.num_key_value_heads * self.head_dim,
-            bias=config.attention_bias
-        )
-        self.o_proj = nn.Linear(
-            config.num_attention_heads * self.head_dim,
-            config.hidden_size,
-            bias=False
-        )
-    def forward(
-        self,
-        hidden_states: torch.Tensor,
-        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
-        attention_mask: Optional[torch.Tensor],
-        past_key_value: Optional[Cache] = None,
-        cache_position: Optional[torch.LongTensor] = None,
-        **kwargs: Unpack[FlashAttentionKwargs],
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-        input_shape = hidden_states.shape[:-1]
-        hidden_shape = (*input_shape, -1, self.head_dim)
-        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
-        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
-        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
-        cos, sin = position_embeddings
-        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
-        if past_key_value is not None:
-            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
-            key_states, value_states = past_key_value.update(
-                key_states, value_states, self.layer_idx, cache_kwargs
-            )
-        attention_interface: Callable = eager_attention_forward
-        if self.config._attn_implementation != "eager":
-            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
-        attn_output, attn_weights = attention_interface(
-            self, query_states, key_states, value_states,
-            attention_mask, dropout=0.0 if not self.training else self.attention_dropout,
-            scaling=self.scaling, **kwargs,
-        )
-        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
-        attn_output = self.o_proj(attn_output)
-        return attn_output, attn_weights
-@use_kernel_forward_from_hub("RMSNorm")
-class GlmRMSNorm(nn.Module):
-    def __init__(self, hidden_size, eps=1e-6):
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-# ... (many more classes and functions would follow)</code></pre>
-    </div>
-</div>
-<p style="text-align: center; font-style: italic; color: #6c757d; margin-top: 1rem;">
-    <strong>Left:</strong> Clean modular definition with inheritance.
-    <strong>Right:</strong> Auto-expanded version with all inherited functionality visible.
-</p>

src/fragments/loc-growth.html DELETED Viewed

@@ -1,6 +0,0 @@
-<iframe
-src="https://molbap-loc-1.hf.space"
-style="width:100%; height:680px; border:0"
-allow="clipboard-read; clipboard-write; fullscreen"
-referrerpolicy="no-referrer-when-downgrade"
-></iframe>

src/fragments/memory-profiler.html DELETED Viewed

@@ -1,16 +0,0 @@
-<div style="border: 1px solid #e2e8f0; border-radius: 8px; background: white; margin: 1.5rem 0;">
-    <div style="padding: 1rem; border-bottom: 1px solid #e2e8f0; background: #f8f9fa;">
-        <h4 style="margin: 0 0 0.5rem 0; color: #495057;">🚀 CUDA Warmup Efficiency Benchmark</h4>
-        <p style="margin: 0; font-size: 0.9em; color: #6c757d;">
-            Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the caching_allocator_warmup function.
-        </p>
-    </div>
-    <div style="padding: 1rem;">
-        <iframe src=https://molbap-cuda-warmup-transformers.hf.space width=100% height=800px frameborder=0 style="border-radius: 8px; background: white;"></iframe>
-    </div>
-    <div style="padding: 1rem; border-top: 1px solid #e2e8f0; background: #f8f9fa; font-size: 0.9em; color: #6c757d;">
-        Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the <code>caching_allocator_warmup</code> function at <code>transformers/src/transformers/modeling_utils.py:6186</code>. This interactive tool loads models twice - once with warmup disabled and once with warmup enabled - to demonstrate the significant loading time improvements.
-    </div>
-</div>

src/fragments/model-timeline.html DELETED Viewed

@@ -1,6 +0,0 @@
-    <iframe
-  src="https://molbap-timeline-1.hf.space"
-  style="width:100%; height:680px; border:0"
-  allow="clipboard-read; clipboard-write; fullscreen"
-  referrerpolicy="no-referrer-when-downgrade"
-></iframe>

src/fragments/model-visualisation.html DELETED Viewed

The diff for this file is too large to render. See raw diff

src/fragments/terminal.html DELETED Viewed

@@ -1,43 +0,0 @@
-<div style="background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 8px; padding: 1rem; margin: 1.5rem 0;">
-    <h4 style="margin-top: 0; color: #495057;">Interactive Terminal</h4>
-    <div style="background: #2d3748; color: #e2e8f0; padding: 1rem; border-radius: 6px; font-family: 'Consolas', 'Monaco', monospace;">
-        <div style="margin-bottom: 1rem;">
-            <input type="text"
-                   id="terminal-input"
-                   placeholder="python -c 'import torch; print(torch.__version__)'"
-                   style="width: calc(100% - 80px); padding: 0.5rem; background: #1a202c; border: 1px solid #4a5568; color: #e2e8f0; border-radius: 4px;">
-            <button id="terminal-run"
-                    style="width: 70px; padding: 0.5rem; margin-left: 8px; background: #3182ce; color: white; border: none; border-radius: 4px; cursor: pointer;">Run</button>
-        </div>
-        <pre id="terminal-output" style="background: #1a202c; padding: 1rem; border-radius: 4px; min-height: 100px; margin: 0; overflow-x: auto;">$ Ready to run commands...</pre>
-    </div>
-    <p style="font-size: 0.9em; color: #6c757d; margin-top: 0.5rem;">
-        <em>Note: This is a simulated terminal. In the original Gradio app, this would execute real Python commands with proper security restrictions.</em>
-    </p>
-</div>
-<script>
-document.addEventListener('DOMContentLoaded', function() {
-    const input = document.getElementById('terminal-input');
-    const button = document.getElementById('terminal-run');
-    const output = document.getElementById('terminal-output');
-    function runCommand() {
-        const command = input.value.trim();
-        if (!command) return;
-        // Simulate command execution
-        output.textContent = `$ ${command}\nSimulated output for: ${command}\n\n` +
-                           `This would execute the command in the original app.\n` +
-                           `Example outputs:\n` +
-                           `- torch version: 2.0.1+cu117\n` +
-                           `- import checks: Success\n` +
-                           `- memory info: Available`;
-    }
-    button.addEventListener('click', runCommand);
-    input.addEventListener('keypress', function(e) {
-        if (e.key === 'Enter') runCommand();
-    });
-});
-</script>

src/fragments/tp-plan.html DELETED Viewed

@@ -1,24 +0,0 @@
-<pre><code class="language-python"># In the model's config (example: ERNIE 4.5-style decoder blocks)
-base_model_tp_plan = {
-    "layers.*.self_attn.q_proj": "colwise",
-    "layers.*.self_attn.k_proj": "colwise",
-    "layers.*.self_attn.v_proj": "colwise",
-    "layers.*.self_attn.o_proj": "rowwise",
-    "layers.*.mlp.gate_proj": "colwise",
-    "layers.*.mlp.up_proj":   "colwise",
-    "layers.*.mlp.down_proj": "rowwise",
-}
-# Runtime
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-model_id = "your/model-or-local-checkpoint"
-model = AutoModelForCausalLM.from_pretrained(
-    model_id,
-    dtype=torch.bfloat16,
-    tp_plan=base_model_tp_plan,   # <-- plan defined above
-)
-tok = AutoTokenizer.from_pretrained(model_id)
-inputs = tok("Hello", return_tensors="pt").to(model.device)
-out = model(**inputs)</code></pre>

src/fragments/warmup_demo.html DELETED Viewed

@@ -1,398 +0,0 @@
-<style>
-    .warmup-demo body {
-        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
-        margin: 0;
-        padding: 20px;
-        background-color: #f5f5f5;
-    }
-    .warmup-demo .container {
-        max-width: 1200px;
-        margin: 0 auto;
-        background: white;
-        border-radius: 12px;
-        padding: 30px;
-        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
-    }
-    .warmup-demo h1 {
-        text-align: center;
-        color: #333;
-        margin-bottom: 10px;
-    }
-    .warmup-demo .subtitle {
-        text-align: center;
-        color: #666;
-        margin-bottom: 30px;
-        font-size: 16px;
-    }
-    .warmup-demo .demo-container {
-        display: flex;
-        gap: 40px;
-        margin-bottom: 30px;
-    }
-    .warmup-demo .side {
-        flex: 1;
-        border: 2px solid #ddd;
-        border-radius: 8px;
-        padding: 20px;
-        background: #fafafa;
-    }
-    .warmup-demo .side h2 {
-        text-align: center;
-        margin-top: 0;
-        color: #333;
-    }
-    .warmup-demo .no-warmup h2 {
-        color: #d63384;
-    }
-    .warmup-demo .with-warmup h2 {
-        color: #198754;
-    }
-    .warmup-demo .memory-area {
-        height: 400px;
-        border: 2px dashed #ccc;
-        border-radius: 6px;
-        padding: 10px;
-        margin: 20px 0;
-        background: #fff;
-        position: relative;
-        overflow: hidden;
-    }
-    .warmup-demo .layer-box {
-        width: 80px;
-        height: 30px;
-        border: 2px solid #666;
-        border-radius: 4px;
-        margin: 3px;
-        display: inline-block;
-        position: relative;
-        background: #fff;
-        transition: all 0.3s ease;
-    }
-    .warmup-demo .layer-box.allocating {
-        background: #e9ecef;
-        border-color: #adb5bd;
-    }
-    .warmup-demo .layer-box.allocating::after {
-        content: "malloc";
-        position: absolute;
-        top: 50%;
-        left: 50%;
-        transform: translate(-50%, -50%);
-        font-size: 10px;
-        color: #666;
-        font-weight: bold;
-    }
-    .warmup-demo .layer-box.loaded {
-        background: #d1e7dd;
-        border-color: #198754;
-    }
-    .warmup-demo .layer-box.loaded::after {
-        content: "data";
-        position: absolute;
-        top: 50%;
-        left: 50%;
-        transform: translate(-50%, -50%);
-        font-size: 10px;
-        color: #198754;
-        font-weight: bold;
-    }
-    .warmup-demo .warmup-container {
-        width: 100%;
-        height: 60px;
-        border: 3px solid #666;
-        border-radius: 6px;
-        margin-bottom: 20px;
-        background: #fff;
-        position: relative;
-        overflow: hidden;
-    }
-    .warmup-demo .warmup-container.allocated {
-        border-color: #0d6efd;
-        background: #e7f1ff;
-    }
-    .warmup-demo .warmup-container::before {
-        content: "Pre-allocated Memory Pool";
-        position: absolute;
-        top: 50%;
-        left: 50%;
-        transform: translate(-50%, -50%);
-        font-size: 14px;
-        color: #666;
-        font-weight: bold;
-        z-index: 1;
-    }
-    .warmup-demo .warmup-container.allocated::before {
-        color: #0d6efd;
-    }
-    .warmup-demo .warmup-fill {
-        height: 100%;
-        background: linear-gradient(90deg, #198754, #20c997);
-        width: 0%;
-        transition: width 0.5s ease;
-        border-radius: 3px;
-        position: relative;
-        z-index: 2;
-    }
-    .warmup-demo .warmup-fill::after {
-        content: "Layer Data Loading";
-        position: absolute;
-        top: 50%;
-        left: 50%;
-        transform: translate(-50%, -50%);
-        font-size: 12px;
-        color: white;
-        font-weight: bold;
-        white-space: nowrap;
-    }
-    .warmup-demo .timing {
-        text-align: center;
-        font-size: 24px;
-        font-weight: bold;
-        margin: 15px 0;
-        min-height: 30px;
-    }
-    .warmup-demo .no-warmup .timing {
-        color: #d63384;
-    }
-    .warmup-demo .with-warmup .timing {
-        color: #198754;
-    }
-    .warmup-demo .controls {
-        text-align: center;
-        margin: 30px 0;
-    }
-    .warmup-demo .btn {
-        background: #0d6efd;
-        color: white;
-        border: none;
-        padding: 12px 24px;
-        border-radius: 6px;
-        font-size: 16px;
-        cursor: pointer;
-        margin: 0 10px;
-        transition: background 0.3s ease;
-    }
-    .warmup-demo .btn:hover {
-        background: #0b5ed7;
-    }
-    .warmup-demo .btn:disabled {
-        background: #6c757d;
-        cursor: not-allowed;
-    }
-    .warmup-demo .description {
-        background: #f8f9fa;
-        padding: 15px;
-        border-radius: 6px;
-        margin-top: 15px;
-        font-size: 14px;
-        line-height: 1.5;
-    }
-    .warmup-demo .phase-indicator {
-        font-size: 14px;
-        color: #666;
-        text-align: center;
-        margin-top: 10px;
-        min-height: 20px;
-    }
-    .warmup-demo .layer-counter {
-        text-align: center;
-        font-size: 16px;
-        color: #495057;
-        margin: 10px 0;
-    }
-</style>
-<div class="warmup-demo">
-    <div class="container">
-        <p class="subtitle">Mem allocation patterns during model loading</p>
-        <div class="controls">
-            <button class="btn" id="startBtn" onclick="startDemo()">Start Animation</button>
-            <button class="btn" id="resetBtn" onclick="resetDemo()">Reset</button>
-        </div>
-        <div class="demo-container">
-            <div class="side no-warmup">
-                <h4 data-no-toc>❌ Without Warmup</h4>
-                <div class="timing" id="noWarmupTime">0.00s</div>
-                <div class="layer-counter" id="noWarmupCounter">Layers loaded: 0/10</div>
-                <div class="phase-indicator" id="noWarmupPhase"></div>
-                <div class="memory-area" id="noWarmupArea"></div>
-                <div class="description">
-                    <strong>Individual Allocations:</strong><br>
-                    Each model layer triggers a separate cudaMalloc() call, creating memory fragmentation and allocation overhead.
-                    <br><br>
-                    📦 <strong>Grey "malloc"</strong> = Memory allocation overhead<br>
-                    ✅ <strong>Green "data"</strong> = Actual layer data loading
-                </div>
-            </div>
-            <div class="side with-warmup">
-                <h4 data-no-toc>✅ With Warmup</h4>
-                <div class="timing" id="warmupTime">0.00s</div>
-                <div class="layer-counter" id="warmupCounter">Layers loaded: 0/10</div>
-                <div class="phase-indicator" id="warmupPhase"></div>
-                <div class="memory-area" id="warmupArea">
-                    <div class="warmup-container" id="warmupContainer">
-                        <div class="warmup-fill" id="warmupFill"></div>
-                    </div>
-                    <div id="warmupLayers"></div>
-                </div>
-                <div class="description">
-                    <strong>Pre-allocated Pool:</strong><br>
-                    The warmup function calculates total memory needed and makes ONE large allocation. Subsequent layers load directly into this pool, eliminating malloc overhead.
-                    <br><br>
-                    🔵 <strong>Blue container</strong> = Single large malloc (warmup)<br>
-                    🟢 <strong>Green progress bar</strong> = Layer data loading (no malloc needed)
-                </div>
-            </div>
-        </div>
-    </div>
-</div>
-<script>
-    let animationSpeed = 1 / 2.4;
-    let isRunning = false;
-    const totalLayers = 10;
-    function startDemo() {
-        if (isRunning) return;
-        isRunning = true;
-        document.getElementById('startBtn').disabled = true;
-        document.getElementById('resetBtn').disabled = true;
-        Promise.all([
-            animateNoWarmup(),
-            animateWithWarmup()
-        ]).then(() => {
-            isRunning = false;
-            document.getElementById('startBtn').disabled = false;
-            document.getElementById('resetBtn').disabled = false;
-        });
-    }
-    function resetDemo() {
-        if (isRunning) return;
-        document.getElementById('noWarmupArea').innerHTML = '';
-        document.getElementById('warmupLayers').innerHTML = '';
-        document.getElementById('warmupFill').style.width = '0%';
-        document.getElementById('warmupContainer').classList.remove('allocated');
-        document.getElementById('noWarmupTime').textContent = '0.00s';
-        document.getElementById('warmupTime').textContent = '0.00s';
-        document.getElementById('noWarmupCounter').textContent = 'Layers loaded: 0/10';
-        document.getElementById('warmupCounter').textContent = 'Layers loaded: 0/10';
-        document.getElementById('noWarmupPhase').textContent = '';
-        document.getElementById('warmupPhase').textContent = '';
-    }
-    async function animateNoWarmup() {
-        const container = document.getElementById('noWarmupArea');
-        const timeEl = document.getElementById('noWarmupTime');
-        const counterEl = document.getElementById('noWarmupCounter');
-        const phaseEl = document.getElementById('noWarmupPhase');
-        let currentTime = 0;
-        const baseDelay = 200 / animationSpeed;
-        phaseEl.textContent = 'Loading model layers...';
-        for (let i = 0; i < totalLayers; i++) {
-            const layerBox = document.createElement('div');
-            layerBox.className = 'layer-box';
-            container.appendChild(layerBox);
-            await sleep(baseDelay * 0.3);
-            layerBox.classList.add('allocating');
-            currentTime += 0.08;
-            timeEl.textContent = currentTime.toFixed(2) + 's';
-            await sleep(baseDelay * 0.7);
-            layerBox.classList.remove('allocating');
-            layerBox.classList.add('loaded');
-            currentTime += 0.12;
-            timeEl.textContent = currentTime.toFixed(2) + 's';
-            counterEl.textContent = `Layers loaded: ${i + 1}/${totalLayers}`;
-        }
-        phaseEl.textContent = 'Complete!';
-    }
-    async function animateWithWarmup() {
-        const container = document.getElementById('warmupLayers');
-        const timeEl = document.getElementById('warmupTime');
-        const counterEl = document.getElementById('warmupCounter');
-        const phaseEl = document.getElementById('warmupPhase');
-        const warmupContainer = document.getElementById('warmupContainer');
-        const warmupFill = document.getElementById('warmupFill');
-        let currentTime = 0;
-        const baseDelay = 200 / animationSpeed;
-        phaseEl.textContent = 'Warming up allocator...';
-        await sleep(baseDelay * 2);
-        warmupContainer.classList.add('allocated');
-        currentTime += 0.3;
-        timeEl.textContent = currentTime.toFixed(2) + 's';
-        phaseEl.textContent = 'Loading model layers...';
-        for (let i = 0; i < totalLayers; i++) {
-            const layerBox = document.createElement('div');
-            layerBox.className = 'layer-box loaded';
-            layerBox.style.width = '40px';
-            layerBox.style.height = '20px';
-            container.appendChild(layerBox);
-            const progress = ((i + 1) / totalLayers) * 100;
-            warmupFill.style.width = progress + '%';
-            await sleep(baseDelay * 0.5);
-            currentTime += 0.08;
-            timeEl.textContent = currentTime.toFixed(2) + 's';
-            counterEl.textContent = `Layers loaded: ${i + 1}/${totalLayers}`;
-        }
-        phaseEl.textContent = 'Complete!';
-    }
-    function sleep(ms) {
-        return new Promise(resolve => setTimeout(resolve, ms));
-    }
-</script>

src/index.js DELETED Viewed

@@ -1,21 +0,0 @@
-// Main JavaScript file for Scaling Insanity
-import './style.css';
-// Import any additional functionality
-console.log('blog loaded');
-// Add any custom JavaScript functionality here
-document.addEventListener('DOMContentLoaded', function() {
-    // Initialize syntax highlighting for code blocks
-    if (window.hljs) {
-        hljs.highlightAll();
-    }
-    // Initialize any interactive components
-    initializeInteractiveComponents();
-});
-function initializeInteractiveComponents() {
-    // This will be expanded as we add interactive components
-    console.log('Interactive components initialized');
-}

src/style.css DELETED Viewed

@@ -1,741 +0,0 @@
-/* style.css - Scaling Insanity */
-/* Import ultrascale-playbook base styles and add transformers-specific styling */
-/* Define colors */
-:root {
-    --distill-gray: rgb(107, 114, 128);
-    --distill-gray-light: rgb(185, 185, 185);
-    --distill-gray-lighter: rgb(228, 228, 228);
-    --distill-gray-lightest: rgb(245, 245, 245);
-    --distill-blue: #007BFF;
-}
-/* Container for the controls */
-[id^="plot-"] {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    gap: 15px; /* Adjust the gap between controls as needed */
-}
-[id^="plot-"] figure {
-    margin-bottom: 0px;
-    margin-top: 0px;
-    padding: 0px;
-}
-.plotly_caption {
-    font-style: italic;
-    margin-top: 10px;
-}
-.plotly_controls {
-    display: flex;
-    flex-wrap: wrap;
-    flex-direction: row;
-    justify-content: center;
-    align-items: flex-start;
-    gap: 30px;
-}
-.plotly_input_container {
-    display: flex;
-    align-items: center;
-    flex-direction: column;
-    gap: 10px;
-}
-/* Style for the select dropdown */
-.plotly_input_container > select {
-    padding: 2px 4px;
-    /* border: 1px solid #ccc; */
-    line-height: 1.5em;
-    text-align: center;
-    border-radius: 4px;
-    font-size: 12px;
-    background-color: var(--distill-gray-lightest);
-    outline: none;
-}
-/* Style for the range input */
-.plotly_slider {
-    display: flex;
-    align-items: center;
-    gap: 10px;
-}
-.plotly_slider > input[type="range"] {
-    -webkit-appearance: none;
-    height: 2px;
-    background: var(--distill-gray-light);
-    border-radius: 5px;
-    outline: none;
-}
-.plotly_slider > span {
-    font-size: 14px;
-    line-height: 1.6em;
-    min-width: 16px;
-}
-.plotly_slider > input[type="range"]::-webkit-slider-thumb {
-    -webkit-appearance: none;
-    appearance: none;
-    width: 18px;
-    height: 18px;
-    border-radius: 50%;
-    background: var(--distill-blue);
-    cursor: pointer;
-}
-.plotly_slider > input[type="range"]::-moz-range-thumb {
-    width: 18px;
-    height: 18px;
-    border-radius: 50%;
-    background: var(--distill-blue);
-    cursor: pointer;
-}
-/* Style for the labels */
-.plotly_input_container > label {
-    font-size: 14px;
-    font-weight: bold;
-}
-.main-plot-container {
-    margin-top: 21px;
-    margin-bottom: 35px;
-}
-.main-plot-container > figure {
-    display: block !important;
-    /* Let this be handled by graph-container */
-    margin-bottom: 0px;
-    margin-top: 0px;
-}
-.main-plot-container > div {
-    display: none !important;
-}
-@media (min-width: 768px) {
-    .main-plot-container > figure {
-        display: none !important;
-    }
-    .main-plot-container > div {
-        display: flex !important;
-    }
-}
-d-byline .byline {
-  grid-template-columns: 1fr;
-  grid-column: text;
-  font-size: 0.9rem;
-  line-height: 1.8em;
-}
-@media (min-width: 768px) {
-  d-byline .byline {
-    grid-template-columns: 5fr 1fr 1fr;
-  }
-}
-#title-plot {
-    margin-top: 0px;
-    margin-bottom: 0px;
-}
-d-contents > nav a.active {
-    text-decoration: underline;
-}
-@media (max-width: 1199px) {
-    d-contents {
-        display: none;
-        background: white;
-        justify-self: start;
-        align-self: start;
-        padding-bottom: 0.5em;
-        margin-bottom: 1em;
-        padding-left: 0.25em;
-        border-bottom: 1px solid rgba(0, 0, 0, 0.1);
-        border-bottom-width: 1px;
-        border-bottom-style: solid;
-        border-bottom-color: rgba(0, 0, 0, 0.1);
-        overflow-y: scroll;
-        height: calc(100vh - 40px);
-        scrollbar-width: none;
-        z-index: -100;
-    }
-}
-d-contents a:hover {
-    border-bottom: none;
-}
-toc-title {
-    font-weight: bold;
-    font-size: 1.2em;
-    color: #333;
-}
-toggle-icon {
-    transition: transform 0.3s;
-}
-toggle-icon.collapsed {
-    transform: rotate(90deg);
-}
-.toc-content {
-    margin-top: 15px;
-    overflow: hidden;
-    /* max-height: 1000px; */
-    transition: max-height 0.3s ease-out;
-}
-.toc-content.collapsed {
-    max-height: 0;
-    margin-top: 0;
-}
-@media (min-width: 1200px) {
-    d-article {
-        /* Ensure d-article does not prevent sticky positioning */
-        overflow: visible;
-    }
-    d-contents {
-        align-self: start;
-        background: white;
-        grid-column-start: 1 !important;
-        grid-column-end: 4 !important;
-        grid-row: auto / span 6;
-        justify-self: end;
-        margin-top: 0em;
-        padding-right: 3em;
-        padding-left: 2em;
-        /* border-right: 1px solid rgba(0, 0, 0, 0.1);
-        border-right-width: 1px;
-        border-right-style: solid;
-        border-right-color: rgba(0, 0, 0, 0.1); */
-        position: -webkit-sticky; /* For Safari */
-        position: sticky;
-        top: 10px; /* Adjust this value if needed */
-        overflow-y: auto;
-        height: calc(100vh - 40px);
-        scrollbar-width: none;
-        transition: max-height 0.3s ease-out;
-        z-index: -100;
-    }
-}
-d-contents nav h3 {
-    margin-top: 0;
-    margin-bottom: 1em;
-}
-d-contents nav div div {
-    color: rgba(0, 0, 0, 0.8);
-    font-weight: bold;
-}
-d-contents nav a {
-    color: rgba(0, 0, 0, 0.8);
-    border-bottom: none;
-    text-decoration: none;
-}
-d-contents li {
-    list-style-type: none;
-}
-d-contents ul, d-article d-contents ul {
-    padding-left: 1em;
-}
-d-contents nav ul li {
-    margin-bottom: .25em;
-}
-d-contents nav a:hover {
-    text-decoration: underline solid rgba(0, 0, 0, 0.6);
-}
-d-contents nav ul {
-    margin-top: 0;
-    margin-bottom: 6px;
-}
-d-contents nav > div {
-    display: block;
-    outline: none;
-    margin-bottom: 0.5em;
-}
-d-contents nav > div > a {
-    font-size: 13px;
-    font-weight: 600;
-}
-d-article aside {
-    margin-bottom: 1em;
-}
-d-article img {
-    max-width: 100%;
-}
-@media (min-width: 768px) {
-    d-article aside {
-        margin-bottom: 0;
-    }
-}
-d-contents nav > div > a:hover,
-d-contents nav > ul > li > a:hover {
-    text-decoration: none;
-}
-.note-box {
-    background-color: #f6f8fa;
-    border-left: 4px solid #444444;
-    padding: 1rem;
-    margin: 1rem 0;  /* Keep this modest margin */
-    border-radius: 6px;
-    /* Add this to ensure the box only takes up needed space */
-    display: inline-block;
-}
-.note-box-title {
-    margin: 0;
-    color: #444444;
-    font-weight: 600;
-    font-size: 1em;
-}
-.note-box-content {
-    margin-top: 0.5rem;
-    margin-bottom: 0;  /* Ensure no bottom margin */
-    color: #24292f;
-    font-size: 0.9em;
-    line-height: 1.5em;
-}
-/* For dark mode support */
-@media (prefers-color-scheme: dark) {
-    .note-box {
-        background-color: #1c1c1c;
-        border-left-color: #888888;
-    }
-    .note-box-title {
-        color: #888888;
-    }
-    .note-box-content {
-        color: #d4d4d4;
-    }
-}
-d-article {
-    font-size: 1.0em;
-}
-.figure-legend {
-    font-size: 0.9em;
-    font-style: italic;
-    color: var(--distill-gray);
-    line-height: 1.5em;
-}
-d-code {
-    font-size: 12px;
-}
-.large-image-background {
-        width: 100vw;
-        padding-top: 10px;
-        padding-bottom: 10px;
-        margin-left: calc(-50vw + 50%);
-        margin-right: calc(-50vw + 50%);
-        background: white;
-        height: fit-content; /* This will make it match the image height */
-        display: flex;
-        justify-content: center; /* This will center your image */
-}
-.large-image-background-transparent {
-    /* width: 100vw; */
-    padding-top: 10px;
-    padding-bottom: 10px;
-    /* margin-left: calc(-50vw + 50%); */
-    margin-left:-100px;
-    margin-right: -100px;
-    /* margin-right: calc(-50vw + 50%); */
-    /* background: white; */
-    height: fit-content; /* This will make it match the image height */
-    display: flex;
-    justify-content: center; /* This will center your image */
-}
-.boxed-image {
-    padding: 0.5rem;
-    background: white;
-    border-radius: 12px;
-    border: 1px solid #e5e7eb;
-    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
-}
-d-article li {
-    margin-bottom: 0.0em;
-}
-d-article ul ul {
-    margin-bottom: 0.0em;
-}
-d-article ol ol {
-    margin-bottom: 0.0em;
-}
-d-article hr {
-    grid-column: text;
-}
-/* Memory visualization */
-#graph-all {
-    min-width: 500px;
-    margin-right: 10px;
-    margin-bottom: 2rem;
-    padding: 0.5rem;
-    background: #f9fafb;
-    border-radius: 12px;
-    border: 1px solid #e5e7eb;
-    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
-}
-/* Main container styles */
-#controls {
-    max-width: 1200px;
-    /* margin: 2rem auto; */
-    margin-bottom: 2rem;
-    margin-left: 10px;
-    padding: 0.6rem;
-    background: #f9fafb;
-    border-radius: 12px;
-    border: 1px solid #e5e7eb;
-    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
-}
-/* Grid layout */
-#controls {
-    display: grid;
-    grid-template-columns: 1fr 1fr;
-    /* gap: 2rem; */
-}
-/* Cell styles */
-.cell {
-    margin-bottom: 0.2rem;
-}
-/* Label styles */
-label {
-    display: block;
-    /* margin-bottom: 0.5rem; */
-    font-size: 0.8rem;
-    font-weight: 500;
-    color: #374151;
-}
-/* Input container for range + number combination */
-.input-container {
-    display: flex;
-    gap: 1rem;
-    align-items: center;
-}
-/* Range input styling */
-input[type="range"] {
-    flex: 1;
-    height: 6px;
-    background: #e5e7eb;
-    border-radius: 3px;
-    appearance: none;
-    outline: none;
-}
-input[type="range"]::-webkit-slider-thumb {
-    appearance: none;
-    width: 16px;
-    height: 16px;
-    background: #3b82f6;
-    border-radius: 50%;
-    cursor: pointer;
-    transition: background 0.15s ease;
-}
-input[type="range"]::-webkit-slider-thumb:hover {
-    background: #2563eb;
-}
-/* Number input styling */
-input[type="number"] {
-    width: 80px;
-    padding: 0.5rem;
-    border: 1px solid #e5e7eb;
-    border-radius: 6px;
-    font-size: 0.9rem;
-    color: #374151;
-}
-/* Select styling */
-select {
-    width: 100%;
-    padding: 0.5rem;
-    border: 1px solid #e5e7eb;
-    border-radius: 6px;
-    background: white;
-    font-size: 0.9rem;
-    color: #374151;
-    cursor: pointer;
-}
-/* Checkbox styling */
-input[type="checkbox"] {
-    width: 1.2rem;
-    height: 1.2rem;
-    margin-right: 0.5rem;
-    border: 2px solid #e5e7eb;
-    border-radius: 4px;
-    cursor: pointer;
-}
-/* Column specific styles */
-.column-1 {
-    padding-right: 0.5rem;
-}
-.column-2 {
-    padding-left: 0.5rem;
-}
-/* Checkbox container */
-.checkbox-container {
-    display: flex;
-    align-items: center;
-    margin-bottom: 1rem;
-}
-/* Memory visualization styles */
-.memory-block {
-    background: #fff;
-    border-radius: 8px;
-    padding: 1rem;
-    margin-bottom: 1rem;
-    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
-}
-.memory-title {
-    font-size: 1.1rem;
-    font-weight: 500;
-    color: #374151;
-    margin-bottom: 0.5rem;
-}
-.memory-value {
-    font-size: 1.5rem;
-    font-weight: 600;
-    color: #3b82f6;
-}
-/* Responsive adjustments */
-@media (max-width: 768px) {
-    #controls {
-        grid-template-columns: 1fr;
-        padding: 1rem;
-    }
-    .column-1, .column-2 {
-        padding: 0;
-    }
-}
-/* Hover states and transitions */
-input:hover, select:hover {
-    border-color: #3b82f6;
-}
-input:focus, select:focus {
-    border-color: #2563eb;
-    outline: none;
-    box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.1);
-}
-/* Add smooth transitions */
-input, select, button {
-    transition: all 0.15s ease;
-}
-/* Preset dropdown special styling */
-select[name="presets"] {
-    background-color: #f3f4f6;
-    font-weight: 500;
-}
-/* Memory graph enhancements */
-.activation-memory {
-    background: #dbeafe;
-    padding: 1rem;
-    border-radius: 8px;
-    margin-bottom: 1rem;
-}
-.gradient-memory {
-    background: #ede9fe;
-    padding: 1rem;
-    border-radius: 8px;
-}
-.order-button-second {
-            background: linear-gradient(135deg, #6DB4C4, #D4A5B8);
-            color: white;
-            font-size: 18px;
-            font-weight: 600;
-            padding: 20px 20px;
-            border: none;
-            border-radius: 12px;
-            cursor: pointer;
-            text-transform: uppercase;
-            letter-spacing: 1px;
-            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
-            transition: all 0.3s ease;
-            position: relative;
-            overflow: hidden;
-        }
-.order-button-second:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
-}
-.order-button:active {
-    transform: translateY(0);
-    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-}
-.order-button-second::before {
-    content: '';
-    position: absolute;
-    top: 0;
-    left: -100%;
-    width: 100%;
-    height: 100%;
-    background: linear-gradient(135deg, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0));
-    transition: left 0.5s ease;
-}
-.order-button-second:hover::before {
-    left: 100%;
-}
-.order-button {
-            background: linear-gradient(135deg, #6DB4C4, #D4A5B8);
-            color: white;
-            font-size: 18px;
-            font-weight: 600;
-            padding: 16px 32px;
-            border: none;
-            border-radius: 12px;
-            cursor: pointer;
-            text-transform: uppercase;
-            letter-spacing: 1px;
-            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
-            transition: all 0.3s ease;
-            position: relative;
-            overflow: hidden;
-        }
-.order-button:hover {
-    transform: translateY(-2px);
-    box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
-}
-.order-button:active {
-    transform: translateY(0);
-    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
-}
-.order-button::before {
-    content: '';
-    position: absolute;
-    top: 0;
-    left: -100%;
-    width: 100%;
-    height: 100%;
-    background: linear-gradient(135deg, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0));
-    transition: left 0.5s ease;
-}
-.order-button:hover::before {
-    left: 100%;
-}
-.order-button-container-second {
-    /* display: flex; */
-    justify-content: center;
-    margin: 0px 0;
-}
-.order-button-container {
-    display: flex;
-    justify-content: center;
-    margin: 0px 0 40px 0;
-}
-d-article img {
-    width: 100%!important;
-}
-iframe, .js-plotly-plot {
-    width: 100%!important;
-    margin-bottom: 20px;
-}
-.modebar-container {
-    display: none;
-}
-#graph-container {
-    display: grid; grid-template-columns: 1fr 1fr; align-items: center;
-}
-@media (max-width: 768px) {
-    #graph-container {
-        grid-template-columns: 1fr;
-    }
-}
-@media (max-width: 1024px) {
-    #graph-container {
-        grid-template-columns: 1fr;
-    }
-    #graph-all {
-        margin-right: 0px;
-    }
-    #controls {
-        margin-left: 0px;
-    }
-}
-.main-plot-container svg {
-    background: transparent !important;
-}
-.large-image-background-transparent {
-    margin-left: 0px;
-    margin-right: 0px;
-}
-/* Import transformers-specific styles */
-@import url('./transformers-custom.css');

src/transformers-custom.css DELETED Viewed

@@ -1,741 +0,0 @@
-/* Transformers-specific styling additions */
-/* Code comparison layout */
-.code-compare {
-    display: grid;
-    grid-template-columns: 1fr 1fr;
-    gap: 1.5rem;
-    margin: 2rem 0;
-    align-items: start;
-}
-.code-compare .code-column {
-    background: #ffffff;
-    border: 1px solid #e2e8f0;
-    border-radius: 8px;
-    overflow: hidden;
-    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-}
-.code-compare .code-header {
-    background: #f8f9fa;
-    padding: 0.75rem 1rem;
-    font-weight: 600;
-    color: #495057;
-    border-bottom: 1px solid #e2e8f0;
-    font-size: 0.9em;
-}
-.code-compare pre {
-    margin: 0;
-    padding: 1rem;
-    background: #ffffff;
-    overflow-x: auto;
-    font-size: 0.85em;
-    line-height: 1.4;
-}
-.code-compare pre code {
-    color: #374151;
-}
-/* Mobile responsiveness for code comparison */
-@media (max-width: 768px) {
-    .code-compare {
-        grid-template-columns: 1fr;
-        gap: 1rem;
-    }
-}
-/* Tenet styling - special highlighting for design principles */
-.tenet-list {
-    margin: 3rem 0;
-}
-.tenet-list ol {
-    counter-reset: tenet-counter -1; /* Start from 0 */
-    list-style: none;
-    padding-left: 0;
-    display: grid;
-    grid-template-columns: 1fr;
-    gap: 2.5rem;
-    max-width: 900px;
-    margin: 0 auto;
-}
-.tenet-list li.tenet {
-    counter-increment: tenet-counter;
-    background: linear-gradient(135deg, #ffffff 0%, #f8f9fa 100%);
-    border: 2px solid #e2e8f0;
-    border-radius: 16px;
-    padding: 2rem 2rem 2rem 4rem;
-    margin: 0;
-    position: relative;
-    box-shadow: 0 12px 35px rgba(0, 0, 0, 0.12);
-    transition: all 0.3s ease;
-    cursor: pointer;
-}
-.tenet-list li.tenet:hover {
-    transform: translateY(-8px) scale(1.02);
-    box-shadow: 0 20px 50px rgba(0, 0, 0, 0.25);
-    border-color: rgba(0, 123, 255, 0.5);
-    background: linear-gradient(135deg, #ffffff 0%, #f0f8ff 100%);
-}
-/* Colorful numbering system */
-.tenet-list li.tenet:nth-child(1):before { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); }
-.tenet-list li.tenet:nth-child(2):before { background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%); }
-.tenet-list li.tenet:nth-child(3):before { background: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%); }
-.tenet-list li.tenet:nth-child(4):before { background: linear-gradient(135deg, #43e97b 0%, #38f9d7 100%); }
-.tenet-list li.tenet:nth-child(5):before { background: linear-gradient(135deg, #fa709a 0%, #fee140 100%); }
-.tenet-list li.tenet:nth-child(6):before { background: linear-gradient(135deg, #a8edea 0%, #fed6e3 100%); }
-.tenet-list li.tenet:nth-child(7):before { background: linear-gradient(135deg, #ff9a9e 0%, #fecfef 100%); }
-.tenet-list li.tenet:nth-child(8):before { background: linear-gradient(135deg, #a18cd1 0%, #fbc2eb 100%); }
-.tenet-list li.tenet:nth-child(9):before { background: linear-gradient(135deg, #ffecd2 0%, #fcb69f 100%); }
-.tenet-list li.tenet:before {
-    content: counter(tenet-counter);
-    position: absolute;
-    top: -12px;
-    left: -12px;
-    color: white;
-    width: 48px;
-    height: 48px;
-    border-radius: 50%;
-    display: flex;
-    align-items: center;
-    justify-content: center;
-    font-size: 1.2em;
-    font-weight: bold;
-    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
-    border: 3px solid white;
-}
-.tenet-list li.tenet strong {
-    color: #1a202c;
-    font-size: 1.1em;
-    display: block;
-    margin-bottom: 0.5rem;
-}
-.tenet-list li.tenet em {
-    color: #4a5568;
-    font-size: 0.95em;
-    font-style: italic;
-    display: block;
-    margin-top: 0.75rem;
-    padding: 1rem;
-    background: rgba(0, 0, 0, 0.03);
-    border-radius: 8px;
-    border-left: 3px solid #e2e8f0;
-}
-.tenet-list li.tenet p {
-    color: #2d3748;
-    line-height: 1.6;
-    margin: 0.5rem 0;
-}
-/* Add a subtle pulse animation for the numbers */
-@keyframes pulse-glow {
-    0% { box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); }
-    50% { box-shadow: 0 4px 20px rgba(0, 0, 0, 0.25); }
-    100% { box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); }
-}
-.tenet-list li.tenet:hover:before {
-    animation: pulse-glow 2s ease-in-out infinite;
-}
-/* Interactive component styling */
-.interactive-demo {
-    border: 1px solid #e2e8f0;
-    border-radius: 12px;
-    background: #ffffff;
-    margin: 2rem 0;
-    overflow: hidden;
-    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.07);
-}
-/* Model visualization fragment styling */
-[id*="plot-model-visualisation"] {
-    margin: 1rem -2rem !important;
-    width: calc(100% + 4rem) !important;
-}
-.interactive-demo .demo-header {
-    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-    color: white;
-    padding: 1rem 1.5rem;
-    font-weight: 600;
-}
-.interactive-demo .demo-content {
-    padding: 1.5rem;
-}
-.interactive-demo .demo-footer {
-    background: #f8f9fa;
-    padding: 1rem 1.5rem;
-    border-top: 1px solid #e2e8f0;
-    color: #6c757d;
-    font-size: 0.9em;
-}
-/* Button styling for interactive elements */
-.btn-primary {
-    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
-    border: none;
-    color: white;
-    padding: 0.75rem 1.5rem;
-    border-radius: 6px;
-    font-weight: 500;
-    cursor: pointer;
-    transition: transform 0.2s, box-shadow 0.2s;
-}
-.btn-primary:hover {
-    transform: translateY(-1px);
-    box-shadow: 0 4px 12px rgba(102, 126, 234, 0.3);
-}
-.btn-primary:disabled {
-    opacity: 0.6;
-    cursor: not-allowed;
-    transform: none;
-    box-shadow: none;
-}
-/* Terminal styling */
-.terminal-container {
-    background: #1a202c;
-    border-radius: 8px;
-    padding: 1rem;
-    color: #e2e8f0;
-    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-    font-size: 0.9em;
-}
-.terminal-input {
-    background: #2d3748;
-    border: 1px solid #4a5568;
-    color: #e2e8f0;
-    padding: 0.5rem;
-    border-radius: 4px;
-    width: 100%;
-    font-family: inherit;
-}
-.terminal-output {
-    background: #0a0e1a;
-    padding: 1rem;
-    border-radius: 4px;
-    white-space: pre-wrap;
-    word-break: break-all;
-    min-height: 100px;
-    max-height: 300px;
-    overflow-y: auto;
-}
-/* Attention visualization styling */
-.attention-matrix {
-    font-family: monospace;
-    font-size: 0.8em;
-    border-collapse: collapse;
-    margin: 1rem 0;
-}
-.attention-matrix td {
-    border: 1px solid #ddd;
-    padding: 4px 8px;
-    text-align: center;
-    min-width: 50px;
-}
-/* Memory chart styling */
-.memory-chart-container {
-    background: #f8f9fa;
-    border: 2px solid #e9ecef;
-    border-radius: 8px;
-    padding: 1rem;
-    margin: 1rem 0;
-}
-/* Image styling improvements */
-img {
-    max-width: 100%;
-    height: auto;
-    border-radius: 8px;
-    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
-    margin: 1.5rem 0;
-}
-/* Table of contents styling - Fixed positioning like ultrascale */
-@media (min-width: 1200px) {
-    d-article {
-        overflow: visible !important;
-    }
-    d-contents {
-        align-self: start !important;
-        background: white !important;
-        grid-column-start: 1 !important;
-        grid-column-end: 4 !important;
-        grid-row: auto / span 6 !important;
-        justify-self: end !important;
-        margin-top: 0em !important;
-        padding-right: 3em !important;
-        padding-left: 2em !important;
-        position: -webkit-sticky !important; /* For Safari */
-        position: sticky !important;
-        top: 10px !important;
-        overflow-y: auto !important;
-        height: calc(100vh - 40px) !important;
-        scrollbar-width: none !important;
-        transition: max-height 0.3s ease-out !important;
-        z-index: -100 !important;
-        display: block !important;
-        visibility: visible !important;
-    }
-}
-@media (max-width: 1199px) {
-    d-contents {
-        display: none !important;
-        background: white !important;
-        justify-self: start !important;
-        align-self: start !important;
-        padding-bottom: 0.5em !important;
-        margin-bottom: 1em !important;
-        padding-left: 0.25em !important;
-        border-bottom: 1px solid rgba(0, 0, 0, 0.1) !important;
-        overflow-y: scroll !important;
-        height: calc(100vh - 40px) !important;
-        scrollbar-width: none !important;
-        z-index: -100 !important;
-    }
-}
-/* Force TOC to be visible and override distill defaults */
-d-contents {
-    display: block !important;
-    visibility: visible !important;
-    opacity: 1 !important;
-}
-/* TOC Navigation styling */
-d-contents .toc-header {
-    margin-bottom: 1.5rem;
-    border-bottom: 2px solid #007bff;
-    padding-bottom: 0.5rem;
-}
-d-contents .toc-title {
-    font-weight: bold;
-    font-size: 1.2em;
-    color: #333;
-}
-d-contents nav a {
-    color: rgba(0, 0, 0, 0.7);
-    text-decoration: none;
-    border-bottom: none;
-    display: block;
-    padding: 0.3rem 0;
-    font-size: 0.9em;
-    line-height: 1.4;
-    transition: color 0.2s ease;
-}
-d-contents nav a:hover {
-    color: #007bff;
-    text-decoration: none;
-}
-d-contents nav a.active {
-    color: #007bff;
-    font-weight: 600;
-}
-d-contents nav div {
-    margin-bottom: 0.2rem;
-}
-/* Smooth scrollbar */
-d-contents {
-    scrollbar-width: thin;
-    scrollbar-color: rgba(0, 123, 255, 0.3) transparent;
-}
-d-contents::-webkit-scrollbar {
-    width: 6px;
-}
-d-contents::-webkit-scrollbar-track {
-    background: transparent;
-}
-d-contents::-webkit-scrollbar-thumb {
-    background: rgba(0, 123, 255, 0.3);
-    border-radius: 3px;
-}
-d-contents::-webkit-scrollbar-thumb:hover {
-    background: rgba(0, 123, 255, 0.5);
-}
-/* Custom tooltip styling for tenet links */
-d-contents nav a[title] {
-    position: relative;
-    cursor: help;
-}
-d-contents nav a[title]:hover {
-    color: #667eea;
-}
-/* Enhanced tooltip using CSS (fallback for title attribute) */
-d-contents nav a[title]:after {
-    content: attr(title);
-    position: absolute;
-    left: 100%;
-    top: 50%;
-    transform: translateY(-50%);
-    background: #1a202c;
-    color: white;
-    padding: 0.75rem 1rem;
-    border-radius: 8px;
-    font-size: 0.85em;
-    white-space: normal;
-    width: 300px;
-    line-height: 1.4;
-    z-index: 1001;
-    opacity: 0;
-    visibility: hidden;
-    transition: opacity 0.3s ease, visibility 0.3s ease;
-    pointer-events: none;
-    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
-}
-d-contents nav a[title]:before {
-    content: '';
-    position: absolute;
-    left: 100%;
-    top: 50%;
-    transform: translate(-8px, -50%);
-    border: 8px solid transparent;
-    border-right-color: #1a202c;
-    z-index: 1002;
-    opacity: 0;
-    visibility: hidden;
-    transition: opacity 0.3s ease, visibility 0.3s ease;
-}
-d-contents nav a[title]:hover:after,
-d-contents nav a[title]:hover:before {
-    opacity: 1;
-    visibility: visible;
-}
-/* Adjust for smaller screens */
-@media (max-width: 1400px) {
-    d-contents nav a[title]:after {
-        left: auto;
-        right: 100%;
-        margin-right: 1rem;
-        width: 250px;
-    }
-    d-contents nav a[title]:before {
-        left: auto;
-        right: 100%;
-        transform: translate(8px, -50%);
-        border-right-color: transparent;
-        border-left-color: #1a202c;
-    }
-}
-/* Improve code syntax highlighting with Prism */
-pre[class*="language-"] {
-    background: #f8f9fa !important;
-    border: 1px solid #e9ecef !important;
-    border-radius: 8px !important;
-    padding: 1.5rem !important;
-    margin: 1.5rem 0 !important;
-    overflow-x: auto !important;
-    font-size: 0.9em !important;
-    line-height: 1.5 !important;
-}
-code[class*="language-"] {
-    background: none !important;
-    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace !important;
-    color: #383a42 !important;
-}
-/* Inline code */
-p code, li code {
-    background: #f1f3f4 !important;
-    padding: 0.2em 0.4em !important;
-    border-radius: 3px !important;
-    font-size: 0.9em !important;
-    color: #d73a49 !important;
-}
-/* Distill article improvements */
-d-article {
-    max-width: none;
-    font-size: 19px;
-    line-height: 1.7 !important;
-    color: #1a1a1a;
-    padding-top: 1rem !important;
-    grid-row-gap: 0 !important;
-}
-d-article > * {
-    grid-column: middle !important;
-    max-width: none;
-}
-/* Adjust for TOC on larger screens */
-@media (min-width: 1200px) {
-    d-article > * {
-        grid-column: text / page-end !important;
-        max-width: none;
-    }
-}
-/* Improve paragraph readability */
-d-article p {
-    font-size: 19px;
-    line-height: 1.5;
-    margin-top: 0 !important;
-    color: #1a1a1a;
-}
-/* Improve heading sizes */
-d-article h1 {
-    font-size: 3rem;
-    line-height: 1.2;
-    margin: 3rem 0 2rem 0;
-    color: #1a202c;
-    font-weight: 700;
-}
-d-article h2 {
-    font-size: 2.5rem;
-    line-height: 1.3;
-    margin: 1.5rem 0 0.75rem 0 !important;
-    padding-bottom: 0.5rem !important;
-    color: #1a202c;
-    font-weight: 650;
-}
-d-article h3 {
-    font-size: 2rem;
-    line-height: 1.4;
-    margin: 2rem 0 1rem 0;
-    color: #1a202c;
-    font-weight: 600;
-}
-d-article h4 {
-    font-size: 1.5rem;
-    line-height: 1.4;
-    margin: 1.5rem 0 1rem 0;
-    color: #2d3748;
-    font-weight: 600;
-}
-/* Improve list readability */
-d-article ul li,
-d-article ol li {
-    font-size: 18px;
-    line-height: 1.7;
-    margin-bottom: 0.5rem;
-}
-/* Enhanced tenet reference styling with custom tooltips */
-a[href^="#source-of-truth"],
-a[href^="#one-model-one-file"],
-a[href^="#code-is-product"],
-a[href^="#standardize-dont-abstract"],
-a[href^="#do-repeat-yourself"],
-a[href^="#minimal-user-api"],
-a[href^="#backwards-compatibility"],
-a[href^="#consistent-public-surface"],
-a[href^="#modular-toolbox"] {
-    position: relative;
-    color: #667eea;
-    font-weight: 600;
-    text-decoration: underline;
-    text-decoration-color: rgba(102, 126, 234, 0.3);
-    transition: all 0.3s ease;
-}
-a[href^="#source-of-truth"]:hover,
-a[href^="#one-model-one-file"]:hover,
-a[href^="#code-is-product"]:hover,
-a[href^="#standardize-dont-abstract"]:hover,
-a[href^="#do-repeat-yourself"]:hover,
-a[href^="#minimal-user-api"]:hover,
-a[href^="#backwards-compatibility"]:hover,
-a[href^="#consistent-public-surface"]:hover,
-a[href^="#modular-toolbox"]:hover {
-    color: #4c51bf;
-    text-decoration-color: #4c51bf;
-    background: rgba(102, 126, 234, 0.1);
-    padding: 2px 4px;
-    border-radius: 4px;
-}
-/* Custom tooltip using data-tooltip attribute */
-a[data-tooltip]:after {
-    content: attr(data-tooltip);
-    position: absolute;
-    bottom: 100%;
-    left: 50%;
-    transform: translateX(-50%);
-    background: #1a202c;
-    color: white;
-    padding: 0.75rem 1rem;
-    border-radius: 8px;
-    font-size: 0.85em;
-    font-weight: 400;
-    white-space: normal;
-    width: 320px;
-    line-height: 1.4;
-    z-index: 1001;
-    opacity: 0;
-    visibility: hidden;
-    transition: opacity 0.3s ease, visibility 0.3s ease;
-    pointer-events: none;
-    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
-    margin-bottom: 8px;
-}
-a[data-tooltip]:before {
-    content: '';
-    position: absolute;
-    bottom: 100%;
-    left: 50%;
-    transform: translateX(-50%);
-    border: 8px solid transparent;
-    border-top-color: #1a202c;
-    z-index: 1002;
-    opacity: 0;
-    visibility: hidden;
-    transition: opacity 0.3s ease, visibility 0.3s ease;
-}
-a[data-tooltip]:hover:after,
-a[data-tooltip]:hover:before {
-    opacity: 1;
-    visibility: visible;
-}
-/* Breadcrumb navigation styling */
-.crumbs {
-    background: linear-gradient(135deg, #f0f4ff 0%, #e6eeff 100%);
-    border-left: 5px solid #667eea;
-    padding: 1.25rem 1.75rem;
-    margin: 2.5rem 0;
-    border-radius: 0 8px 8px 0;
-    box-shadow: 0 2px 8px rgba(102, 126, 234, 0.12);
-    font-size: 0.95em;
-    line-height: 1.6;
-    color: #4a5568;
-}
-.crumbs strong {
-    color: #667eea;
-    font-weight: 700;
-}
-.crumbs code {
-    background: rgba(102, 126, 234, 0.1);
-    padding: 0.15em 0.4em;
-    border-radius: 3px;
-    font-size: 0.9em;
-    color: #4c51bf;
-}
-.crumbs a {
-    color: #667eea;
-    font-weight: 500;
-}
-/* Improve blockquote styling */
-d-article blockquote {
-    font-size: 19px;
-    line-height: 1.8;
-    padding: 1.5rem 2rem;
-    margin: 2rem 0;
-    border-left: 4px solid #667eea;
-    background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 50%);
-    border-radius: 0 8px 8px 0;
-    font-style: italic;
-    color: #4a5568;
-}
-/* Link capsule styling - only for external HTTP(S) links */
-d-article a[href^="http://"],
-d-article a[href^="https://"] {
-    background: linear-gradient(135deg, #e3f2fd 0%, #bbdefb 100%);
-    color: #1565c0;
-    text-decoration: none;
-    padding: 0.15em 0.5em;
-    border-radius: 12px;
-    border: 1px solid #90caf9;
-    display: inline-block;
-    transition: all 0.3s ease;
-    font-weight: 500;
-    box-shadow: 0 1px 3px rgba(21, 101, 192, 0.15);
-}
-d-article a[href^="http://"]:hover,
-d-article a[href^="https://"]:hover {
-    background: linear-gradient(135deg, #2196f3 0%, #1976d2 100%);
-    color: white;
-    border-color: #1565c0;
-    transform: translateY(-1px);
-    box-shadow: 0 4px 12px rgba(21, 101, 192, 0.3);
-}
-d-article a[href^="http://"]:active,
-d-article a[href^="https://"]:active {
-    transform: translateY(0);
-    box-shadow: 0 1px 3px rgba(21, 101, 192, 0.2);
-}
-/* Full width elements */
-d-article .code-compare,
-d-article .interactive-demo,
-d-article .memory-chart-container {
-    max-width: none;
-    width: 100%;
-    margin-left: 0;
-    margin-right: 0;
-}
-/* Responsive design improvements */
-@media (max-width: 1200px) {
-    d-article .code-compare,
-    d-article .interactive-demo {
-        max-width: 95%;
-        margin-left: auto;
-        margin-right: auto;
-    }
-}
-@media (max-width: 768px) {
-    .tenet-list li.tenet {
-        padding: 1rem;
-    }
-    .interactive-demo .demo-content {
-        padding: 1rem;
-    }
-}

webpack.config.js CHANGED Viewed

@@ -257,7 +257,10 @@ module.exports = {
     "title": "${appConfig.fullTitle}",
     "description": "${appConfig.description}",
     "published": "Aug 21, 2025",
-    "authors": [{"author": "Pablo Montalvo", "authorURL": "https://huggingface.co/Molbap"}]
   }</script>
     </d-front-matter>
     <d-title>

     "title": "${appConfig.fullTitle}",
     "description": "${appConfig.description}",
     "published": "Aug 21, 2025",
+    "authors": [{"author": "Pablo Montalvo", "authorURL": "https://huggingface.co/Molbap"},
+    {"author": "Lysandre Debut", "authorURL": "https://huggingface.co/lysandre"},
+    {"author": "Pedro Cuenca", "authorURL": "https://huggingface.co/pcuenq"}
+    {"author": "Yoni Gozlan", "authorURL": "https://huggingface.co/yonigozlan"}]
   }</script>
     </d-front-matter>
     <d-title>