dist

dist/fragments/attention-visualizer.html

@@ -0,0 +1,46 @@
+<div style="border: 1px solid #e2e8f0; border-radius: 8px; background: white; margin: 1.5rem 0;">
+    <div style="padding: 1rem; border-bottom: 1px solid #e2e8f0; background: #f8f9fa;">
+        <h4 style="margin: 0 0 0.5rem 0; color: #495057;">🔍 Attention Mask Visualizer</h4>
+        <p style="margin: 0; font-size: 0.9em; color: #6c757d;">
+            Visualize attention patterns in transformer models. This helps debug attention mask issues.
+        </p>
+    </div>
+    
+    <div style="padding: 1rem;">
+        <div style="display: grid; grid-template-columns: 1fr auto; gap: 1rem; align-items: start; margin-bottom: 1rem;">
+            <div>
+                <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Model:</label>
+                <select id=model-select style="width: 100%; padding: 0.5rem; border: 1px solid #d1d5db; border-radius: 6px; background: white;">
+                    <option value=openai-community/gpt2>openai-community/gpt2</option>
+                    <option value=google/gemma-2-2b>google/gemma-2-2b</option>
+                    <option value=microsoft/DialoGPT-small>microsoft/DialoGPT-small</option>
+                </select>
+            </div>
+            
+            <div>
+                <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Action:</label>
+                <button id=visualize-btn style="padding: 0.5rem 1rem; background: #3b82f6; color: white; border: none; border-radius: 6px; cursor: pointer; font-weight: 500;">
+                    🚀 Visualize
+                </button>
+            </div>
+        </div>
+        
+        <div style="margin-bottom: 1rem;">
+            <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Prompt:</label>
+            <textarea id=prompt-input style="width: 100%; padding: 0.75rem; border: 1px solid #d1d5db; border-radius: 6px; resize: vertical; font-family: monospace; font-size: 0.9em;" rows=3 placeholder="You are an assistant. Make sure you print me."></textarea>
+        </div>
+        
+        <div id=attention-output style="min-height: 200px; background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 6px; padding: 1rem;">
+            <div style="text-align: center; color: #6c757d; font-style: italic;">
+                Click "Visualize" to generate attention visualization
+            </div>
+        </div>
+    </div>
+    
+    <div style="padding: 1rem; border-top: 1px solid #e2e8f0; background: #f8f9fa; font-size: 0.9em; color: #6c757d;">
+        <strong>Note:</strong> This is a demonstration. In the original Gradio app, this would use GPU processing with ZeroGPU 
+        to generate real attention visualizations from transformer models.
+    </div>
+</div>
+
+<script>document.addEventListener("DOMContentLoaded",function(){let e=document.getElementById("model-select"),t=document.getElementById("prompt-input"),n=document.getElementById("visualize-btn"),i=document.getElementById("attention-output");t.value="You are an assistant. Make sure you print me.",n.addEventListener("click",function(){let o=e.value,r=t.value.trim();if(!r){i.innerHTML='<div style="color: #e53e3e;">Please enter a prompt</div>';return}n.disabled=!0,n.textContent="Processing...",i.innerHTML='<div style="text-align: center; color: #6c757d;"><em>Generating attention visualization...</em></div>',setTimeout(()=>{let e=r.split(" ").slice(0,8),t='<div style="margin-bottom: 1rem;"><strong>Model:</strong> '+o+"</div>";t+='<div style="margin-bottom: 1rem;"><strong>Tokens:</strong> '+e.join(" • ")+'</div><div><strong>Attention Matrix (Layer 0, Head 0):</strong></div><table style="margin-top: 0.5rem; border-collapse: collapse; font-family: monospace; font-size: 0.8em;">';for(let n=0;n<e.length;n++){t+="<tr>";for(let n=0;n<e.length;n++){let e=Math.random(),n=`rgba(59, 130, 246, ${e})`;t+=`<td style="border: 1px solid #ddd; padding: 4px; background: ${n}; text-align: center; min-width: 40px;">${e.toFixed(2)}</td>`}t+="</tr>"}i.innerHTML=t+='</table><div style="margin-top: 1rem; font-size: 0.9em; color: #6c757d;"><em>Darker blue = higher attention weight</em></div>',n.disabled=!1,n.textContent="\uD83D\uDE80 Visualize"},2e3)})})</script>

dist/fragments/d3-graph.html

@@ -0,0 +1,12 @@
+<div class=interactive-demo>
+    <div class=demo-header>
+        <h3>🔗 Model Dependency Graph</h3>
+    </div>
+    <div class=demo-content>
+        <iframe src=static/d3_dependency_graph.html width=100% height=600px frameborder=0 style="border-radius: 8px; background: white;"></iframe>
+    </div>
+    <div class=demo-footer>
+        Interactive dependency graph showing real relationships between Transformers models. 🟡 Base models (HuggingFace logo), 🔵 Derived modular models. Click and drag to explore!
+    </div>
+</div>
+

dist/fragments/glm-compare.html

@@ -0,0 +1,149 @@
+<div class=code-compare style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin: 1.5rem 0;">
+    <div class=code-column style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
+        <div class=code-header style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
+            modular_glm.py
+        </div>
+        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em;"><code class=language-python>class GlmMLP(Phi3MLP):
+    pass
+
+class GlmAttention(LlamaAttention):
+    def __init__(self, config, layer_idx=None):
+        super().__init__(config, layer_idx)
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, 
+            config.hidden_size, 
+            bias=False
+        )
+
+class GlmForCausalLM(LlamaForCausalLM):
+    pass</code></pre>
+    </div>
+    
+    <div class=code-column style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
+        <div class=code-header style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
+            modeling_glm.py (auto-expanded)
+        </div>
+        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em; max-height: 400px;"><code class=language-python>class GlmMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.gate_up_proj = nn.Linear(
+            config.hidden_size, 
+            2 * config.intermediate_size, 
+            bias=False
+        )
+        self.down_proj = nn.Linear(
+            config.intermediate_size, 
+            config.hidden_size, 
+            bias=False
+        )
+        self.activation_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        up_states = self.gate_up_proj(hidden_states)
+        gate, up_states = up_states.chunk(2, dim=-1)
+        up_states = up_states * self.activation_fn(gate)
+        return self.down_proj(up_states)
+
+
+class GlmAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: GlmConfig, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(
+            config, "head_dim", 
+            config.hidden_size // config.num_attention_heads
+        )
+        self.num_key_value_groups = (
+            config.num_attention_heads // config.num_key_value_heads
+        )
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_attention_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_key_value_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_key_value_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, 
+            config.hidden_size, 
+            bias=False
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_value: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
+
+        attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self, query_states, key_states, value_states,
+            attention_mask, dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling, **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class GlmRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+# ... (many more classes and functions would follow)</code></pre>
+    </div>
+</div>
+
+<p style="text-align: center; font-style: italic; color: #6c757d; margin-top: 1rem;">
+    <strong>Left:</strong> Clean modular definition with inheritance. 
+    <strong>Right:</strong> Auto-expanded version with all inherited functionality visible.
+</p>

dist/fragments/memory-profiler.html

@@ -0,0 +1,16 @@
+<div style="border: 1px solid #e2e8f0; border-radius: 8px; background: white; margin: 1.5rem 0;">
+    <div style="padding: 1rem; border-bottom: 1px solid #e2e8f0; background: #f8f9fa;">
+        <h4 style="margin: 0 0 0.5rem 0; color: #495057;">🚀 CUDA Warmup Efficiency Benchmark</h4>
+        <p style="margin: 0; font-size: 0.9em; color: #6c757d;">
+            Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the caching_allocator_warmup function.
+        </p>
+    </div>
+    
+    <div style="padding: 1rem;">
+        <iframe src=https://molbap-cuda-warmup-transformers.hf.space width=100% height=800px frameborder=0 style="border-radius: 8px; background: white;"></iframe>
+    </div>
+    
+    <div style="padding: 1rem; border-top: 1px solid #e2e8f0; background: #f8f9fa; font-size: 0.9em; color: #6c757d;">
+        Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the <code>caching_allocator_warmup</code> function at <code>transformers/src/transformers/modeling_utils.py:6186</code>. This interactive tool loads models twice - once with warmup disabled and once with warmup enabled - to demonstrate the significant loading time improvements.
+    </div>
+</div>

dist/fragments/terminal.html

@@ -0,0 +1,22 @@
+<div style="background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 8px; padding: 1rem; margin: 1.5rem 0;">
+    <h4 style="margin-top: 0; color: #495057;">Interactive Terminal</h4>
+    <div style="background: #2d3748; color: #e2e8f0; padding: 1rem; border-radius: 6px; font-family: 'Consolas', 'Monaco', monospace;">
+        <div style="margin-bottom: 1rem;">
+            <input type=text id=terminal-input placeholder="python -c 'import torch; print(torch.__version__)'" style="width: calc(100% - 80px); padding: 0.5rem; background: #1a202c; border: 1px solid #4a5568; color: #e2e8f0; border-radius: 4px;">
+            <button id=terminal-run style="width: 70px; padding: 0.5rem; margin-left: 8px; background: #3182ce; color: white; border: none; border-radius: 4px; cursor: pointer;">Run</button>
+        </div>
+        <pre id=terminal-output style="background: #1a202c; padding: 1rem; border-radius: 4px; min-height: 100px; margin: 0; overflow-x: auto;">$ Ready to run commands...</pre>
+    </div>
+    <p style="font-size: 0.9em; color: #6c757d; margin-top: 0.5rem;">
+        <em>Note: This is a simulated terminal. In the original Gradio app, this would execute real Python commands with proper security restrictions.</em>
+    </p>
+</div>
+
+<script>document.addEventListener("DOMContentLoaded",function(){let e=document.getElementById("terminal-input"),t=document.getElementById("terminal-run"),n=document.getElementById("terminal-output");function o(){let t=e.value.trim();t&&(n.textContent=`$ ${t}
+Simulated output for: ${t}
+
+This would execute the command in the original app.
+Example outputs:
+- torch version: 2.0.1+cu117
+- import checks: Success
+- memory info: Available`)}t.addEventListener("click",o),e.addEventListener("keypress",function(e){"Enter"===e.key&&o()})})</script>

dist/index.html

@@ -0,0 +1,615 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="distill.bundle.js" type="module" fetchpriority="high" blocking></script>
+    <script src="main.bundle.js" type="module" fetchpriority="low" defer></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/components/prism-core.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/plugins/autoloader/prism-autoloader.min.js"></script>
+    <script src="https://d3js.org/d3.v7.min.js"></script>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta charset="utf8">
+    <title>Transformers Feature Showcase</title>
+    <link rel="stylesheet" href="style.css">
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.29.0/themes/prism.min.css">
+</head>
+<body>
+    <d-front-matter>
+        <script id='distill-front-matter' type="text/json">{
+    "title": "Transformers Feature Showcase",
+    "description": "An interactive demonstration of transformers library features and design philosophy.",
+    "published": "Aug 21, 2025",
+    "authors": [{"author": "Pablo Montalvo", "authorURL": "https://huggingface.co/Molbap"}]
+  }</script>
+    </d-front-matter>
+    <d-title>
+        <h1>Transformers Feature Showcase</h1>
+        <p>An interactive demonstration of transformers library features and design philosophy.</p>
+    </d-title>
+    <d-byline></d-byline>
+    <d-article>
+        <d-contents>
+            <nav role="navigation" class="l-text figcaption">
+                <div class="toc-header"><span class="toc-title">Table of Contents</span></div>
+                <div class="toc-content">
+                    <div><a href="#introduction">Introduction</a></div>
+                    <div style="margin-left: 1.2em;"><a href="#what-you-will-learn">What you will learn</a></div>
+                    <div><a href="#source-of-truth">0. Source of truth</a></div>
+                    <div><a href="#one-model-one-file">1. One model, one file</a></div>
+                    <div><a href="#code-is-product">2. Code is product</a></div>
+                    <div><a href="#standardize-dont-abstract">3. Standardize, don't abstract</a></div>
+                    <div><a href="#do-repeat-yourself">4. DRY* (DO Repeat Yourself)</a></div>
+                    <div><a href="#minimal-user-api">5. Minimal user API</a></div>
+                    <div><a href="#backwards-compatibility">6. Backwards compatibility</a></div>
+                    <div><a href="#consistent-public-surface">7. Consistent public surface</a></div>
+                    <div><a href="#modular">Going modular</a></div>
+                    <div><a href="#attention-classes">External Attention classes</a></div>
+                    <div><a href="#encoders-ftw">Encoders win!</a></div>
+                </div>
+            </nav>
+        </d-contents>
+        <h2>Introduction</h2>
+<p>The <code>transformers</code> library, built with <code>PyTorch</code>, supports all state-of-the-art LLMs, many VLMs, task-specific vision language models, video models, audio models, table models, classical encoders, to a global count of almost 400 models. The name of the library itself is mostly majority driven as many models are not even transformers architectures, like Mamba/RWKV.  Regardless, each of these is wrought by the research and engineering team that created them, then harmonized into a now famous interface, and callable with a simple <code>.from_pretrained</code>. Inference and training are supported. The library supports ML courses, cookbooks, and several thousands other open-source libraries depend on it. All models are tested as part of a daily CI ensuring their preservation and reproducibility. Most importantly, it is open-source and has been written by the community for a large part.</p>
+<p>The ML wave has not stopped, there’s more and more models being added. <code>Transformers</code> is widely used, and we read the feedback that users post. Whether it’s about a function that had 300+ keyword arguments, duplicated code and helpers, and mentions of <code>Copied from ... </code> everywhere, along with optimisation concerns. Text-only models are relatively tamed, but multimodal models remain to be harmonized.</p>
+<p>Here we will dissect what is the design philosophy of transformers, as a continuation from the existing older <a href="https://huggingface.co/docs/transformers/en/philosophy">philosophy</a> page, and an accompanying <a href="https://huggingface.co/blog/transformers-design-philosophy">blog post from 2022</a> . Some time ago I dare not say how long, we discussed with transformers maintainers about the state of things. A lot of recent developments were satisfactory, but if we were only talking about these, self-congratulation would be the only goalpost. Reflecting on this philosophy now, as models pile up, is essential and will drive new developments.</p>
+<h3>What you will learn</h3>
+<p>Every reader, whether an OSS maintainer, power user, or casual fine-tuner, will walk away knowing how to reason about the <code>transformers</code> code base, how to use it better, how to meaningfully contribute to it.
+This will also showcase new features you might have missed so you’ll be up-to-date.</p>
+<p>So, what are the principles of <code>transformers</code>? We will try to summarize the foundations on which we’ve built everything, and write the “tenets” of the library.  They behave like <em>software interfaces</em>, hence it is crucial that they are explicitly written down. However opinionated they are, they have evolved over time.</p>
+<div class="tenet-list">
+<ol>
+<li class="tenet">
+<a id="source-of-truth"></a>
+<strong>Source of Truth</strong>
+<p>We should be a source of truth for all model definitions. This is not a tenet, but something that still guides our decisions. Model implementations should be reliable, reproducible, and faithful to the original performances.</p>
+<em>This overarching guideline ensures quality and reproducibility across all models in the library.</em>
+</li> 
+<li class="tenet">
+<a id="one-model-one-file"></a>
+<strong>One Model, One File</strong>
+<p>All inference (and most of training, loss is separate, not a part of model) logic visible, top‑to‑bottom.</p>
+<em>Every model should be completely understandable by reading a single file from top to bottom.</em>
+</li> 
+<li class="tenet">
+<a id="code-is-product"></a>
+<strong>Code is Product</strong>
+<p>Optimize for reading, diffing, and tweaking, our users are power users. Variables can be explicit, full words, even several words, readability is primordial.</p>
+<em>Code quality matters as much as functionality - optimize for human readers, not just computers.</em>
+</li> 
+<li class="tenet">
+<a id="standardize-dont-abstract"></a>
+<strong>Standardize, Don't Abstract</strong>
+<p>If it's model behavior, keep it in the file; abstractions only for generic infra.</p>
+<em>Model-specific logic belongs in the model file, not hidden behind abstractions.</em>
+</li>
+<li class="tenet">
+<a id="do-repeat-yourself"></a>
+<strong>DRY* (DO Repeat Yourself)</strong>
+<p>Copy when it helps users; keep successors in sync without centralizing behavior.</p>
+<p><strong>Amendment:</strong> With the introduction and global adoption of <a href="#modular">modular</a> transformers, we do not repeat any logic in the modular files, but end user files remain faithful to the original tenet.</p>
+<em>Strategic duplication can improve readability and maintainability when done thoughtfully.</em>
+</li>
+<li class="tenet">
+<a id="minimal-user-api"></a>
+<strong>Minimal User API</strong>
+<p>Config, model, preprocessing; from_pretrained, save_pretrained, push_to_hub. We want the least amount of codepaths. Reading should be obvious, configurations should be obvious.</p>
+<em>Keep the public interface simple and predictable - users should know what to expect.</em>
+</li>
+<li class="tenet">
+<a id="backwards-compatibility"></a>
+<strong>Backwards Compatibility</strong>
+<p>Evolve by additive standardization, <strong>never</strong> break public APIs.</p>
+<p><strong>Note:</strong> Some models are showing almost no use, we also stopped adding new features for non-torch frameworks. Still, we adapt to models existing on the hub.</p>
+<em>Once something is public, it stays public - evolution through addition, not breaking changes.</em>
+</li> 
+<li class="tenet">
+<a id="consistent-public-surface"></a>
+<strong>Consistent Public Surface</strong>
+<p>Same argument names, same outputs, hidden states and attentions exposed, enforced by tests.</p>
+<em>All models should feel familiar - consistent interfaces reduce cognitive load.</em>
+</li>  
+<li class="tenet">
+<a id="modular-toolbox"></a>
+<strong>Modular Toolbox (Not Framework)</strong>
+<p>We ARE a toolbox. What we are not is a framework: you should not be FORCED to rewrite every modeling, but it is <em>better</em> for your model to be able to inherit from PreTrainedModel and have enabled TensorParallel, from_pretrained, sharding, push_to_hub, loss, as well as PEFT/TRL/SGLang/vLLM.</p>
+<em>This is the largest change. Provide tools and utilities, but don't force users into a rigid framework.</em>
+</li>
+</ol>
+</div>  
+<p>When a PR is merged, it is because the contribution is worthwhile, and that the  <code>transformers</code> team finds the design of the contribution to be aligned with what is above.</p>
+<p>Does all the code in the library follow strictly these tenets? No. The library is a gigantic house with connected nooks, corridors, crannies everywhere built by thousands of different workers. We <em>try</em> to make it so all the code added is inline, lest we break <a href="#backwards-compatibility">backwards compatibility</a>.</p>
+<p>For instance, one function essential to the implementation of <a href="https://huggingface.co/papers/2104.09864">Rotary Positional Embeddings</a> is identical in 70  <code>modeling_&lt;file&gt;.py</code> across <code>src/transformers/models/.</code>  Why keep it? Because removing it would make those files unloadable checkpoints rather than self-contained blueprints. We <a href="#do-repeat-yourself">do repeat ourselves</a>.</p>
+<pre><code class="language-python">def rotate_half(x):
+    &quot;&quot;&quot;Rotates half the hidden dims of the input.&quot;&quot;&quot;
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2 :]
+    return torch.cat((-x2, x1), dim=-1)
+</code></pre>
+<p>You can use a script such as [[top_methods.py]]  to look at all methods of a given name across your codebase and look at their differences and similarities, that’s what I did (+ a hash to avoid quadraticity).</p>
+<p>So… why keep it in all modeling files? Because if we were to remove it, the model would not work anymore. Think of the modeling files as a car (I know, what a novel metaphor! But, it works out.). All manual transmission cars have a clutch, but we want each <em>view</em> of one of our cars to be able to function. Remove the clutch, you can’t drive. Remove the doors, might be uncomfortable but you’ll get there. So doors can go, but you <em>have</em> to keep the clutch, even though you know perfectly how it works.</p>
+<p>As I was looking for things to improve and make better, it’s one of the iterations I attempted: a function is almost everywhere the same, let’s import it from some common file? But no! Goes against</p>
+<h2><a id="modular"></a> Going modular</h2>
+<p>However, both of these works were already pointing at some drawbacks, which have been iteratively addressed. <a href="https://huggingface.co/docs/transformers/en/modular_transformers">Transformers has gone modular</a> , allowing a form of inheritance without breaking <a href="#one-model-one-file">One model, One file</a>. If you’re familiar with this, you can <a href="#%5Eattention-classes">skip this section</a> and go to the next one.</p>
+<p>We amended the principle of <a href="#do-repeat-yourself">DRY*</a> by removing progressively all pieces of code that were “copied from” another file.</p>
+<p>It is explained in details in the documentation above, but overall it works like this, you define a <code>modular_</code> file that can inherit from <em>any function across all other modeling, configuration and processor files</em>:</p>
+<summary>Auto-generated modeling code</summary>
+<p><div class=code-compare style="display: grid; grid-template-columns: 1fr 1fr; gap: 1rem; margin: 1.5rem 0;">
+    <div class=code-column style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
+        <div class=code-header style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
+            modular_glm.py
+        </div>
+        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em;"><code class=language-python>class GlmMLP(Phi3MLP):
+    pass
+
+class GlmAttention(LlamaAttention):
+    def __init__(self, config, layer_idx=None):
+        super().__init__(config, layer_idx)
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, 
+            config.hidden_size, 
+            bias=False
+        )
+
+class GlmForCausalLM(LlamaForCausalLM):
+    pass</code></pre>
+    </div>
+    
+    <div class=code-column style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden;">
+        <div class=code-header style="background: #f8f9fa; padding: 0.75rem 1rem; font-weight: 600; color: #495057; border-bottom: 1px solid #e2e8f0;">
+            modeling_glm.py (auto-expanded)
+        </div>
+        <pre style="margin: 0; padding: 1rem; background: #ffffff; overflow-x: auto; font-size: 0.9em; max-height: 400px;"><code class=language-python>class GlmMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.gate_up_proj = nn.Linear(
+            config.hidden_size, 
+            2 * config.intermediate_size, 
+            bias=False
+        )
+        self.down_proj = nn.Linear(
+            config.intermediate_size, 
+            config.hidden_size, 
+            bias=False
+        )
+        self.activation_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+        up_states = self.gate_up_proj(hidden_states)
+        gate, up_states = up_states.chunk(2, dim=-1)
+        up_states = up_states * self.activation_fn(gate)
+        return self.down_proj(up_states)
+
+
+class GlmAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: GlmConfig, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        self.head_dim = getattr(
+            config, "head_dim", 
+            config.hidden_size // config.num_attention_heads
+        )
+        self.num_key_value_groups = (
+            config.num_attention_heads // config.num_key_value_heads
+        )
+        self.scaling = self.head_dim**-0.5
+        self.attention_dropout = config.attention_dropout
+        self.is_causal = True
+
+        self.q_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_attention_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.k_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_key_value_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.v_proj = nn.Linear(
+            config.hidden_size, 
+            config.num_key_value_heads * self.head_dim, 
+            bias=config.attention_bias
+        )
+        self.o_proj = nn.Linear(
+            config.num_attention_heads * self.head_dim, 
+            config.hidden_size, 
+            bias=False
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_value: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        key_states = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(
+                key_states, value_states, self.layer_idx, cache_kwargs
+            )
+
+        attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self, query_states, key_states, value_states,
+            attention_mask, dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling, **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class GlmRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+# ... (many more classes and functions would follow)</code></pre>
+    </div>
+</div>
+
+<p style="text-align: center; font-style: italic; color: #6c757d; margin-top: 1rem;">
+    <strong>Left:</strong> Clean modular definition with inheritance. 
+    <strong>Right:</strong> Auto-expanded version with all inherited functionality visible.
+</p></p>
+<h2><a id="attention-classes"></a> External Attention classes</h2>
+<p>A chronological iteration over <a href="#modular">modular</a>, and a big improvement in terms of readabilty, was to remove the various attention-backend-specific attention classes across the repository. Before, we were adding specific torch operations for each backend (sdpa, flash-attention iterations, flex attention) but it wasn’t a <a href="#minimal-user-api">minimal user api</a>.</p>
+<p>What will forever stay in the modeling code is the <code>eager_attention_forward</code> because it is a core part of the modeling,</p>
+<pre><code class="language-python">attention_interface: Callable = eager_attention_forward
+if self.config._attn_implementation != &quot;eager&quot;:
+    attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+</code></pre>
+<p>We often read and understand that <code>kwargs</code> are criticized, and we are typing them however we can, but we cannot enforce them all the time because other libraries such as vLLM don’'t use the same kwargs.</p>
+<p>It is a strength of the new attention interface, where it can be plugged in various backends, because most of the signature is not enforced. We INFORM but do not ENFORCE. That way, the current system is a <a href="#minimal-user-api">minimal user api</a>.</p>
+<p>For a better <em>information</em>, we plan to use <code>python</code>features such as <code>Annotated</code> for example, to inform users of what we expect typically in an argument. That way, higher-level information could be included directly in the type annotations, telling for instance the expected dimensions and contents of a tensor.</p>
+<h2><a id="simpler-tensor-parallelism"></a> Simpler Tensor Parallelism</h2>
+<p>We want to touch minimally to the modeling code, and only modify it when <em>architectural changes</em> are involved. For instance, for tensor parallelism, we instead now specify a simple <code>tp_plan</code>.</p>
+<h2><a id="layers-attentions-caches"></a> Layers, attentions and caches</h2>
+<p>With th</p>
+<h2><a id="community-kernels"></a>Community Kernels</h2>
+<p>The same principle extends to normalization, activation, and other hot paths. The model defines <strong>semantics</strong>; a kernel defines <strong>how</strong> to execute them faster. We annotate the module to borrow a community‑provided forward, keeping a <a href="#consistent-public-surface">consistent public surface</a></p>
+<pre><code class="language-python">@use_kernel_forward_from_hub(&quot;RMSNorm&quot;)
+class GlmRMSNorm(nn.Module):
+    ...
+</code></pre>
+<p>Plus, this opened another angle of contribution for the community. People who are GPU whisperers can check on the <a href="https://huggingface.co/blog/hello-hf-kernels">kernel community blog post</a> to learn more about it!</p>
+<h2>The good modularity</h2>
+<p>Now, we have a form of inheritance in our codebase. Some models become standards, and model contributors are given the opportunity to <em>define standards</em>. Pushing the boundaries of scientific knowledge can translate into the boundaries of engineering if this effort is made, and we’re striving for it.</p>
+<p>My capacity for abstraction is not that great, compared to other computer scientists and engineers: I need to look at little doodles and drawings, especially when components pile up.</p>
+<p>So I wanted to take a look at the current <strong>state of modularity</strong> across the repository. How many models are defined using components of others?</p>
+<p>To get this graph, I used the heuristic of modular inheritance.</p>
+<ol>
+<li>Does this model have a <code>modular</code> file?</li>
+<li>In this <code>modular</code> file, what models, configurations and processings are imported?</li>
+<li>Recurse through the model list that way.</li>
+</ol>
+<p>So what do we see? Llama is a basis for many models, and it shows.
+Radically different architectures such as mamba have spawned their own dependency subgraph.
+<div class=interactive-demo>
+    <div class=demo-header>
+        <h3>🔗 Model Dependency Graph</h3>
+    </div>
+    <div class=demo-content>
+        <iframe src=static/d3_dependency_graph.html width=100% height=600px frameborder=0 style="border-radius: 8px; background: white;"></iframe>
+    </div>
+    <div class=demo-footer>
+        Interactive dependency graph showing real relationships between Transformers models. 🟡 Base models (HuggingFace logo), 🔵 Derived modular models. Click and drag to explore!
+    </div>
+</div>
+
+</p>
+<p><img src="static/graph_modular_related_models.png" alt="Graph showing modular related models"></p>
+<p>But there is no similar miracle for VLMs across the board.
+As you can see, there is a small DETR island, a little llava pocket, and so on, but it’s not comparable to the centrality observed.</p>
+<p>One problem is, this is only for <code>modular</code> models. Several models do NOT have a modular file. In other words, we have a big “hidden space here.”</p>
+<h2>Too many models, yet not enough, are alike</h2>
+<p>So I looked into Jaccard similarity, which we use to measure set differences. I know that code is more than a set of characters stringed together, but it is a correct proxy for now. You can check out  [[find_dependencies.py]] .</p>
+<p><div style="background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 8px; padding: 1rem; margin: 1.5rem 0;">
+    <h4 style="margin-top: 0; color: #495057;">Interactive Terminal</h4>
+    <div style="background: #2d3748; color: #e2e8f0; padding: 1rem; border-radius: 6px; font-family: 'Consolas', 'Monaco', monospace;">
+        <div style="margin-bottom: 1rem;">
+            <input type=text id=terminal-input placeholder="python -c 'import torch; print(torch.__version__)'" style="width: calc(100% - 80px); padding: 0.5rem; background: #1a202c; border: 1px solid #4a5568; color: #e2e8f0; border-radius: 4px;">
+            <button id=terminal-run style="width: 70px; padding: 0.5rem; margin-left: 8px; background: #3182ce; color: white; border: none; border-radius: 4px; cursor: pointer;">Run</button>
+        </div>
+        <pre id=terminal-output style="background: #1a202c; padding: 1rem; border-radius: 4px; min-height: 100px; margin: 0; overflow-x: auto;">$ Ready to run commands...</pre>
+    </div>
+    <p style="font-size: 0.9em; color: #6c757d; margin-top: 0.5rem;">
+        <em>Note: This is a simulated terminal. In the original Gradio app, this would execute real Python commands with proper security restrictions.</em>
+    </p>
+</div>
+
+<script>document.addEventListener("DOMContentLoaded",function(){let e=document.getElementById("terminal-input"),t=document.getElementById("terminal-run"),n=document.getElementById("terminal-output");function o(){let t=e.value.trim();t&&(n.textContent=`$ ${t}
+Simulated output for: ${t}
+
+This would execute the command in the original app.
+Example outputs:
+- torch version: 2.0.1+cu117
+- import checks: Success
+- memory info: Available`)}t.addEventListener("click",o),e.addEventListener("keypress",function(e){"Enter"===e.key&&o()})})</script></p>
+<p><img src="static/Jaccard_similarity_plot.png" alt="Jaccard similarity plot showing model relationships"></p>
+<p>The yellow areas are places where models are very different to each other. We can see islands here and there corresponding to model families. Llava goes with Llava-onevision, LlavaNext, LlavaNext-video, etc.</p>
+<h2>VLM improvements, avoiding abstraction</h2>
+<p>We don’t have cookbook for common VLM patterns (image token scatter, multi‑tower encoders, cross‑attn bridges). This is one of the main improvement points where we can work.</p>
+<p>So initially I thought of abstracting away the mixing of <code>inputs_embeds</code>, the tensor fed into an llm decoder in 95% of the existing VLMs. It would have looked like something like</p>
+<pre><code class="language-python">class InputsEmbeddingMixerMixin(nn.Module):
+    #
+</code></pre>
+<p>But this is breaking <a href="#standardize-dont-abstract">Standardize, don’t abstract</a>. Embedding mixin is part of the model, removing it would break it. A user opening <code>modeling_qwen2.5_vl</code> should not have to go to another file.</p>
+<p>This is the current state of abstractions across a modeling file:</p>
+<p><img src="static/Bloatedness_visualizer.png" alt="Bloatedness visualizer showing abstraction levels"></p>
+<p>The following <a href="https://github.com/huggingface/transformers/pull/39777">Pull request to standardize placeholder masking</a> is a good example of what kind of changes are acceptable. In a VLM, we always need to insert embeddings from various encoders at various positions, so we can have a function to do it. For Qwen2 VL, for instance, it will look like this:</p>
+<pre><code class="language-python">    def get_placeholder_mask(
+        self,
+        input_ids: torch.LongTensor,
+        inputs_embeds: torch.FloatTensor,
+        image_features: torch.FloatTensor = None,
+        video_features: torch.FloatTensor = None,
+    ):
+        &quot;&quot;&quot;
+        Obtains multimodal placeholdr mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
+        equal to the length of multimodal features. If the lengths are different, an error is raised.
+        &quot;&quot;&quot;
+        if input_ids is None:
+            special_image_mask = inputs_embeds == self.get_input_embeddings()(
+                torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
+            )
+            special_image_mask = special_image_mask.all(-1)
+            special_video_mask = inputs_embeds == self.get_input_embeddings()(
+                torch.tensor(self.config.video_token_id, dtype=torch.long, device=inputs_embeds.device)
+            )
+            special_video_mask = special_video_mask.all(-1)
+        else:
+            special_image_mask = input_ids == self.config.image_token_id
+            special_video_mask = input_ids == self.config.video_token_id
+
+        n_image_tokens = special_image_mask.sum()
+        special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+        if image_features is not None and inputs_embeds[special_image_mask].numel() != image_features.numel():
+            raise ValueError(
+                f&quot;Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}&quot;
+            )
+
+        n_video_tokens = special_video_mask.sum()
+        special_video_mask = special_video_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+        if video_features is not None and inputs_embeds[special_video_mask].numel() != video_features.numel():
+            raise ValueError(
+                f&quot;Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}&quot;
+            )
+
+        return special_image_mask, special_video_mask
+</code></pre>
+<p>But this is <em>within</em> the modeling file, not in the <code>PreTrainedModel</code> base class. It will not move away from it, because it’d break the self-contained logic of the model.</p>
+<h2>Modularity candidates</h2>
+<p>So the question abounds naturally: How can we modularize more?
+I took again a similarity measure and looked at the existing graphs. The tool is available on this <a href="https://huggingface.co/spaces/Molbap/transformers-modular-refactor">ZeroGPU-enabled Space</a>. It scans the whole transformers repository, and outputs a graph of candidates across models, using either a Jaccard similarity index (simple) or a SentenceTransformers embedding model. It is understandable that <a href="#encoders-ftw">encoder models still have a lion’s share of the game.</a> See also <a href="https://huggingface.co/blog/train-sparse-encoder">Tom Aarsen and Arhur Bresnu’s great blog post on the topic of sparse embeddings.</a>.</p>
+<p><img src="static/modular_candidates.png" alt="Modular candidates analysis"></p>
+<h2><a id="encoders-ftw"></a> The neverending stories of encoder models.</h2>
+<p>Models popularity speaks for itself! This is because the usage of encoders lies in embeddings obviously. So we have to keep the encoders part viable, usable, fine-tune-able.</p>
+<p><img src="static/popular_models_barplot.png" alt="Popular models bar plot"></p>
+<h2>On image processing and processors</h2>
+<p>Choosing to be a <code>torch</code>-first software meant relieving a tremendous amount of support from <code>jax </code> and <code>TensorFlow</code> , and it also meant that we could be more lenient into the amount of torch-dependent utilities that we were able to add. One of these is the <em>fast processing</em> of images. Where they were before assumed to be minimal ndarrays, making stronger assumptions and enforcing <code>torch</code> and <code>torchvision</code>native inputs allowed up to speed up massively the processing time for each model.</p>
+<p>The gains in performance are immense, up to 20x speed for most models when compiled torchvision ops.</p>
+<h2>Reduce barrier to entry/contribution</h2>
+<p>This is an overall objective, no transformers without community.</p>
+<p>We didn’t want to make a toolbox, old tenet, because <em>having a framework means forcing users into it</em>. It restrains flexibility and creativity, which are the fertile soil for new ideas to grow.
+Among the most valuable contributions to <code>transformers</code>is of course the addition of new models.</p>
+<h2>A surgical toolbox for model development</h2>
+<h3>Attention visualisation</h3>
+<p>If all models have the same API internally for attention computation, it allows us to build cool tools to visualize the inner workings of the attention mechanism. One particular piece of
+machinery is the <code>attention mask</code>, cause of confusion. Thankfully, we can fix it.</p>
+<p><div style="border: 1px solid #e2e8f0; border-radius: 8px; background: white; margin: 1.5rem 0;">
+    <div style="padding: 1rem; border-bottom: 1px solid #e2e8f0; background: #f8f9fa;">
+        <h4 style="margin: 0 0 0.5rem 0; color: #495057;">🔍 Attention Mask Visualizer</h4>
+        <p style="margin: 0; font-size: 0.9em; color: #6c757d;">
+            Visualize attention patterns in transformer models. This helps debug attention mask issues.
+        </p>
+    </div>
+    
+    <div style="padding: 1rem;">
+        <div style="display: grid; grid-template-columns: 1fr auto; gap: 1rem; align-items: start; margin-bottom: 1rem;">
+            <div>
+                <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Model:</label>
+                <select id=model-select style="width: 100%; padding: 0.5rem; border: 1px solid #d1d5db; border-radius: 6px; background: white;">
+                    <option value=openai-community/gpt2>openai-community/gpt2</option>
+                    <option value=google/gemma-2-2b>google/gemma-2-2b</option>
+                    <option value=microsoft/DialoGPT-small>microsoft/DialoGPT-small</option>
+                </select>
+            </div>
+            
+            <div>
+                <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Action:</label>
+                <button id=visualize-btn style="padding: 0.5rem 1rem; background: #3b82f6; color: white; border: none; border-radius: 6px; cursor: pointer; font-weight: 500;">
+                    🚀 Visualize
+                </button>
+            </div>
+        </div>
+        
+        <div style="margin-bottom: 1rem;">
+            <label style="display: block; font-weight: 600; margin-bottom: 0.5rem; color: #374151;">Prompt:</label>
+            <textarea id=prompt-input style="width: 100%; padding: 0.75rem; border: 1px solid #d1d5db; border-radius: 6px; resize: vertical; font-family: monospace; font-size: 0.9em;" rows=3 placeholder="You are an assistant. Make sure you print me."></textarea>
+        </div>
+        
+        <div id=attention-output style="min-height: 200px; background: #f8f9fa; border: 1px solid #e9ecef; border-radius: 6px; padding: 1rem;">
+            <div style="text-align: center; color: #6c757d; font-style: italic;">
+                Click "Visualize" to generate attention visualization
+            </div>
+        </div>
+    </div>
+    
+    <div style="padding: 1rem; border-top: 1px solid #e2e8f0; background: #f8f9fa; font-size: 0.9em; color: #6c757d;">
+        <strong>Note:</strong> This is a demonstration. In the original Gradio app, this would use GPU processing with ZeroGPU 
+        to generate real attention visualizations from transformer models.
+    </div>
+</div>
+
+<script>document.addEventListener("DOMContentLoaded",function(){let e=document.getElementById("model-select"),t=document.getElementById("prompt-input"),n=document.getElementById("visualize-btn"),i=document.getElementById("attention-output");t.value="You are an assistant. Make sure you print me.",n.addEventListener("click",function(){let o=e.value,r=t.value.trim();if(!r){i.innerHTML='<div style="color: #e53e3e;">Please enter a prompt</div>';return}n.disabled=!0,n.textContent="Processing...",i.innerHTML='<div style="text-align: center; color: #6c757d;"><em>Generating attention visualization...</em></div>',setTimeout(()=>{let e=r.split(" ").slice(0,8),t='<div style="margin-bottom: 1rem;"><strong>Model:</strong> '+o+"</div>";t+='<div style="margin-bottom: 1rem;"><strong>Tokens:</strong> '+e.join(" • ")+'</div><div><strong>Attention Matrix (Layer 0, Head 0):</strong></div><table style="margin-top: 0.5rem; border-collapse: collapse; font-family: monospace; font-size: 0.8em;">';for(let n=0;n<e.length;n++){t+="<tr>";for(let n=0;n<e.length;n++){let e=Math.random(),n=`rgba(59, 130, 246, ${e})`;t+=`<td style="border: 1px solid #ddd; padding: 4px; background: ${n}; text-align: center; min-width: 40px;">${e.toFixed(2)}</td>`}t+="</tr>"}i.innerHTML=t+='</table><div style="margin-top: 1rem; font-size: 0.9em; color: #6c757d;"><em>Darker blue = higher attention weight</em></div>',n.disabled=!1,n.textContent="\uD83D\uDE80 Visualize"},2e3)})})</script></p>
+<p>Because it is all PyTorch (and it is even more now that we support only PyTorch), we can easily debug any model when we want to add it to transformers. We now have a power-user tool for porting or adding models, that wraps a forward pass, intercepts every submodule call, and logs shapes, dtypes, and sample statistics of inputs/outputs to nested JSON.</p>
+<p>It just works with PyTorch models and is especially useful when aligning outputs with a reference implementation, aligned with our core guideline, <a href="#source-of-truth">source of truth for model definitions</a>.</p>
+<p><img src="static/model_debugger.png" alt="Model debugger interface"></p>
+<h3>Transformers-serve</h3>
+<p>Having all these models readily available allows to use all of them with transformers-serve, and enable interfacing with them with an Open API-like pattern.</p>
+<p>#### add example</p>
+<h2>Community reusability</h2>
+<p>Adding a model to transformers means:</p>
+<ul>
+<li>having it immediately available to the community</li>
+<li>usable in vLLM, SGLang, and so on without additional code.</li>
+</ul>
+<p>## Inner cooking: CUDA Warmup</p>
+<p>Having a clean <em>external</em> API allows us to work on the true inner workings of transformers. One of the few recent additions was the <em>CUDA warmup</em> via <code>caching_allocator_warmup</code> which improved massively the loading time by pre-allocating GPU memory to avoid malloc bottlenecks during model loading.</p>
+<p><div style="border: 1px solid #e2e8f0; border-radius: 8px; background: white; margin: 1.5rem 0;">
+    <div style="padding: 1rem; border-bottom: 1px solid #e2e8f0; background: #f8f9fa;">
+        <h4 style="margin: 0 0 0.5rem 0; color: #495057;">🚀 CUDA Warmup Efficiency Benchmark</h4>
+        <p style="margin: 0; font-size: 0.9em; color: #6c757d;">
+            Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the caching_allocator_warmup function.
+        </p>
+    </div>
+    
+    <div style="padding: 1rem;">
+        <iframe src=https://molbap-cuda-warmup-transformers.hf.space width=100% height=800px frameborder=0 style="border-radius: 8px; background: white;"></iframe>
+    </div>
+    
+    <div style="padding: 1rem; border-top: 1px solid #e2e8f0; background: #f8f9fa; font-size: 0.9em; color: #6c757d;">
+        Real CUDA warmup benchmarking with actual Transformers models. Measure the performance impact of the <code>caching_allocator_warmup</code> function at <code>transformers/src/transformers/modeling_utils.py:6186</code>. This interactive tool loads models twice - once with warmup disabled and once with warmup enabled - to demonstrate the significant loading time improvements.
+    </div>
+</div></p>
+<h3>Linkedin post (to remove)</h3>
+<p>Linkedin post for videos:</p>
+<p>In transformers, how do we deal with cross-model dependencies, while supporting ~400 models? Maybe you’ve seen the same 200-lines functions in too many <em>modeling_file.py</em>? Duplication isn’t inevitable.</p>
+<p>The “one‑model/one‑file” rule keeps every model readable and runnable. It also means identical code is copied hundreds of times. Maintenance hurts, contributor PRs snowball, and vision–language models especially end up in siloed forks.</p>
+<p>modular_*.py fixes the trade‑off, by auto-generating the modeling file from a modular file, which can use inheritance.</p>
+<p>With a small analyser I’ve mapped which models already share modular pieces and which 100‑plus still repeat themselves. Red nodes in the graph = lowest‑hanging fruit for refactor; blue = already modular.</p>
+<p>The result: contributors can focus on novel layers instead of boilerplate, reviews shrink from “new file diff” to “does this override make sense?”, and the codebase stays something you can actually open and read.</p>
+<p>If you maintain or ship models on top of Transformers, take a look at modular, in 2025 it’s how we keep shipping breadth without the bloat. 🛠️</p>
+
+    </d-article>
+    
+                        <script>
+                        function initializeTOC() {
+                            const article = document.querySelector('d-article');
+                            const toc = document.querySelector('d-contents');
+                            if (toc) {
+                                const headings = article.querySelectorAll('h1, h2, h3, h4');
+                                let ToC = '<nav role="navigation" class="l-text figcaption">';
+                                ToC += '<div class="toc-header"><span class="toc-title">Table of Contents</span></div>';
+                                ToC += '<div class="toc-content">';
+                                
+                                headings.forEach((heading, index) => {
+                                    const id = heading.id || 'heading-' + index;
+                                    if (!heading.id) heading.id = id;
+                                    const level = parseInt(heading.tagName.charAt(1));
+                                    const indent = level === 1 ? '' : 'style="margin-left: ' + ((level - 1) * 1.2) + 'em;"';
+                                    ToC += '<div ' + indent + '><a href="#' + id + '">' + heading.textContent + '</a></div>';
+                                });
+                                
+                                ToC += '</div></nav>';
+                                toc.innerHTML = ToC;
+                                toc.setAttribute('prerendered', 'true');
+                                
+                                // Extract tenet text for tooltips
+                                const tenetTooltips = {
+                                    'source-of-truth': 'We should be a source of truth for all model definitions. Model implementations should be reliable, reproducible, and faithful to the original performances.',
+                                    'one-model-one-file': 'All inference (and most of training, loss is separate, not a part of model) logic visible, top‑to‑bottom.',
+                                    'code-is-product': 'Optimize for reading, diffing, and tweaking, our users are power users. Variables can be explicit, full words, even several words, readability is primordial.',
+                                    'standardize-dont-abstract': 'If it\'s model behavior, keep it in the file; abstractions only for generic infra.',
+                                    'do-repeat-yourself': 'Copy when it helps users; keep successors in sync without centralizing behavior.',
+                                    'minimal-user-api': 'Config, model, preprocessing; from_pretrained, save_pretrained, push_to_hub. We want the least amount of codepaths.',
+                                    'backwards-compatibility': 'Evolve by additive standardization, never break public APIs.',
+                                    'consistent-public-surface': 'Same argument names, same outputs, hidden states and attentions exposed.',
+                                };
+
+                                // Add smooth scrolling and active state
+                                const tocLinks = document.querySelectorAll('d-contents a');
+                                tocLinks.forEach(link => {
+                                    const href = link.getAttribute('href');
+                                    const anchor = href ? href.substring(1) : '';
+                                    
+                                    // Add tooltip if this is a tenet link
+                                    if (tenetTooltips[anchor]) {
+                                        link.setAttribute('title', tenetTooltips[anchor]);
+                                        link.style.position = 'relative';
+                                    }
+                                    
+                                    link.addEventListener('click', function(e) {
+                                        e.preventDefault();
+                                        const target = document.querySelector(this.getAttribute('href'));
+                                        if (target) {
+                                            target.scrollIntoView({ behavior: 'smooth' });
+                                        }
+                                    });
+                                });
+                                
+                                // Update active state on scroll
+                                window.addEventListener('scroll', function() {
+                                    const scrollPos = window.scrollY + 100;
+                                    headings.forEach((heading) => {
+                                        const link = document.querySelector('d-contents a[href="#' + heading.id + '"]');
+                                        if (link) {
+                                            if (heading.offsetTop <= scrollPos && 
+                                                heading.offsetTop + heading.offsetHeight > scrollPos) {
+                                                link.classList.add('active');
+                                            } else {
+                                                link.classList.remove('active');
+                                            }
+                                        }
+                                    });
+                                });
+                            }
+                        }
+                        
+                        // Initialize Prism syntax highlighting
+                        function initializeSyntaxHighlighting() {
+                            if (typeof Prism !== 'undefined') {
+                                Prism.highlightAll();
+                            }
+                        }
+                        
+                        // Try multiple times to ensure it runs after distill.js
+                        document.addEventListener('DOMContentLoaded', function() {
+                            initializeTOC();
+                            initializeSyntaxHighlighting();
+                        });
+                        setTimeout(function() {
+                            initializeTOC();
+                            initializeSyntaxHighlighting();
+                        }, 100);
+                        setTimeout(function() {
+                            initializeTOC();
+                            initializeSyntaxHighlighting();
+                        }, 500);
+                        setTimeout(function() {
+                            initializeTOC();
+                            initializeSyntaxHighlighting();
+                        }, 1000);
+                        </script>
+</body>
+</html>

dist/static/d3_dependency_graph.html

@@ -0,0 +1,1902 @@
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<title>Transformers Modular Model Dependencies</title>
+<style>
+    /* Google‑font – small fallback cost & optional */
+    @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600&display=swap');
+
+    :root {
+        --base‑size: 60px; /* icon radius helper */
+    }
+
+    body {
+        font-family: 'Inter', Arial, sans-serif;
+        margin: 0;
+        overflow: hidden;
+        background-color: transparent; /* requested transparency */
+    }
+
+    svg {
+        width: 100vw;
+        height: 100vh;
+    }
+
+    .link {
+        stroke: #999;
+        stroke-opacity: 0.6;
+    }
+
+    .node-label {
+        fill: #333;
+        pointer-events: none;
+        text-anchor: middle;
+        font-weight: 600;
+    }
+
+    .link-label {
+        fill: #555;
+        font-size: 10px;
+        pointer-events: none;
+        text-anchor: middle;
+    }
+
+    .node.base path { fill: #ffbe0b; }
+    .node.derived circle { fill: #1f77b4; }
+
+    /* Legend styling */
+    #legend {
+        position: fixed;
+        top: 18px;
+        left: 18px;
+        font-size: 20px;
+        background: rgba(255,255,255,0.92);
+        padding: 18px 28px;
+        border-radius: 10px;
+        border: 1.5px solid #bbb;
+        font-family: 'Inter', Arial, sans-serif;
+        box-shadow: 0 2px 8px rgba(0,0,0,0.08);
+        z-index: 1000;
+    }
+</style>
+</head>
+<body>
+<div id="legend">🟡 base model (HF icon)<br>🔵 derived modular model<br>Edge label: #classes imported</div>
+<svg id="dependency-graph"></svg>
+<script src="https://d3js.org/d3.v7.min.js"></script>
+<script>
+const graphData = {
+    "nodes": [
+        {
+            "id": "aimv2",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "arcee",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "aria",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "auto",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "aya_vision",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "bamba",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "bart",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "beit",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "bigbird_pegasus",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "biogpt",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "bitnet",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "blip",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "blip_2",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "chameleon",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "clip",
+            "is_base": true,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "cohere",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "cohere2",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "colpali",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "colqwen2",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "conditional_detr",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "csm",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "d_fine",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "data2vec",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "deepseek_v2",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "deepseek_v3",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "deepseek_vl",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "deepseek_vl_hybrid",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "deformable_detr",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "depth_anything",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "detr",
+            "is_base": true,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "dia",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "diffllama",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "dinov2",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "dinov2_with_registers",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "doge",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "dots1",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "dpt",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "emu3",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "eomt",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "ernie4_5",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "ernie4_5_moe",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "esm",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "evolla",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "exaone4",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "falcon_h1",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "falcon_mamba",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "gemma",
+            "is_base": false,
+            "size": 1.3414634146341464
+        },
+        {
+            "id": "gemma2",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "gemma3",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "gemma3n",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "glm",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "glm4",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "glm4_moe",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "glm4v",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "got_ocr2",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "gpt_neox",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "granite",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "granitemoe",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "granitemoehybrid",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "granitemoeshared",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "grounding_dino",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "helium",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "hgnet_v2",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "hubert",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "idefics",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "idefics3",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "ijepa",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "image_processing_base",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "informer",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "instructblip",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "instructblipvideo",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "internvl",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "jamba",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "janus",
+            "is_base": false,
+            "size": 1.3902439024390243
+        },
+        {
+            "id": "kyutai_speech_to_text",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "lfm2",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "lightglue",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "llama",
+            "is_base": true,
+            "size": 3.0
+        },
+        {
+            "id": "llama4",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "llava",
+            "is_base": true,
+            "size": 1.3414634146341464
+        },
+        {
+            "id": "llava_next",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "llava_next_video",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "llava_onevision",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "mamba",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "mamba2",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "mask2former",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "maskformer",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "mbart",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "mimi",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "minimax",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "mistral",
+            "is_base": false,
+            "size": 1.3414634146341464
+        },
+        {
+            "id": "mistral3",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "mixtral",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "mlcd",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "modeling_outputs",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "modernbert",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "modernbert_decoder",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "moonshine",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "moshi",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "nemotron",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "olmo",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "olmo2",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "opt",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "owlv2",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "owlvit",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "paligemma",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "perception_lm",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "phi",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "phi3",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "phi4_multimodal",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "plbart",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "prompt_depth_anything",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "qwen2",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "qwen2_5_omni",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "qwen2_5_vl",
+            "is_base": false,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "qwen2_audio",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "qwen2_moe",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "qwen2_vl",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "qwen3",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "qwen3_moe",
+            "is_base": false,
+            "size": 1.2439024390243902
+        },
+        {
+            "id": "rt_detr",
+            "is_base": false,
+            "size": 1.1951219512195121
+        },
+        {
+            "id": "rt_detr_v2",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "sam",
+            "is_base": true,
+            "size": 1.146341463414634
+        },
+        {
+            "id": "sam_hq",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "sew",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "siglip",
+            "is_base": true,
+            "size": 1.2926829268292683
+        },
+        {
+            "id": "siglip2",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "smollm3",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "smolvlm",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "starcoder2",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "superglue",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "t5gemma",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "time_series_transformer",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "timesfm",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "timm_wrapper",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "unispeech",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "unispeech_sat",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "vipllava",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "vit",
+            "is_base": true,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "voxtral",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "wav2vec2",
+            "is_base": true,
+            "size": 1.3902439024390243
+        },
+        {
+            "id": "wav2vec2_bert",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "wav2vec2_conformer",
+            "is_base": false,
+            "size": 1.0975609756097562
+        },
+        {
+            "id": "wavlm",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "whisper",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "yolos",
+            "is_base": false,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "zamba",
+            "is_base": true,
+            "size": 1.048780487804878
+        },
+        {
+            "id": "zamba2",
+            "is_base": false,
+            "size": 1.146341463414634
+        }
+    ],
+    "links": [
+        {
+            "source": "llama",
+            "target": "doge",
+            "label": "8 classes"
+        },
+        {
+            "source": "mixtral",
+            "target": "doge",
+            "label": "2 classes"
+        },
+        {
+            "source": "mixtral",
+            "target": "minimax",
+            "label": "11 classes"
+        },
+        {
+            "source": "clip",
+            "target": "phi",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "phi",
+            "label": "8 classes"
+        },
+        {
+            "source": "qwen2_vl",
+            "target": "qwen2_5_vl",
+            "label": "15 classes"
+        },
+        {
+            "source": "ernie4_5",
+            "target": "ernie4_5_moe",
+            "label": "3 classes"
+        },
+        {
+            "source": "llama",
+            "target": "ernie4_5_moe",
+            "label": "2 classes"
+        },
+        {
+            "source": "mixtral",
+            "target": "ernie4_5_moe",
+            "label": "2 classes"
+        },
+        {
+            "source": "qwen3_moe",
+            "target": "ernie4_5_moe",
+            "label": "2 classes"
+        },
+        {
+            "source": "llama",
+            "target": "mistral",
+            "label": "10 classes"
+        },
+        {
+            "source": "llama",
+            "target": "gpt_neox",
+            "label": "4 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "phi3",
+            "label": "7 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "unispeech",
+            "label": "9 classes"
+        },
+        {
+            "source": "llama",
+            "target": "olmo",
+            "label": "8 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "helium",
+            "label": "3 classes"
+        },
+        {
+            "source": "granite",
+            "target": "helium",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "helium",
+            "label": "5 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "bitnet",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "bitnet",
+            "label": "7 classes"
+        },
+        {
+            "source": "maskformer",
+            "target": "mask2former",
+            "label": "1 classes"
+        },
+        {
+            "source": "jamba",
+            "target": "falcon_h1",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "falcon_h1",
+            "label": "7 classes"
+        },
+        {
+            "source": "mamba2",
+            "target": "falcon_h1",
+            "label": "4 classes"
+        },
+        {
+            "source": "llava",
+            "target": "got_ocr2",
+            "label": "6 classes"
+        },
+        {
+            "source": "sam",
+            "target": "got_ocr2",
+            "label": "5 classes"
+        },
+        {
+            "source": "esm",
+            "target": "evolla",
+            "label": "9 classes"
+        },
+        {
+            "source": "llama",
+            "target": "evolla",
+            "label": "6 classes"
+        },
+        {
+            "source": "gemma2",
+            "target": "gemma3n",
+            "label": "6 classes"
+        },
+        {
+            "source": "gemma3",
+            "target": "gemma3n",
+            "label": "6 classes"
+        },
+        {
+            "source": "paligemma",
+            "target": "gemma3n",
+            "label": "4 classes"
+        },
+        {
+            "source": "timm_wrapper",
+            "target": "gemma3n",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "csm",
+            "label": "8 classes"
+        },
+        {
+            "source": "owlvit",
+            "target": "owlv2",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "zamba2",
+            "label": "2 classes"
+        },
+        {
+            "source": "mamba2",
+            "target": "zamba2",
+            "label": "3 classes"
+        },
+        {
+            "source": "zamba",
+            "target": "zamba2",
+            "label": "10 classes"
+        },
+        {
+            "source": "blip",
+            "target": "janus",
+            "label": "1 classes"
+        },
+        {
+            "source": "blip_2",
+            "target": "janus",
+            "label": "1 classes"
+        },
+        {
+            "source": "chameleon",
+            "target": "janus",
+            "label": "6 classes"
+        },
+        {
+            "source": "idefics",
+            "target": "janus",
+            "label": "2 classes"
+        },
+        {
+            "source": "llama",
+            "target": "janus",
+            "label": "1 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "janus",
+            "label": "4 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "wav2vec2_conformer",
+            "label": "13 classes"
+        },
+        {
+            "source": "clip",
+            "target": "mlcd",
+            "label": "7 classes"
+        },
+        {
+            "source": "llama",
+            "target": "mlcd",
+            "label": "1 classes"
+        },
+        {
+            "source": "qwen2_vl",
+            "target": "mlcd",
+            "label": "2 classes"
+        },
+        {
+            "source": "gemma2",
+            "target": "gemma3",
+            "label": "10 classes"
+        },
+        {
+            "source": "paligemma",
+            "target": "gemma3",
+            "label": "4 classes"
+        },
+        {
+            "source": "instructblip",
+            "target": "instructblipvideo",
+            "label": "9 classes"
+        },
+        {
+            "source": "auto",
+            "target": "instructblipvideo",
+            "label": "1 classes"
+        },
+        {
+            "source": "glm4",
+            "target": "glm4v",
+            "label": "3 classes"
+        },
+        {
+            "source": "qwen2_5_vl",
+            "target": "glm4v",
+            "label": "16 classes"
+        },
+        {
+            "source": "llama",
+            "target": "exaone4",
+            "label": "10 classes"
+        },
+        {
+            "source": "olmo2",
+            "target": "exaone4",
+            "label": "2 classes"
+        },
+        {
+            "source": "cohere",
+            "target": "glm4_moe",
+            "label": "1 classes"
+        },
+        {
+            "source": "deepseek_v3",
+            "target": "glm4_moe",
+            "label": "7 classes"
+        },
+        {
+            "source": "gpt_neox",
+            "target": "glm4_moe",
+            "label": "1 classes"
+        },
+        {
+            "source": "detr",
+            "target": "conditional_detr",
+            "label": "1 classes"
+        },
+        {
+            "source": "detr",
+            "target": "grounding_dino",
+            "label": "1 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "qwen3",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "qwen3",
+            "label": "1 classes"
+        },
+        {
+            "source": "qwen2",
+            "target": "qwen3",
+            "label": "10 classes"
+        },
+        {
+            "source": "llava_next",
+            "target": "llava_onevision",
+            "label": "1 classes"
+        },
+        {
+            "source": "llava_next_video",
+            "target": "llava_onevision",
+            "label": "9 classes"
+        },
+        {
+            "source": "llava",
+            "target": "vipllava",
+            "label": "5 classes"
+        },
+        {
+            "source": "detr",
+            "target": "deformable_detr",
+            "label": "1 classes"
+        },
+        {
+            "source": "llava",
+            "target": "perception_lm",
+            "label": "5 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "wavlm",
+            "label": "9 classes"
+        },
+        {
+            "source": "llama",
+            "target": "glm",
+            "label": "4 classes"
+        },
+        {
+            "source": "phi3",
+            "target": "glm",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "timesfm",
+            "label": "1 classes"
+        },
+        {
+            "source": "phi4_multimodal",
+            "target": "timesfm",
+            "label": "1 classes"
+        },
+        {
+            "source": "image_processing_base",
+            "target": "dpt",
+            "label": "1 classes"
+        },
+        {
+            "source": "beit",
+            "target": "dpt",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "gemma",
+            "label": "5 classes"
+        },
+        {
+            "source": "llama",
+            "target": "kyutai_speech_to_text",
+            "label": "1 classes"
+        },
+        {
+            "source": "mimi",
+            "target": "kyutai_speech_to_text",
+            "label": "1 classes"
+        },
+        {
+            "source": "moshi",
+            "target": "kyutai_speech_to_text",
+            "label": "2 classes"
+        },
+        {
+            "source": "llama",
+            "target": "granite",
+            "label": "5 classes"
+        },
+        {
+            "source": "idefics3",
+            "target": "smolvlm",
+            "label": "9 classes"
+        },
+        {
+            "source": "granitemoe",
+            "target": "granitemoeshared",
+            "label": "4 classes"
+        },
+        {
+            "source": "glm",
+            "target": "moonshine",
+            "label": "3 classes"
+        },
+        {
+            "source": "llama",
+            "target": "moonshine",
+            "label": "3 classes"
+        },
+        {
+            "source": "whisper",
+            "target": "moonshine",
+            "label": "2 classes"
+        },
+        {
+            "source": "llava",
+            "target": "aya_vision",
+            "label": "6 classes"
+        },
+        {
+            "source": "deepseek_v3",
+            "target": "dots1",
+            "label": "5 classes"
+        },
+        {
+            "source": "qwen3",
+            "target": "dots1",
+            "label": "6 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "starcoder2",
+            "label": "9 classes"
+        },
+        {
+            "source": "modeling_outputs",
+            "target": "sam_hq",
+            "label": "1 classes"
+        },
+        {
+            "source": "sam",
+            "target": "sam_hq",
+            "label": "15 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "wav2vec2_bert",
+            "label": "3 classes"
+        },
+        {
+            "source": "wav2vec2_conformer",
+            "target": "wav2vec2_bert",
+            "label": "6 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "mixtral",
+            "label": "9 classes"
+        },
+        {
+            "source": "chameleon",
+            "target": "emu3",
+            "label": "2 classes"
+        },
+        {
+            "source": "llama",
+            "target": "emu3",
+            "label": "5 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "emu3",
+            "label": "1 classes"
+        },
+        {
+            "source": "paligemma",
+            "target": "colpali",
+            "label": "3 classes"
+        },
+        {
+            "source": "phi3",
+            "target": "phi4_multimodal",
+            "label": "7 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "phi4_multimodal",
+            "label": "9 classes"
+        },
+        {
+            "source": "qwen2_audio",
+            "target": "voxtral",
+            "label": "4 classes"
+        },
+        {
+            "source": "idefics",
+            "target": "deepseek_vl",
+            "label": "2 classes"
+        },
+        {
+            "source": "janus",
+            "target": "deepseek_vl",
+            "label": "4 classes"
+        },
+        {
+            "source": "glm",
+            "target": "glm4",
+            "label": "4 classes"
+        },
+        {
+            "source": "phi3",
+            "target": "glm4",
+            "label": "1 classes"
+        },
+        {
+            "source": "gemma2",
+            "target": "t5gemma",
+            "label": "9 classes"
+        },
+        {
+            "source": "auto",
+            "target": "lightglue",
+            "label": "1 classes"
+        },
+        {
+            "source": "clip",
+            "target": "lightglue",
+            "label": "1 classes"
+        },
+        {
+            "source": "cohere",
+            "target": "lightglue",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "lightglue",
+            "label": "2 classes"
+        },
+        {
+            "source": "superglue",
+            "target": "lightglue",
+            "label": "2 classes"
+        },
+        {
+            "source": "llava_next",
+            "target": "llava_next_video",
+            "label": "7 classes"
+        },
+        {
+            "source": "rt_detr",
+            "target": "hgnet_v2",
+            "label": "1 classes"
+        },
+        {
+            "source": "deepseek_vl",
+            "target": "deepseek_vl_hybrid",
+            "label": "7 classes"
+        },
+        {
+            "source": "idefics",
+            "target": "deepseek_vl_hybrid",
+            "label": "2 classes"
+        },
+        {
+            "source": "sam",
+            "target": "deepseek_vl_hybrid",
+            "label": "2 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "data2vec",
+            "label": "11 classes"
+        },
+        {
+            "source": "depth_anything",
+            "target": "prompt_depth_anything",
+            "label": "7 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "modernbert",
+            "label": "2 classes"
+        },
+        {
+            "source": "bamba",
+            "target": "lfm2",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "lfm2",
+            "label": "8 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "sew",
+            "label": "11 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "hubert",
+            "label": "7 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "gemma2",
+            "label": "9 classes"
+        },
+        {
+            "source": "detr",
+            "target": "rt_detr",
+            "label": "2 classes"
+        },
+        {
+            "source": "rt_detr",
+            "target": "d_fine",
+            "label": "12 classes"
+        },
+        {
+            "source": "rt_detr_v2",
+            "target": "d_fine",
+            "label": "1 classes"
+        },
+        {
+            "source": "llava",
+            "target": "mistral3",
+            "label": "6 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "mistral3",
+            "label": "1 classes"
+        },
+        {
+            "source": "modernbert",
+            "target": "modernbert_decoder",
+            "label": "6 classes"
+        },
+        {
+            "source": "llama",
+            "target": "aria",
+            "label": "8 classes"
+        },
+        {
+            "source": "llava",
+            "target": "aria",
+            "label": "4 classes"
+        },
+        {
+            "source": "llava_next",
+            "target": "aria",
+            "label": "1 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "siglip2",
+            "label": "16 classes"
+        },
+        {
+            "source": "llama",
+            "target": "arcee",
+            "label": "5 classes"
+        },
+        {
+            "source": "nemotron",
+            "target": "arcee",
+            "label": "1 classes"
+        },
+        {
+            "source": "mamba",
+            "target": "falcon_mamba",
+            "label": "10 classes"
+        },
+        {
+            "source": "llama",
+            "target": "deepseek_v2",
+            "label": "9 classes"
+        },
+        {
+            "source": "llama4",
+            "target": "deepseek_v2",
+            "label": "1 classes"
+        },
+        {
+            "source": "bart",
+            "target": "informer",
+            "label": "1 classes"
+        },
+        {
+            "source": "time_series_transformer",
+            "target": "informer",
+            "label": "12 classes"
+        },
+        {
+            "source": "colpali",
+            "target": "colqwen2",
+            "label": "3 classes"
+        },
+        {
+            "source": "bamba",
+            "target": "granitemoehybrid",
+            "label": "4 classes"
+        },
+        {
+            "source": "granitemoeshared",
+            "target": "granitemoehybrid",
+            "label": "7 classes"
+        },
+        {
+            "source": "bart",
+            "target": "plbart",
+            "label": "5 classes"
+        },
+        {
+            "source": "bigbird_pegasus",
+            "target": "plbart",
+            "label": "1 classes"
+        },
+        {
+            "source": "mbart",
+            "target": "plbart",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "qwen3_moe",
+            "label": "4 classes"
+        },
+        {
+            "source": "mixtral",
+            "target": "qwen3_moe",
+            "label": "3 classes"
+        },
+        {
+            "source": "qwen2_moe",
+            "target": "qwen3_moe",
+            "label": "1 classes"
+        },
+        {
+            "source": "qwen3",
+            "target": "qwen3_moe",
+            "label": "1 classes"
+        },
+        {
+            "source": "clip",
+            "target": "internvl",
+            "label": "1 classes"
+        },
+        {
+            "source": "janus",
+            "target": "internvl",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "internvl",
+            "label": "1 classes"
+        },
+        {
+            "source": "llava",
+            "target": "internvl",
+            "label": "5 classes"
+        },
+        {
+            "source": "glm",
+            "target": "ernie4_5",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "ernie4_5",
+            "label": "4 classes"
+        },
+        {
+            "source": "dinov2",
+            "target": "eomt",
+            "label": "4 classes"
+        },
+        {
+            "source": "mask2former",
+            "target": "eomt",
+            "label": "2 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "eomt",
+            "label": "1 classes"
+        },
+        {
+            "source": "vit",
+            "target": "eomt",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "dia",
+            "label": "4 classes"
+        },
+        {
+            "source": "phi3",
+            "target": "dia",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "deepseek_v3",
+            "label": "9 classes"
+        },
+        {
+            "source": "jamba",
+            "target": "bamba",
+            "label": "2 classes"
+        },
+        {
+            "source": "llama",
+            "target": "bamba",
+            "label": "6 classes"
+        },
+        {
+            "source": "mamba2",
+            "target": "bamba",
+            "label": "4 classes"
+        },
+        {
+            "source": "llama",
+            "target": "olmo2",
+            "label": "3 classes"
+        },
+        {
+            "source": "olmo",
+            "target": "olmo2",
+            "label": "7 classes"
+        },
+        {
+            "source": "clip",
+            "target": "aimv2",
+            "label": "3 classes"
+        },
+        {
+            "source": "llama",
+            "target": "aimv2",
+            "label": "2 classes"
+        },
+        {
+            "source": "siglip",
+            "target": "aimv2",
+            "label": "6 classes"
+        },
+        {
+            "source": "gemma",
+            "target": "diffllama",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "diffllama",
+            "label": "8 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "diffllama",
+            "label": "1 classes"
+        },
+        {
+            "source": "rt_detr",
+            "target": "rt_detr_v2",
+            "label": "6 classes"
+        },
+        {
+            "source": "vit",
+            "target": "ijepa",
+            "label": "3 classes"
+        },
+        {
+            "source": "llama",
+            "target": "smollm3",
+            "label": "9 classes"
+        },
+        {
+            "source": "qwen2",
+            "target": "smollm3",
+            "label": "1 classes"
+        },
+        {
+            "source": "cohere",
+            "target": "cohere2",
+            "label": "8 classes"
+        },
+        {
+            "source": "gemma2",
+            "target": "cohere2",
+            "label": "1 classes"
+        },
+        {
+            "source": "bart",
+            "target": "biogpt",
+            "label": "3 classes"
+        },
+        {
+            "source": "opt",
+            "target": "biogpt",
+            "label": "1 classes"
+        },
+        {
+            "source": "detr",
+            "target": "yolos",
+            "label": "1 classes"
+        },
+        {
+            "source": "wav2vec2",
+            "target": "unispeech_sat",
+            "label": "11 classes"
+        },
+        {
+            "source": "llama",
+            "target": "qwen2",
+            "label": "10 classes"
+        },
+        {
+            "source": "mistral",
+            "target": "qwen2",
+            "label": "1 classes"
+        },
+        {
+            "source": "llama",
+            "target": "cohere",
+            "label": "6 classes"
+        },
+        {
+            "source": "llama",
+            "target": "qwen2_5_omni",
+            "label": "1 classes"
+        },
+        {
+            "source": "qwen2_5_vl",
+            "target": "qwen2_5_omni",
+            "label": "8 classes"
+        },
+        {
+            "source": "qwen2_audio",
+            "target": "qwen2_5_omni",
+            "label": "2 classes"
+        },
+        {
+            "source": "qwen2_vl",
+            "target": "qwen2_5_omni",
+            "label": "1 classes"
+        },
+        {
+            "source": "dinov2",
+            "target": "dinov2_with_registers",
+            "label": "6 classes"
+        }
+    ]
+};
+const hfLogoPath = "M21.2,6.7c-0.2-0.2-0.5-0.3-0.8-0.3H3.6C3.3,6.4,3,6.5,2.8,6.7s-0.3,0.5-0.3,0.8v10.8c0,0.3,0.1,0.5,0.3,0.8 c0.2,0.2,0.5,0.3,0.8,0.3h16.8c0.3,0,0.5-0.1,0.8-0.3c0.2-0.2,0.3-0.5,0.3-0.8V7.5C21.5,7.2,21.4,6.9,21.2,6.7z M12,17.8L5.9,9.4h3.1 V8.3h6v1.1h3.1L12,17.8z"; // kept for potential future use
+
+const width  = window.innerWidth;
+const height = window.innerHeight;
+
+const svg = d3.select('#dependency-graph')
+    .call(
+        d3.zoom().on('zoom', (event) => {
+            g.attr('transform', event.transform);
+        })
+    );
+
+const g = svg.append('g');
+
+// Forces – tweaked for tighter graph
+const simulation = d3.forceSimulation(graphData.nodes)
+    .force('link',   d3.forceLink(graphData.links).id(d => d.id).distance(500))
+    .force('charge', d3.forceManyBody().strength(-500))
+    .force('center', d3.forceCenter(width / 2, height / 2))
+    .force('collide', d3.forceCollide(0.01 * parseFloat(getComputedStyle(document.documentElement).getPropertyValue('--base‑size'))));
+
+// Links
+const link = g.append('g')
+    .selectAll('line')
+    .data(graphData.links)
+    .join('line')
+    .attr('class', 'link')
+    .attr('stroke-width', 1.5);
+
+// Link‑labels (#classes)
+const linkLabel = g.append('g')
+    .selectAll('text')
+    .data(graphData.links)
+    .join('text')
+    .attr('class', 'link-label')
+    .text(d => d.label);
+
+// Nodes (base vs derived)
+const node = g.append('g')
+    .selectAll('g')
+    .data(graphData.nodes)
+    .join('g')
+    .attr('class', d => d.is_base ? 'node base' : 'node derived')
+    .call(d3.drag()
+        .on('start', dragstarted)
+        .on('drag', dragged)
+        .on('end', dragended)
+    );
+
+// Base‑model icon (styled circle instead of external image)
+node.filter(d => d.is_base)
+    .append('circle')
+    .attr('r', parseFloat(getComputedStyle(document.documentElement).getPropertyValue('--base‑size')) / 2)
+    .attr('fill', '#FFD21E')
+    .attr('stroke', '#FF9D00')
+    .attr('stroke-width', 3);
+
+// Add 🤗 emoji as text for base models
+node.filter(d => d.is_base)
+    .append('text')
+    .attr('text-anchor', 'middle')
+    .attr('dy', '0.35em')
+    .style('font-size', '24px')
+    .text('🤗');
+
+// Base‑model label (below icon)
+node.filter(d => d.is_base)
+    .append('text')
+    .attr('class', 'node-label')
+    .attr('y', parseFloat(getComputedStyle(document.documentElement).getPropertyValue('--base‑size')) / 2 + 8)
+    .style('font-size', '40px')
+    .text(d => d.id);
+
+// Derived‑model circle + label w/ background rect
+const derived = node.filter(d => !d.is_base);
+
+derived.append('circle')
+  .attr('r', d => 20 * d.size);  // scaled
+
+const labelGroup = derived.append('g').attr('class', 'label-group');
+labelGroup.append('rect')
+    .attr('x', -45)
+    .attr('y', -18)
+    .attr('width', 90)
+    .attr('height', 36)
+    .attr('rx', 8)
+    .attr('fill', '#fffbe6')
+    .attr('stroke', '#ccc');
+labelGroup.append('text')
+    .attr('class', 'node-label')
+    .attr('dy', '0.35em')
+    .style('font-size', '18px')
+    .text(d => d.id);
+
+// Tick
+simulation.on('tick', () => {
+    link.attr('x1', d => d.source.x)
+        .attr('y1', d => d.source.y)
+        .attr('x2', d => d.target.x)
+        .attr('y2', d => d.target.y);
+
+    linkLabel.attr('x', d => (d.source.x + d.target.x) / 2)
+             .attr('y', d => (d.source.y + d.target.y) / 2);
+
+    node.attr('transform', d => `translate(${d.x}, ${d.y})`);
+});
+
+// Drag helpers
+function dragstarted(event, d) {
+    if (!event.active) simulation.alphaTarget(0.3).restart();
+    d.fx = d.x; d.fy = d.y;
+}
+function dragged(event, d) {
+    d.fx = event.x; d.fy = event.y;
+}
+function dragended(event, d) {
+    if (!event.active) simulation.alphaTarget(0);
+    d.fx = null; d.fy = null;
+}
+</script>
+</body>
+</html>

dist/style.css

@@ -0,0 +1,741 @@
+/* style.css - Transformers Playthrough */
+
+/* Import ultrascale-playbook base styles and add transformers-specific styling */
+/* Define colors */
+:root {
+    --distill-gray: rgb(107, 114, 128);
+    --distill-gray-light: rgb(185, 185, 185);
+    --distill-gray-lighter: rgb(228, 228, 228);
+    --distill-gray-lightest: rgb(245, 245, 245);
+    --distill-blue: #007BFF;
+}
+
+/* Container for the controls */
+[id^="plot-"] {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: 15px; /* Adjust the gap between controls as needed */
+}
+[id^="plot-"] figure {
+    margin-bottom: 0px;
+    margin-top: 0px;
+    padding: 0px;
+}
+.plotly_caption {
+    font-style: italic;
+    margin-top: 10px;
+}
+
+.plotly_controls {
+    display: flex;
+    flex-wrap: wrap;
+    flex-direction: row;
+    justify-content: center;
+    align-items: flex-start;
+    gap: 30px;
+}
+
+
+.plotly_input_container {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    gap: 10px;
+}
+
+/* Style for the select dropdown */
+.plotly_input_container > select {
+    padding: 2px 4px;
+    /* border: 1px solid #ccc; */
+    line-height: 1.5em;
+    text-align: center;
+    border-radius: 4px;
+    font-size: 12px;
+    background-color: var(--distill-gray-lightest);
+    outline: none;
+}
+
+/* Style for the range input */
+
+.plotly_slider {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+}
+
+.plotly_slider > input[type="range"] {
+    -webkit-appearance: none;
+    height: 2px;
+    background: var(--distill-gray-light);
+    border-radius: 5px;
+    outline: none;
+}
+
+.plotly_slider > span {
+    font-size: 14px;
+    line-height: 1.6em;
+    min-width: 16px;
+}
+
+.plotly_slider > input[type="range"]::-webkit-slider-thumb {
+    -webkit-appearance: none;
+    appearance: none;
+    width: 18px;
+    height: 18px;
+    border-radius: 50%;
+    background: var(--distill-blue);
+    cursor: pointer;
+}
+
+.plotly_slider > input[type="range"]::-moz-range-thumb {
+    width: 18px;
+    height: 18px;
+    border-radius: 50%;
+    background: var(--distill-blue);
+    cursor: pointer;
+}
+
+/* Style for the labels */
+.plotly_input_container > label {
+    font-size: 14px;
+    font-weight: bold;
+}
+
+.main-plot-container {
+    margin-top: 21px;
+    margin-bottom: 35px;
+}
+
+.main-plot-container > figure {
+    display: block !important;
+    /* Let this be handled by graph-container */
+    margin-bottom: 0px;
+    margin-top: 0px;
+}
+.main-plot-container > div {
+    display: none !important;
+}
+
+
+@media (min-width: 768px) {
+    .main-plot-container > figure {
+        display: none !important;
+    }
+    .main-plot-container > div {
+        display: flex !important;
+    }
+}
+
+d-byline .byline {
+  grid-template-columns: 1fr;
+  grid-column: text;
+  font-size: 0.9rem;
+  line-height: 1.8em;
+}
+
+@media (min-width: 768px) {
+  d-byline .byline {
+    grid-template-columns: 5fr 1fr 1fr;
+  }
+}
+
+#title-plot {
+    margin-top: 0px;
+    margin-bottom: 0px;
+}
+
+d-contents > nav a.active {
+    text-decoration: underline;
+}
+
+@media (max-width: 1199px) {
+    d-contents {
+        display: none;
+        background: white;
+        justify-self: start;
+        align-self: start;
+        padding-bottom: 0.5em;
+        margin-bottom: 1em;
+        padding-left: 0.25em;
+        border-bottom: 1px solid rgba(0, 0, 0, 0.1);
+        border-bottom-width: 1px;
+        border-bottom-style: solid;
+        border-bottom-color: rgba(0, 0, 0, 0.1);
+        overflow-y: scroll;
+        height: calc(100vh - 40px);
+        scrollbar-width: none;
+        z-index: -100;
+    }
+}
+
+d-contents a:hover {
+    border-bottom: none;
+}
+
+toc-title {
+    font-weight: bold;
+    font-size: 1.2em;
+    color: #333;
+}
+
+toggle-icon {
+    transition: transform 0.3s;
+}
+
+toggle-icon.collapsed {
+    transform: rotate(90deg);
+}
+
+.toc-content {
+    margin-top: 15px;
+    overflow: hidden;
+    /* max-height: 1000px; */
+    transition: max-height 0.3s ease-out;
+}
+
+.toc-content.collapsed {
+    max-height: 0;
+    margin-top: 0;
+}
+
+@media (min-width: 1200px) {
+    d-article {
+        /* Ensure d-article does not prevent sticky positioning */
+        overflow: visible;
+    }
+
+    d-contents {
+        align-self: start;
+        background: white;
+        grid-column-start: 1 !important;
+        grid-column-end: 4 !important;
+        grid-row: auto / span 6;
+        justify-self: end;
+        margin-top: 0em;
+        padding-right: 3em;
+        padding-left: 2em;
+        /* border-right: 1px solid rgba(0, 0, 0, 0.1);
+        border-right-width: 1px;
+        border-right-style: solid;
+        border-right-color: rgba(0, 0, 0, 0.1); */
+        position: -webkit-sticky; /* For Safari */
+        position: sticky;
+        top: 10px; /* Adjust this value if needed */
+        overflow-y: auto;
+        height: calc(100vh - 40px);
+        scrollbar-width: none;
+        transition: max-height 0.3s ease-out;
+        z-index: -100;
+    }
+}
+
+d-contents nav h3 {
+    margin-top: 0;
+    margin-bottom: 1em;
+}
+
+d-contents nav div div {
+    color: rgba(0, 0, 0, 0.8);
+    font-weight: bold;
+}
+
+d-contents nav a {
+    color: rgba(0, 0, 0, 0.8);
+    border-bottom: none;
+    text-decoration: none;
+}
+
+d-contents li {
+    list-style-type: none;
+}
+
+d-contents ul, d-article d-contents ul {
+    padding-left: 1em;
+}
+
+d-contents nav ul li {
+    margin-bottom: .25em;
+}
+
+d-contents nav a:hover {
+    text-decoration: underline solid rgba(0, 0, 0, 0.6);
+}
+
+d-contents nav ul {
+    margin-top: 0;
+    margin-bottom: 6px;
+}
+
+
+d-contents nav > div {
+    display: block;
+    outline: none;
+    margin-bottom: 0.5em;
+}
+
+d-contents nav > div > a {
+    font-size: 13px;
+    font-weight: 600;
+}
+
+d-article aside {
+    margin-bottom: 1em;
+}
+
+d-article img {
+    max-width: 100%;
+}
+
+@media (min-width: 768px) {
+    d-article aside {
+        margin-bottom: 0;
+    }
+}
+
+d-contents nav > div > a:hover,
+d-contents nav > ul > li > a:hover {
+    text-decoration: none;
+}
+
+.note-box {
+    background-color: #f6f8fa;
+    border-left: 4px solid #444444;
+    padding: 1rem;
+    margin: 1rem 0;  /* Keep this modest margin */
+    border-radius: 6px;
+    /* Add this to ensure the box only takes up needed space */
+    display: inline-block;
+}
+
+.note-box-title {
+    margin: 0;
+    color: #444444;
+    font-weight: 600;
+    font-size: 1em;
+}
+
+.note-box-content {
+    margin-top: 0.5rem;
+    margin-bottom: 0;  /* Ensure no bottom margin */
+    color: #24292f;
+    font-size: 0.9em;
+    line-height: 1.5em;
+}
+
+/* For dark mode support */
+@media (prefers-color-scheme: dark) {
+    .note-box {
+        background-color: #1c1c1c;
+        border-left-color: #888888;
+    }
+    .note-box-title {
+        color: #888888;
+    }
+    .note-box-content {
+        color: #d4d4d4;
+    }
+}
+
+d-article {
+    font-size: 1.0em;
+}
+
+.figure-legend {
+    font-size: 0.9em;
+    font-style: italic;
+    color: var(--distill-gray);
+    line-height: 1.5em;
+}
+
+d-code {
+    font-size: 12px;
+}
+
+.large-image-background {
+        width: 100vw;
+        padding-top: 10px;
+        padding-bottom: 10px;
+        margin-left: calc(-50vw + 50%);
+        margin-right: calc(-50vw + 50%);
+        background: white;
+        height: fit-content; /* This will make it match the image height */
+        display: flex;
+        justify-content: center; /* This will center your image */
+}
+
+.large-image-background-transparent {
+    /* width: 100vw; */
+    padding-top: 10px;
+    padding-bottom: 10px;
+    /* margin-left: calc(-50vw + 50%); */
+    margin-left:-100px;
+    margin-right: -100px;
+    /* margin-right: calc(-50vw + 50%); */
+    /* background: white; */
+    height: fit-content; /* This will make it match the image height */
+    display: flex;
+    justify-content: center; /* This will center your image */
+}
+
+.boxed-image {
+    padding: 0.5rem;
+    background: white;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+d-article li {
+    margin-bottom: 0.0em;
+}
+
+d-article ul ul {
+    margin-bottom: 0.0em;
+}
+
+d-article ol ol {
+    margin-bottom: 0.0em;
+}
+
+d-article hr {
+    grid-column: text;
+}
+
+/* Memory visualization */
+#graph-all {
+    min-width: 500px;
+    margin-right: 10px;
+    margin-bottom: 2rem;
+    padding: 0.5rem;
+    background: #f9fafb;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+
+/* Main container styles */
+#controls {
+    max-width: 1200px;
+    /* margin: 2rem auto; */
+    margin-bottom: 2rem;
+    margin-left: 10px;
+    padding: 0.6rem;
+    background: #f9fafb;
+    border-radius: 12px;
+    border: 1px solid #e5e7eb;
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+}
+
+/* Grid layout */
+#controls {
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    /* gap: 2rem; */
+}
+
+/* Cell styles */
+.cell {
+    margin-bottom: 0.2rem;
+}
+
+/* Label styles */
+label {
+    display: block;
+    /* margin-bottom: 0.5rem; */
+    font-size: 0.8rem;
+    font-weight: 500;
+    color: #374151;
+}
+
+/* Input container for range + number combination */
+.input-container {
+    display: flex;
+    gap: 1rem;
+    align-items: center;
+}
+
+/* Range input styling */
+input[type="range"] {
+    flex: 1;
+    height: 6px;
+    background: #e5e7eb;
+    border-radius: 3px;
+    appearance: none;
+    outline: none;
+}
+
+input[type="range"]::-webkit-slider-thumb {
+    appearance: none;
+    width: 16px;
+    height: 16px;
+    background: #3b82f6;
+    border-radius: 50%;
+    cursor: pointer;
+    transition: background 0.15s ease;
+}
+
+input[type="range"]::-webkit-slider-thumb:hover {
+    background: #2563eb;
+}
+
+/* Number input styling */
+input[type="number"] {
+    width: 80px;
+    padding: 0.5rem;
+    border: 1px solid #e5e7eb;
+    border-radius: 6px;
+    font-size: 0.9rem;
+    color: #374151;
+}
+
+/* Select styling */
+select {
+    width: 100%;
+    padding: 0.5rem;
+    border: 1px solid #e5e7eb;
+    border-radius: 6px;
+    background: white;
+    font-size: 0.9rem;
+    color: #374151;
+    cursor: pointer;
+}
+
+/* Checkbox styling */
+input[type="checkbox"] {
+    width: 1.2rem;
+    height: 1.2rem;
+    margin-right: 0.5rem;
+    border: 2px solid #e5e7eb;
+    border-radius: 4px;
+    cursor: pointer;
+}
+
+/* Column specific styles */
+.column-1 {
+    padding-right: 0.5rem;
+}
+
+.column-2 {
+    padding-left: 0.5rem;
+}
+
+/* Checkbox container */
+.checkbox-container {
+    display: flex;
+    align-items: center;
+    margin-bottom: 1rem;
+}
+
+/* Memory visualization styles */
+.memory-block {
+    background: #fff;
+    border-radius: 8px;
+    padding: 1rem;
+    margin-bottom: 1rem;
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
+}
+
+.memory-title {
+    font-size: 1.1rem;
+    font-weight: 500;
+    color: #374151;
+    margin-bottom: 0.5rem;
+}
+
+.memory-value {
+    font-size: 1.5rem;
+    font-weight: 600;
+    color: #3b82f6;
+}
+
+/* Responsive adjustments */
+@media (max-width: 768px) {
+    #controls {
+        grid-template-columns: 1fr;
+        padding: 1rem;
+    }
+
+    .column-1, .column-2 {
+        padding: 0;
+    }
+}
+
+/* Hover states and transitions */
+input:hover, select:hover {
+    border-color: #3b82f6;
+}
+
+input:focus, select:focus {
+    border-color: #2563eb;
+    outline: none;
+    box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.1);
+}
+
+/* Add smooth transitions */
+input, select, button {
+    transition: all 0.15s ease;
+}
+
+/* Preset dropdown special styling */
+select[name="presets"] {
+    background-color: #f3f4f6;
+    font-weight: 500;
+}
+
+/* Memory graph enhancements */
+.activation-memory {
+    background: #dbeafe;
+    padding: 1rem;
+    border-radius: 8px;
+    margin-bottom: 1rem;
+}
+
+.gradient-memory {
+    background: #ede9fe;
+    padding: 1rem;
+    border-radius: 8px;
+}
+
+.order-button-second {
+            background: linear-gradient(135deg, #6DB4C4, #D4A5B8);
+            color: white;
+            font-size: 18px;
+            font-weight: 600;
+            padding: 20px 20px;
+            border: none;
+            border-radius: 12px;
+            cursor: pointer;
+            text-transform: uppercase;
+            letter-spacing: 1px;
+            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+            transition: all 0.3s ease;
+            position: relative;
+            overflow: hidden;
+        }
+.order-button-second:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
+}
+
+.order-button:active {
+    transform: translateY(0);
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
+}
+
+.order-button-second::before {
+    content: '';
+    position: absolute;
+    top: 0;
+    left: -100%;
+    width: 100%;
+    height: 100%;
+    background: linear-gradient(135deg, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0));
+    transition: left 0.5s ease;
+}
+
+.order-button-second:hover::before {
+    left: 100%;
+}
+
+.order-button {
+            background: linear-gradient(135deg, #6DB4C4, #D4A5B8);
+            color: white;
+            font-size: 18px;
+            font-weight: 600;
+            padding: 16px 32px;
+            border: none;
+            border-radius: 12px;
+            cursor: pointer;
+            text-transform: uppercase;
+            letter-spacing: 1px;
+            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+            transition: all 0.3s ease;
+            position: relative;
+            overflow: hidden;
+        }
+
+.order-button:hover {
+    transform: translateY(-2px);
+    box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
+}
+
+.order-button:active {
+    transform: translateY(0);
+    box-shadow: 0 2px 10px rgba(0, 0, 0, 0.2);
+}
+
+.order-button::before {
+    content: '';
+    position: absolute;
+    top: 0;
+    left: -100%;
+    width: 100%;
+    height: 100%;
+    background: linear-gradient(135deg, rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 0));
+    transition: left 0.5s ease;
+}
+
+.order-button:hover::before {
+    left: 100%;
+}
+.order-button-container-second {
+    /* display: flex; */
+    justify-content: center;
+    margin: 0px 0;
+}
+
+.order-button-container {
+    display: flex;
+    justify-content: center;
+    margin: 0px 0 40px 0;
+}
+
+d-article img {
+    width: 100%!important;
+}
+
+
+iframe, .js-plotly-plot {
+    width: 100%!important;
+    margin-bottom: 20px;
+}
+
+.modebar-container {
+    display: none;
+}
+
+#graph-container {
+    display: grid; grid-template-columns: 1fr 1fr; align-items: center;
+}
+
+@media (max-width: 768px) {
+    #graph-container {
+        grid-template-columns: 1fr;
+    }
+}
+
+@media (max-width: 1024px) {
+    #graph-container {
+        grid-template-columns: 1fr;
+    }
+    #graph-all {
+        margin-right: 0px;
+    }
+    #controls {
+        margin-left: 0px;
+    }
+}
+
+.main-plot-container svg {
+    background: transparent !important;
+}
+
+.large-image-background-transparent {
+    margin-left: 0px;
+    margin-right: 0px;
+}
+
+/* Import transformers-specific styles */
+@import url('./transformers-custom.css');