Added new Help section with common use-case examples

app.py

@@ -37,7 +37,7 @@ def human_readable_metadata(
     key: str,
     typ: int,
     val: Any,
-) -> tuple[str, int | str, Any]:
+) -> tuple[str, str, Any]:
     typ = GGUFValueType(typ).name
 
     if typ == 'ARRAY':
@@ -68,134 +68,472 @@ def human_readable_metadata(
 
 with gr.Blocks(
 ) as blocks:
-    with gr.Row():
-        hf_search = HuggingfaceHubSearch(
-            label = "Search Huggingface Hub",
-            placeholder = "Search for models on Huggingface",
-            search_type = "model",
-            sumbit_on_select = True,
-            scale = 2,
-        )
+    with gr.Tab("Editor"):
+        with gr.Row():
+            hf_search = HuggingfaceHubSearch(
+                label = "Search Huggingface Hub",
+                placeholder = "Search for models on Huggingface",
+                search_type = "model",
+                sumbit_on_select = True,
+                scale = 2,
+            )
 
-        hf_branch = gr.Dropdown(
-            None,
-            label = "Branch",
-            scale = 1,
-        )
+            hf_branch = gr.Dropdown(
+                None,
+                label = "Branch",
+                scale = 1,
+            )
+
+            gr.LoginButton(
+                "Sign in to access gated/private repos",
+                scale = 1,
+            )
 
-        gr.LoginButton(
-            "Sign in to access gated/private repos",
-            scale = 1,
+        hf_file = FileExplorer(
+            visible=False,
         )
 
-    hf_file = FileExplorer(
-        visible=False,
-    )
+        with gr.Row():
+            with gr.Column():
+                meta_keys = gr.Dropdown(
+                    None,
+                    label = "Modify Metadata",
+                    info = "Search by metadata key name",
+                    allow_custom_value = True,
+                    visible = False,
+                )
+
+            with gr.Column():
+                meta_types = gr.Dropdown(
+                    [e.name for e in GGUFValueType],
+                    label = "Metadata Type",
+                    info = "Select data type",
+                    type = "index",
+                    visible = False,
+                )
+
+            with gr.Column():
+                btn_delete = gr.Button(
+                    "Remove Key",
+                    variant = "stop",
+                    visible = False,
+                )
+
+        meta_boolean = gr.Checkbox(
+            label = "Boolean",
+            info = "Click to update value",
+            visible = False,
+        )
 
-    with gr.Row():
-        with gr.Column():
-            meta_keys = gr.Dropdown(
-                None,
-                label = "Modify Metadata",
-                info = "Search by metadata key name",
+        with gr.Row():
+            meta_token_select = gr.Dropdown(
+                label = "Select token",
+                info = "Search by token name",
+                type = "index",
                 allow_custom_value = True,
                 visible = False,
             )
 
-        with gr.Column():
-            meta_types = gr.Dropdown(
-                [e.name for e in GGUFValueType],
-                label = "Metadata Type",
-                info = "Select data type",
+            meta_token_type = gr.Dropdown(
+                [e.name for e in TokenType],
+                label = "Token type",
+                info = "Select token type",
                 type = "index",
                 visible = False,
             )
 
-        with gr.Column():
-            btn_delete = gr.Button(
-                "Remove Key",
-                variant = "stop",
+            meta_lookup = gr.Dropdown(
+                label = "Lookup token",
+                info = "Search by token name",
+                type = "index",
+                allow_custom_value = True,
                 visible = False,
             )
 
-    meta_boolean = gr.Checkbox(
-        label = "Boolean",
-        info = "Click to update value",
-        visible = False,
-    )
+            meta_number = gr.Number(
+                label = "Number",
+                info = "Enter to update value",
+                visible = False,
+            )
 
-    with gr.Row():
-        meta_token_select = gr.Dropdown(
-            label = "Select token",
-            info = "Search by token name",
-            type = "index",
-            allow_custom_value = True,
+        meta_string = gr.Textbox(
+            label = "String",
+            info = "Enter to update value (Shift+Enter for new line)",
             visible = False,
         )
 
-        meta_token_type = gr.Dropdown(
-            [e.name for e in TokenType],
-            label = "Token type",
-            info = "Select token type",
-            type = "index",
+        meta_array = gr.Matrix(
+            None,
+            label = "Unsupported",
+            row_count = (1, "fixed"),
+            height = "1rem",
+            interactive = False,
             visible = False,
         )
 
-        meta_lookup = gr.Dropdown(
-            label = "Lookup token",
-            info = "Search by token name",
-            type = "index",
-            allow_custom_value = True,
+        meta_changes = gr.HighlightedText(
+            None,
+            label = "Metadata Changes",
+            color_map = {"add": "green", "rem": "red"},
+            interactive = False,
             visible = False,
         )
 
-        meta_number = gr.Number(
-            info = "Enter to update value",
+        btn_download = gr.Button(
+            "Download GGUF",
+            variant = "primary",
             visible = False,
         )
 
-    meta_string = gr.Textbox(
-        info = "Enter to update value (Shift+Enter for new line)",
-        visible = False,
-    )
+        file_meta = gr.Matrix(
+            None,
+            col_count = (3, "fixed"),
+            headers = [
+                "Metadata Name",
+                "Type",
+                "Value",
+            ],
+            datatype = ["str", "str", "str"],
+            column_widths = ["35%", "15%", "50%"],
+            wrap = True,
+            interactive = False,
+            visible = False,
+        )
 
-    meta_array = gr.Matrix(
-        None,
-        label = "Unsupported",
-        row_count = (1, "fixed"),
-        height = "1rem",
-        interactive = False,
-        visible = False,
-    )
+    with gr.Tab("Help"):
+        gr.Markdown(
+            """# Huggingface GGUF Editor
 
-    meta_changes = gr.HighlightedText(
-        None,
-        label = "Metadata Changes",
-        color_map = {"add": "green", "rem": "red"},
-        interactive = False,
-        visible = False,
-    )
+An advanced GGUF editor, reading GGUF files directly from Huggingface repositories and applying changes to your own copies.
 
-    btn_download = gr.Button(
-        "Download GGUF",
-        variant = "primary",
-        visible = False,
-    )
+Below you will find a collection of example use-cases to show you how to perform a few common GGUF editing operations:
+            """,
+        )
 
-    file_meta = gr.Matrix(
-        None,
-        col_count = (3, "fixed"),
-        headers = [
-            "Metadata Name",
-            "Type",
-            "Value",
-        ],
-        datatype = ["str", "str", "str"],
-        column_widths = ["35%", "15%", "50%"],
-        wrap = True,
-        interactive = False,
-        visible = False,
-    )
+        with gr.Column(render = False) as example_group:
+            example_description = gr.Markdown(
+                visible = False,
+            )
+
+            with gr.Row():
+                with gr.Column():
+                    example_keys = gr.Dropdown(
+                        allow_custom_value = True,
+                        visible = False,
+                    )
+
+                with gr.Column():
+                    example_types = gr.Dropdown(
+                        allow_custom_value = True,
+                        visible = False,
+                    )
+
+                with gr.Column():
+                    example_delete = gr.Button(
+                        interactive = False,
+                        visible = False,
+                    )
+
+            example_boolean = gr.Checkbox(
+                visible = False,
+            )
+
+            with gr.Row():
+                example_token_select = gr.Dropdown(
+                    allow_custom_value = True,
+                    visible = False,
+                )
+
+                example_token_type = gr.Dropdown(
+                    allow_custom_value = True,
+                    visible = False,
+                )
+
+                example_number = gr.Number(
+                    visible = False,
+                )
+
+            example_string = gr.Textbox(
+                visible = False,
+            )
+
+        example_components = [
+            example_description,
+            example_keys,
+            example_types,
+            example_delete,
+            example_boolean,
+            example_token_select,
+            example_token_type,
+            example_number,
+            example_string,
+        ]
+        example_defaults = {
+            example_description: dict(
+                value = "",
+                visible = False,
+            ),
+            example_keys: dict(
+                value = "",
+                label = meta_keys.label,
+                info = "Select this metadata key",
+                visible = False,
+            ),
+            example_types: dict(
+                value = "",
+                label = meta_types.label,
+                info = "This will have the correct type set automatically",
+                visible = False,
+            ),
+            example_delete: dict(
+                value = btn_delete.value,
+                variant = btn_delete.variant,
+                visible = False,
+            ),
+            example_boolean: dict(
+                value = False,
+                label = meta_boolean.label,
+                info = "",
+                visible = False,
+            ),
+            example_token_select: dict(
+                value = "",
+                label = meta_token_select.label,
+                visible = False,
+            ),
+            example_token_type: dict(
+                value = "",
+                label = meta_token_type.label,
+                visible = False,
+            ),
+            example_number: dict(
+                value = 0,
+                precision = 0,
+                label = meta_number.label,
+                info = "",
+                visible = False,
+            ),
+            example_string: dict(
+                value = "",
+                label = meta_string.label,
+                info = "",
+                visible = False,
+            ),
+        }
+        example_properties = [
+            dict(
+                label = 'Fix "missing pre-tokenizer type" warning',
+                outputs = {
+                    example_description: dict(
+                        value = """## Fixing Pre-Tokenizer warning
+
+Custom Pre-Tokenization was added to `llama.cpp` April 29th 2024, and since then basically every model using BPE tokenization need support added to `llama.cpp` to work correctly.
+
+Models converted using the conversion script before the support for this specific model was added will either be missing the pre-tokenizer metadata or be set incorrectly to `default`.
+
+See the models list in [llama.cpp/convert_hf_to_gguf_update.py](https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf_update.py#L67) to find out which pre-tokenizer to choose.
+
+Setting the correct pre-tokenizer is often enough to fix the model's tokenizer, however if it has been quantized using an `imatrix` it should be re-quantized for best performance.
+
+Removing this metadata key from a model will cause `llama.cpp` to output a warning if BPE tokenization is used, it currently has no effect on any other tokenizers.
+                        """,
+                        visible = True,
+                    ),
+                    example_keys: dict(
+                        value = "tokenizer.ggml.pre",
+                        visible = True,
+                    ),
+                    example_types: dict(
+                        value = GGUFValueType.STRING.name,
+                        visible = True,
+                    ),
+                    example_delete: dict(
+                        visible = True,
+                    ),
+                    example_string: dict(
+                        info = "Fill in pre-tokenizer name, can be f.ex. deepseek-llm, command-r, tekken, etc. you will need to do some research to find the correct one",
+                        value = "llama-bpe",
+                        visible = True,
+                    ),
+                },
+            ),
+            dict(
+                label = "Add missing (Fill-in-Middle, EOT, etc) or change incorrect (BOS, EOS, etc) tokens",
+                outputs = {
+                    example_description: dict(
+                        value = """## Add missing/change incorrect tokens
+
+Sometimes converted models will be missing declarations of important tokens like EOT, Fill-in-Middle (prefix, suffix, middle) for various reasons.
+Other times they may have the incorrect tokens set as BOS, EOS, etc. Either way, missing or incorrectly declared tokens means inference will not work as expected.
+
+Token declaration is made with the metadata key(s) named "tokenizer.ggml.`token name`\_token\_id" which contains the ID (index number) of the token in the token list (`tokenizer.ggml.tokens`).
+
+A recurring issue is misconfigured EOS/EOT/EOM tokens, the need to set each of these and what they should be will vary between models, but the effect when these are incorrect is usually the same;
+infinte generation responses, ie. inference does not know when to stop. Typically this would be because f.ex. EOS has been set to <|endoftext|> instead of <|im\_end|> (again, model specific, just an example).
+
+Another issue, mainly for code models, is that Fill-in-Middle tokens have not been declared (note; not all models have or use such tokens), causing sub-par results for filling in blanks in code/text.
+There are 3 main metadata keys that need to be present for this; tokenizer.ggml.`prefix`\_token\_id, `suffix` and `middle`, sometimes also EOT/EOM if it differs from EOS in this mode.
+They are usually named fim\_`something` or just `PRE`, `SUF` and `MID`, take extra care with DeepSeek-based models where prefix is (...fim...)`begin`, suffix is `hole` and middle is `end`.
+                        """,
+                        visible = True,
+                    ),
+                    example_keys: dict(
+                        value = "tokenizer.ggml.prefix_token_id",
+                        info = "Select or enter any metadata key ending with _token_id",
+                        visible = True,
+                    ),
+                    example_types: dict(
+                        value = GGUFValueType.UINT32.name,
+                        visible = True,
+                    ),
+                    example_token_select: dict(
+                        value = "<fim_prefix>",
+                        label = meta_lookup.label,
+                        info = "You can search for the correct token by parts of its name here, then select the correct one from the list of options",
+                        visible = True,
+                    ),
+                    example_number: dict(
+                        value = 92295,
+                        info = "The token ID will be automatically filled in when you select the token, but you can also fill in the ID directly",
+                        visible = True,
+                    ),
+                },
+            ),
+            dict(
+                label = "Setting the correct token type for a token",
+                outputs = {
+                    example_description: dict(
+                        value = """## Changing a token's type
+
+A common issue is not declaring special control tokens as such, leading to bad tokenization of them when used (usually in the chat template), causing poor responses from the model.
+
+Take f.ex. a model with an incorrectly configured <|im\_start|> token as a normal token instead of a special control token, given the following prompt:
+```
+<|im_start|>Hello World<|im_end|>
+```
+
+This prompt would then be incorrectly tokenized as follows:
+```
+	    27 ('<')
+	    91 ('|')
+	   318 ('im')
+	  4906 ('_start')
+	    91 ('|')
+	    29 ('>')
+	  9707 ('Hello')
+	  4337 (' World')
+	151645 ('<|im_end|>')
+```
+
+instead of:
+```
+	151644 ('<|im_start|>')
+	  9707 ('Hello')
+	  4337 (' World')
+	151645 ('<|im_end|>')
+```
+
+Take care to also adjust the value for this token in `tokenizer.ggml.scores` (if it exists) similarly to other special control tokens.
+
+**WARNING**: Even though you have the option to, you should never remove the `tokenizer.ggml.token_type` key!
+                        """,
+                        visible = True,
+                    ),
+                    example_keys: dict(
+                        value = "tokenizer.ggml.token_type",
+                        visible = True,
+                    ),
+                    example_types: dict(
+                        value = GGUFValueType.INT32.name,
+                        visible = True,
+                    ),
+                    example_delete: dict(
+                        visible = True,
+                    ),
+                    example_token_select: dict(
+                        value = "<|im_start|>",
+                        info = "You can search for the token by parts of its name here, then select it from the list of options",
+                        visible = True,
+                    ),
+                    example_token_type: dict(
+                        value = TokenType.CONTROL.name,
+                        info = "Select the appropriate token type, in this case we set it as a special control token",
+                        visible = True,
+                    ),
+                },
+            ),
+            dict(
+                label = "Updating or adding a chat template",
+                outputs = {
+                    example_description: dict(
+                        value = """## Modifying the Chat Template
+
+The chat template is a very important part of the model metadata as this provides a template for how to format the conversation prompt to the model.
+It's not uncommon for these to have bugs (or sometimes just be plain wrong), requiring you to update them to be able to prompt the model correctly.
+
+It's also possible to have multiple chat templates for different purposes, the main ones being RAG and Tools, but you can create any additional template you want.
+The standard metadata key for RAG is `tokenizer.chat_template.rag` and Tools is `tokenizer.chat_template.tool_use`, any metadata key added starting with `tokenizer.chat_template.` will be added as a custom chat template.
+
+Any framework based on `llama-cpp-python` will let you select which chat template to use with the `chat_format` option, available as `chat_template.default`, `chat_template.rag`, `chat_template.tool_use`, etc...
+                        """,
+                        visible = True,
+                    ),
+                    example_keys: dict(
+                        value = "tokenizer.chat_template",
+                        info = 'Select this or enter any key starting with "tokenizer.chat_template."',
+                        visible = True,
+                    ),
+                    example_types: dict(
+                        value = GGUFValueType.STRING.name,
+                        visible = True,
+                    ),
+                    example_delete: dict(
+                        visible = True,
+                    ),
+                    example_string: dict(
+                        info = "Paste in the updated chat template or make changes here. Using an external Jinja2 editor is recommended",
+                        value = "{%- for message in messages %}\n    {{- '<|' + message['role'] + '|>\\n' }}\n    {{- message['content'] + eos_token }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|assistant|>\\n' }}\n{%- endif %}",
+                        visible = True,
+                    ),
+                },
+            ),
+        ]
+
+        examples = gr.Dataset(
+            label = "Choose an example",
+            type = "index",
+            samples = [[]] * len(example_properties),
+            sample_labels = [x["label"] for x in example_properties],
+        )
+
+        @gr.on(
+            triggers = [
+                examples.click,
+            ],
+            inputs = [
+                examples,
+            ],
+            outputs = [
+            ] + example_components,
+            show_progress = "hidden",
+        )
+        def show_example(
+            value: int,
+        ):
+            outputs = example_properties[value]["outputs"]
+            non_outputs = example_components - outputs.keys()
+            all_outputs = dict(((k, type(k)(**(example_defaults[k] | v))) for k, v in outputs.items()))
+
+            for output in non_outputs:
+                all_outputs[output] = type(output)(**example_defaults[output])
+
+            return all_outputs
+
+        for k, v in example_defaults.items():
+            for prop, val in v.items():
+                setattr(k, prop, val)
+
+        example_group.render()
 
     meta_state = gr.State() # init_state
     # BUG: For some reason using gr.State initial value turns tuple to list?