Upload sd_token_similarity_calculator.ipynb

sd_token_similarity_calculator.ipynb

@@ -116,28 +116,10 @@
       "metadata": {
         "id": "Ch9puvwKH1s3",
         "collapsed": true,
-        "cellView": "form",
-        "outputId": "8101e515-49f2-41d4-b03b-4195d56f50de",
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        }
+        "cellView": "form"
       },
-      "execution_count": 1,
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Cloning into 'sd_tokens'...\n",
-            "remote: Enumerating objects: 10, done.\u001b[K\n",
-            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
-            "remote: Compressing objects: 100% (7/7), done.\u001b[K\n",
-            "remote: Total 10 (delta 1), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
-            "Unpacking objects: 100% (10/10), 306.93 KiB | 1.19 MiB/s, done.\n",
-            "/content/sd_tokens\n"
-          ]
-        }
-      ]
+      "execution_count": null,
+      "outputs": []
     },
     {
       "cell_type": "code",
@@ -306,7 +288,16 @@
     {
       "cell_type": "code",
       "source": [
-        "# @title 🪐🖼️ -> 📝 Image to prompt : Create suggestions of things to add to prompt to match image\n",
+        "# @title 🪐🖼️ -> 📝 Slow Recursive Token Image interrogator\n",
+        "\n",
+        "# @markdown # What do you want to to mimic?\n",
+        "use = '🖼️image_encoding from image' # @param ['📝text_encoding from prompt', '🖼️image_encoding from image']\n",
+        "# @markdown --------------------------\n",
+        "use_token_padding = True # param {type:\"boolean\"} <---- Enabled by default\n",
+        "prompt = \"photo of a banana\" # @param {\"type\":\"string\",\"placeholder\":\"Write a prompt\"}\n",
+        "\n",
+        "prompt_A = prompt\n",
+        "\n",
         "from google.colab import files\n",
         "def upload_files():\n",
         "  from google.colab import files\n",
@@ -316,61 +307,80 @@
         "  return list(uploaded.keys())\n",
         "#Get image\n",
         "# You can use \"http://images.cocodataset.org/val2017/000000039769.jpg\" for testing\n",
-        "url = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {\"type\":\"string\",\"placeholder\":\"leave empty for local upload (scroll down to see it)\"}\n",
+        "image_url = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {\"type\":\"string\",\"placeholder\":\"leave empty for local upload (scroll down to see it)\"}\n",
+        "\n",
+        "\n",
+        "colab_image_path = \"\" # @param {\"type\":\"string\",\"placeholder\": \"eval. as '/content/sd_tokens/' + **your input**\"}\n",
         "\n",
-        "colab_image_path = \"\" # @param {\"type\":\"string\",\"placeholder\":\"(optional) Write colab image path to load from\"}\n",
+        "# @markdown --------------------------\n",
         "from PIL import Image\n",
         "import requests\n",
         "image_A = \"\"\n",
         "\n",
         "#----#\n",
-        "if url == \"\":\n",
-        "  import cv2\n",
-        "  from google.colab.patches import cv2_imshow\n",
-        "  # Open the image.\n",
-        "  if colab_image_path == \"\":\n",
-        "    keys =  upload_files()\n",
-        "    for key in keys:\n",
-        "      image_A = cv2.imread(\"/content/sd_tokens/\" + key)\n",
-        "      colab_image_path = \"/content/sd_tokens/\" + key\n",
-        "  else:\n",
-        "     image_A = cv2.imread(colab_image_path)\n",
-        "else:\n",
-        "  image_A = Image.open(requests.get(url, stream=True).raw)\n",
         "\n",
+        "if(use == '🖼️image_encoding from image'):\n",
+        "  if image_url == \"\":\n",
+        "    import cv2\n",
+        "    from google.colab.patches import cv2_imshow\n",
+        "    # Open the image.\n",
+        "    if colab_image_path == \"\":\n",
+        "      keys =  upload_files()\n",
+        "      for key in keys:\n",
+        "        image_A = cv2.imread(\"/content/sd_tokens/\" + key)\n",
+        "        colab_image_path = \"/content/sd_tokens/\" + key\n",
+        "    else:\n",
+        "      image_A = cv2.imread(\"/content/sd_tokens/\" + colab_image_path)\n",
+        "  else:\n",
+        "    image_A = Image.open(requests.get(image_url, stream=True).raw)\n",
+        "#------#\n",
         "\n",
-        "# Get image features\n",
+        "from transformers import AutoTokenizer\n",
+        "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
         "from transformers import  CLIPProcessor, CLIPModel\n",
         "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\" , clean_up_tokenization_spaces = True)\n",
         "model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
-        "inputs = processor(images=image_A, return_tensors=\"pt\")\n",
-        "image_features = model.get_image_features(**inputs)\n",
-        "image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)\n",
-        "prompt_A = \"the image\"\n",
-        "name_A = prompt_A\n",
+        "\n",
+        "\n",
+        "if(use == '🖼️image_encoding from image'):\n",
+        "  # Get image features\n",
+        "  inputs = processor(images=image_A, return_tensors=\"pt\")\n",
+        "  image_features = model.get_image_features(**inputs)\n",
+        "  image_features = image_features / image_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "  name_A = \"the image\"\n",
         "#-----#\n",
         "\n",
-        "# @markdown Set conditions for the output\n",
+        "\n",
+        "if(use == '📝text_encoding from prompt'):\n",
+        "  # Get text features\n",
+        "  inputs = tokenizer(text = prompt, padding=True, return_tensors=\"pt\")\n",
+        "  text_features_A = model.get_text_features(**inputs)\n",
+        "  name_A = prompt\n",
+        "#-----#\n",
+        "\n",
+        "\n",
+        "# @markdown # The output...\n",
         "must_start_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
         "must_contain = \"banana \" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
         "must_end_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
         "token_B = must_contain\n",
         "\n",
-        "# @markdown Limit the search\n",
-        "use_token_padding = True # @param {type:\"boolean\"}\n",
+        "# @markdown -----\n",
+        "\n",
+        "# @markdown # Use a range of tokens from the vocab.json (slow method)\n",
         "start_search_at_ID = 27700 # @param {type:\"slider\", min:0, max: 49407, step:100}\n",
-        "search_range = 288 # @param {type:\"slider\", min:100, max: 2000, step:0}\n",
+        "search_range = 100 # @param {type:\"slider\", min:100, max: 2000, step:0}\n",
         "restrictions = 'None' # @param [\"None\", \"Suffix only\", \"Prefix only\"]\n",
         "\n",
-        "# @markdown Limit char size of included token\n",
-        "min_char_size = 3 # @param {type:\"slider\", min:0, max: 20, step:1}\n",
-        "char_range = 14 # @param {type:\"slider\", min:0, max: 20, step:1}\n",
+        "#markdown Limit char size of included token <----- Disabled\n",
+        "min_char_size = 0 #param {type:\"slider\", min:0, max: 20, step:1}\n",
+        "char_range = 50 #param {type:\"slider\", min:0, max: 20, step:1}\n",
         "\n",
-        "#Tokenize input B\n",
-        "from transformers import AutoTokenizer\n",
-        "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
-        "tokenizer_output = tokenizer(text = token_B)\n",
-        "input_ids = tokenizer_output['input_ids']\n",
+        "\n",
+        "# markdown # ...or paste prompt items\n",
+        "# markdown Format must be {item1|item2|...}. You can aquire prompt items using the Randomizer in the fusion gen: https://perchance.org/fusion-ai-image-generator\n",
+        "_enable = False # param {\"type\":\"boolean\"}\n",
+        "prompt_items = \"\" # param {\"type\":\"string\",\"placeholder\":\"{item1|item2|...}\"}\n",
         "#-----#\n",
         "name_B = must_contain\n",
         "#-----#\n",
@@ -412,17 +422,29 @@
         "  if len(name_C) > min_char_size + char_range:\n",
         "    continue\n",
         "  #-----#\n",
-        "\n",
         "  name_CB = must_start_with + name_C + name_B + must_end_with\n",
         "  if is_Prefix>0:\n",
         "    name_CB = must_start_with + ' ' + name_C.strip() + '-' + name_B.strip() + ' ' + must_end_with\n",
         "  #-----#\n",
-        "  ids_CB = processor.tokenizer(text=name_CB, padding=use_token_padding, return_tensors=\"pt\")\n",
-        "  text_features = model.get_text_features(**ids_CB)\n",
-        "  text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
-        "  logit_scale = model.logit_scale.exp()\n",
-        "  torch.matmul(text_features, image_features.t()) * logit_scale\n",
-        "  sim_CB = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "\n",
+        "  if(use == '🖼️image_encoding from image'):\n",
+        "    ids_CB = processor.tokenizer(text=name_CB, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "    text_features = model.get_text_features(**ids_CB)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    logit_scale = model.logit_scale.exp()\n",
+        "    torch.matmul(text_features, image_features.t()) * logit_scale\n",
+        "    sim_CB = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "  #-----#\n",
+        "\n",
+        "  if(use == '📝text_encoding from prompt'):\n",
+        "    ids_CB = processor.tokenizer(text=name_CB, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "    text_features = model.get_text_features(**ids_CB)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    sim_CB = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
+        "  #-----#\n",
+        "\n",
+        "\n",
+        "\n",
         "  #-----#\n",
         "  if restrictions == \"Prefix only\":\n",
         "    result = sim_CB\n",
@@ -430,13 +452,23 @@
         "    dots[index] = result\n",
         "    continue\n",
         "  #-----#\n",
-        "  name_BC = must_start_with + name_B + name_C + must_end_with\n",
-        "  ids_BC = processor.tokenizer(text=name_BC, padding=use_token_padding, return_tensors=\"pt\")\n",
-        "  text_features = model.get_text_features(**ids_BC)\n",
-        "  text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
-        "  logit_scale = model.logit_scale.exp()\n",
-        "  torch.matmul(text_features, image_features.t()) * logit_scale\n",
-        "  sim_BC = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "\n",
+        "  if(use == '🖼️image_encoding from image'):\n",
+        "    name_BC = must_start_with + name_B + name_C + must_end_with\n",
+        "    ids_BC = processor.tokenizer(text=name_BC, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "    text_features = model.get_text_features(**ids_BC)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    logit_scale = model.logit_scale.exp()\n",
+        "    torch.matmul(text_features, image_features.t()) * logit_scale\n",
+        "    sim_BC = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "  #-----#\n",
+        "\n",
+        "  if(use == '📝text_encoding from prompt'):\n",
+        "    name_BC = must_start_with + name_B + name_C + must_end_with\n",
+        "    ids_BC = processor.tokenizer(text=name_BC, padding=use_token_padding, return_tensors=\"pt\")\n",
+        "    text_features = model.get_text_features(**ids_BC)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    sim_BC = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
         "  #-----#\n",
         "\n",
         "  result = sim_CB\n",
@@ -451,7 +483,9 @@
         "\n",
         "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
         "\n",
-        "# @markdown Print options\n",
+        "\n",
+        "# @markdown ----------\n",
+        "# @markdown # Print options\n",
         "list_size = 100 # @param {type:'number'}\n",
         "print_ID = False # @param {type:\"boolean\"}\n",
         "print_Similarity = True # @param {type:\"boolean\"}\n",
@@ -531,13 +565,25 @@
         "  #----#\n",
         "  ids = processor.tokenizer(text=name, padding=use_token_padding, return_tensors=\"pt\")\n",
         "\n",
-        "  text_features = model.get_text_features(**ids)\n",
-        "  text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
-        "  logit_scale = model.logit_scale.exp()\n",
-        "  torch.matmul(text_features, image_features.t()) * logit_scale\n",
-        "  sim = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "  if(use == '🖼️image_encoding from image'):\n",
+        "    text_features = model.get_text_features(**ids)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    logit_scale = model.logit_scale.exp()\n",
+        "    torch.matmul(text_features, image_features.t()) * logit_scale\n",
+        "    sim = torch.nn.functional.cosine_similarity(text_features, image_features) * logit_scale\n",
+        "  #-----#\n",
+        "\n",
+        "  if(use == '📝text_encoding from prompt'):\n",
+        "    text_features = model.get_text_features(**ids)\n",
+        "    text_features = text_features / text_features.norm(p=2, dim=-1, keepdim=True)\n",
+        "    sim = torch.nn.functional.cosine_similarity(text_features, text_features_A)\n",
+        "  #-----#\n",
+        "\n",
+        "\n",
         "  dots[index] = sim\n",
         "  names[index] = name\n",
+        "\n",
+        "\n",
         "#------#\n",
         "\n",
         "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
@@ -604,153 +650,6 @@
         "id": "hyK423TQCRup"
       }
     },
-    {
-      "cell_type": "code",
-      "source": [
-        "# @title 🪐📝 Prompt to prompt : Add single token to existing prompt to match another prompt\n",
-        "# @markdown Write a text to match against...\n",
-        "prompt_A = \"photo of a banana\" # @param {\"type\":\"string\",\"placeholder\":\"Write a prompt\"}\n",
-        "\n",
-        "# @markdown Set conditions for the output\n",
-        "must_start_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
-        "must_contain = \"yellow\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
-        "must_end_with = \"\" # @param {\"type\":\"string\",\"placeholder\":\"write a text\"}\n",
-        "token_B = must_contain\n",
-        "\n",
-        "# @markdown Limit the search\n",
-        "use_token_padding = True # @param {type:\"boolean\"}\n",
-        "start_search_at_ID = 12500 # @param {type:\"slider\", min:0, max: 49407, step:100}\n",
-        "search_range = 500 # @param {type:\"slider\", min:0, max: 2000, step:100}\n",
-        "restrictions = 'Suffix only' # @param [\"None\", \"Suffix only\", \"Prefix only\"]\n",
-        "\n",
-        "# @markdown Limit char size of included token\n",
-        "min_char_size = 3 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
-        "char_range = 5 # @param {type:\"slider\", min:0, max: 50, step:1}\n",
-        "\n",
-        "#Tokenize input B\n",
-        "from transformers import AutoTokenizer\n",
-        "tokenizer = AutoTokenizer.from_pretrained(\"openai/clip-vit-large-patch14\", clean_up_tokenization_spaces = False)\n",
-        "tokenizer_output = tokenizer(text = token_B)\n",
-        "input_ids = tokenizer_output['input_ids']\n",
-        "#-----#\n",
-        "name_B = must_contain\n",
-        "#-----#\n",
-        "\n",
-        "from transformers import  CLIPProcessor, CLIPModel\n",
-        "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-large-patch14\" , clean_up_tokenization_spaces = True)\n",
-        "model = CLIPModel.from_pretrained(\"openai/clip-vit-large-patch14\")\n",
-        "#-------#\n",
-        "ids_A = processor.tokenizer(text=prompt_A, padding=use_token_padding, return_tensors=\"pt\")\n",
-        "text_encoding_A = model.get_text_features(**ids_A)\n",
-        "A =  text_encoding_A[0]\n",
-        "_A = LA.vector_norm(A, ord=2)\n",
-        "name_A = prompt_A\n",
-        "print(f'a text_encoding was created for the prompt \"{prompt_A}\" ')\n",
-        "print('')\n",
-        "#----#\n",
-        "\n",
-        "START = start_search_at_ID\n",
-        "RANGE =  min(search_range , 49407 - start_search_at_ID)\n",
-        "\n",
-        "dots = torch.zeros(RANGE)\n",
-        "is_BC = torch.zeros(RANGE)\n",
-        "for index in range(RANGE):\n",
-        "  id_C = START + index\n",
-        "  C = token[id_C]\n",
-        "  _C = LA.vector_norm(C, ord=2)\n",
-        "  name_C = vocab[id_C]\n",
-        "\n",
-        "  # Decide if we should process prefix/suffix tokens\n",
-        "  if name_C.find('</w>')<=-1:\n",
-        "    if restrictions != \"Prefix only\":\n",
-        "      continue\n",
-        "  else:\n",
-        "    if restrictions == \"Prefix only\":\n",
-        "      continue\n",
-        "  #-----#\n",
-        "\n",
-        "  # Decide if char-size is within range\n",
-        "  if len(name_C) < min_char_size:\n",
-        "    continue\n",
-        "  if len(name_C) > min_char_size + char_range:\n",
-        "    continue\n",
-        "  #-----#\n",
-        "\n",
-        "  name_CB = must_start_with + name_C + name_B + must_end_with\n",
-        "  if restrictions == \"Prefix only\":\n",
-        "    name_CB = must_start_with +  name_C + '-' + name_B + must_end_with\n",
-        "  #-----#\n",
-        "  ids_CB = processor.tokenizer(text=name_CB, padding=use_token_padding, return_tensors=\"pt\")\n",
-        "  text_encoding_CB = model.get_text_features(**ids_CB)\n",
-        "  CB = text_encoding_CB[0]\n",
-        "  _CB =  LA.vector_norm(CB, ord=2)\n",
-        "  sim_CB = torch.dot(A,CB)/(_A*_CB)\n",
-        "  #-----#\n",
-        "  if restrictions == \"Prefix only\":\n",
-        "    result = sim_CB\n",
-        "    result = result.item()\n",
-        "    dots[index] = result\n",
-        "    continue\n",
-        "  #-----#\n",
-        "  name_BC = must_start_with + name_B + name_C + must_end_with\n",
-        "  ids_BC = processor.tokenizer(text=name_BC, padding=use_token_padding, return_tensors=\"pt\")\n",
-        "  text_encoding_BC = model.get_text_features(**ids_BC)\n",
-        "  BC = text_encoding_BC[0]\n",
-        "  _BC =  LA.vector_norm(BC, ord=2)\n",
-        "  sim_BC = torch.dot(A,BC)/(_A*_BC)\n",
-        "  #-----#\n",
-        "\n",
-        "  result = sim_CB\n",
-        "  if(sim_BC > sim_CB):\n",
-        "    is_BC[index] = 1\n",
-        "    result = sim_BC\n",
-        "\n",
-        "  #result = absolute_value(result.item())\n",
-        "  result = result.item()\n",
-        "  dots[index] = result\n",
-        "#----#\n",
-        "\n",
-        "sorted, indices = torch.sort(dots,dim=0 , descending=True)\n",
-        "\n",
-        "# @markdown Print options\n",
-        "list_size = 100 # @param {type:'number'}\n",
-        "print_ID = False # @param {type:\"boolean\"}\n",
-        "print_Similarity = True # @param {type:\"boolean\"}\n",
-        "print_Name = True # @param {type:\"boolean\"}\n",
-        "print_Divider = True # @param {type:\"boolean\"}\n",
-        "\n",
-        "\n",
-        "if (print_Divider):\n",
-        "  print('//---//')\n",
-        "\n",
-        "print('')\n",
-        "print(f'These token pairings within the range ID = {START} to ID = {START + RANGE} most closely match the text_encoding for the prompt \"{prompt_A}\" : ')\n",
-        "print('')\n",
-        "\n",
-        "for index in range(min(list_size,RANGE)):\n",
-        "  id = START + indices[index].item()\n",
-        "  if (print_Name):\n",
-        "    if(is_BC[index]>0):\n",
-        "      print(must_start_with +  name_B + vocab[id] + must_end_with)\n",
-        "    else:\n",
-        "      if restrictions == \"Prefix only\":\n",
-        "        print(must_start_with +   vocab[id] + '-'  + name_B + must_end_with)\n",
-        "      else:\n",
-        "        print(must_start_with +   vocab[id] + name_B + must_end_with)\n",
-        "  if (print_ID):\n",
-        "    print(f'ID = {id}') # IDs\n",
-        "  if (print_Similarity):\n",
-        "    print(f'similiarity = {round(sorted[index].item()*100,2)} %')\n",
-        "  if (print_Divider):\n",
-        "    print('--------')"
-      ],
-      "metadata": {
-        "cellView": "form",
-        "id": "uDtcm-l8UCJk"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
     {
       "cell_type": "markdown",
       "source": [