some more craziness

package-lock.json

@@ -11,7 +11,7 @@
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
         "@jcoreio/async-throttle": "^1.6.0",
-        "@mediapipe/tasks-vision": "^0.10.12",
+        "@mediapipe/tasks-vision": "^0.10.13-rc.20240419",
         "@photo-sphere-viewer/core": "^5.7.2",
         "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2",
         "@photo-sphere-viewer/gyroscope-plugin": "^5.7.2",
@@ -1492,14 +1492,14 @@
       }
     },
     "node_modules/@mediapipe/tasks-vision": {
-      "version": "0.10.12",
-      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.12.tgz",
-      "integrity": "sha512-688Vukid7hvGmx+7hzS/EQ3Q4diz4eeX4/FYDw8f/t56UjFueD8LTvA2rX5BCIwvT0oy8QHKh5uKIyct1AOFtQ=="
+      "version": "0.10.13-rc.20240419",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240419.tgz",
+      "integrity": "sha512-IfIFVSkektnlo9FjqhDYe5LzbOJTdx9kjs65lXmerqPQSOA3akYwp5zt2Rtm1G4JrG1Isv1A15yfiSlqm5pGdQ=="
     },
     "node_modules/@next/env": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.4.tgz",
-      "integrity": "sha512-e7X7bbn3Z6DWnDi75UWn+REgAbLEqxI8Tq2pkFOFAMpWAWApz/YCUhtWMWn410h8Q2fYiYL7Yg5OlxMOCfFjJQ=="
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.2.tgz",
+      "integrity": "sha512-sk72qRfM1Q90XZWYRoJKu/UWlTgihrASiYw/scb15u+tyzcze3bOuJ/UV6TBOQEeUaxOkRqGeuGUdiiuxc5oqw=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "13.4.10",
@@ -1510,9 +1510,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.4.tgz",
-      "integrity": "sha512-ubmUkbmW65nIAOmoxT1IROZdmmJMmdYvXIe8211send9ZYJu+SqxSnJM4TrPj9wmL6g9Atvj0S/2cFmMSS99jg==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.2.tgz",
+      "integrity": "sha512-3iPgMhzbalizGwHNFUcGnDhFPSgVBHQ8aqSTAMxB5BvJG0oYrDf1WOJZlbXBgunOEj/8KMVbejEur/FpvFsgFQ==",
       "cpu": [
         "arm64"
       ],
@@ -1525,9 +1525,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.4.tgz",
-      "integrity": "sha512-b0Xo1ELj3u7IkZWAKcJPJEhBop117U78l70nfoQGo4xUSvv0PJSTaV4U9xQBLvZlnjsYkc8RwQN1HoH/oQmLlQ==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.2.tgz",
+      "integrity": "sha512-x7Afi/jt0ZBRUZHTi49yyej4o8znfIMHO4RvThuoc0P+uli8Jd99y5GKjxoYunPKsXL09xBXEM1+OQy2xEL0Ag==",
       "cpu": [
         "x64"
       ],
@@ -1540,9 +1540,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.4.tgz",
-      "integrity": "sha512-457G0hcLrdYA/u1O2XkRMsDKId5VKe3uKPvrKVOyuARa6nXrdhJOOYU9hkKKyQTMru1B8qEP78IAhf/1XnVqKA==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.2.tgz",
+      "integrity": "sha512-zbfPtkk7L41ODMJwSp5VbmPozPmMMQrzAc0HAUomVeVIIwlDGs/UCqLJvLNDt4jpWgc21SjjyIn762lNGrMaUA==",
       "cpu": [
         "arm64"
       ],
@@ -1555,9 +1555,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.4.tgz",
-      "integrity": "sha512-l/kMG+z6MB+fKA9KdtyprkTQ1ihlJcBh66cf0HvqGP+rXBbOXX0dpJatjZbHeunvEHoBBS69GYQG5ry78JMy3g==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.2.tgz",
+      "integrity": "sha512-wPbS3pI/JU16rm3XdLvvTmlsmm1nd+sBa2ohXgBZcShX4TgOjD4R+RqHKlI1cjo/jDZKXt6OxmcU0Iys0OC/yg==",
       "cpu": [
         "arm64"
       ],
@@ -1570,9 +1570,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.4.tgz",
-      "integrity": "sha512-BapIFZ3ZRnvQ1uWbmqEGJuPT9cgLwvKtxhK/L2t4QYO7l+/DxXuIGjvp1x8rvfa/x1FFSsipERZK70pewbtJtw==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.2.tgz",
+      "integrity": "sha512-NqWOHqqq8iC9tuHvZxjQ2tX+jWy2X9y8NX2mcB4sj2bIccuCxbIZrU/ThFPZZPauygajZuVQ6zediejQHwZHwQ==",
       "cpu": [
         "x64"
       ],
@@ -1585,9 +1585,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.4.tgz",
-      "integrity": "sha512-mqVxTwk4XuBl49qn2A5UmzFImoL1iLm0KQQwtdRJRKl21ylQwwGCxJtIYo2rbfkZHoSKlh/YgztY0qH3wG1xIg==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.2.tgz",
+      "integrity": "sha512-lGepHhwb9sGhCcU7999+iK1ZZT+6rrIoVg40MP7DZski9GIZP80wORSbt5kJzh9v2x2ev2lxC6VgwMQT0PcgTA==",
       "cpu": [
         "x64"
       ],
@@ -1600,9 +1600,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.4.tgz",
-      "integrity": "sha512-xzxF4ErcumXjO2Pvg/wVGrtr9QQJLk3IyQX1ddAC/fi6/5jZCZ9xpuL9Tzc4KPWMFq8GGWFVDMshZOdHGdkvag==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.2.tgz",
+      "integrity": "sha512-TZSh/48SfcLEQ4rD25VVn2kdIgUWmMflRX3OiyPwGNXn3NiyPqhqei/BaqCYXViIQ+6QsG9R0C8LftMqy8JPMA==",
       "cpu": [
         "arm64"
       ],
@@ -1615,9 +1615,9 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.4.tgz",
-      "integrity": "sha512-WZiz8OdbkpRw6/IU/lredZWKKZopUMhcI2F+XiMAcPja0uZYdMTZQRoQ0WZcvinn9xZAidimE7tN9W5v9Yyfyw==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.2.tgz",
+      "integrity": "sha512-M0tBVNMEBJN2ZNQWlcekMn6pvLria7Sa2Fai5znm7CCJz4pP3lrvlSxhKdkCerk0D9E0bqx5yAo3o2Q7RrD4gA==",
       "cpu": [
         "ia32"
       ],
@@ -1630,9 +1630,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.4.tgz",
-      "integrity": "sha512-4Rto21sPfw555sZ/XNLqfxDUNeLhNYGO2dlPqsnuCg8N8a2a9u1ltqBOPQ4vj1Gf7eJC0W2hHG2eYUHuiXgY2w==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.2.tgz",
+      "integrity": "sha512-a/20E/wtTJZ3Ykv3f/8F0l7TtgQa2LWHU2oNB9bsu0VjqGuGGHmm/q6waoUNQYTVPYrrlxxaHjJcDV6aiSTt/w==",
       "cpu": [
         "x64"
       ],
@@ -2859,15 +2859,21 @@
       }
     },
     "node_modules/@rushstack/eslint-patch": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.1.tgz",
-      "integrity": "sha512-S3Kq8e7LqxkA9s7HKLqXGTGck1uwis5vAXan3FnU5yw1Ec5hsSGnq4s/UCaSqABPOnOTg7zASLyst7+ohgWexg=="
+      "version": "1.10.2",
+      "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.2.tgz",
+      "integrity": "sha512-hw437iINopmQuxWPSUEvqE56NCPsiU8N4AYtfHmJFckclktzK9YQJieD3XkDCDH4OjL+C7zgPUh73R/nrcHrqw=="
+    },
+    "node_modules/@swc/counter": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz",
+      "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="
     },
     "node_modules/@swc/helpers": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz",
-      "integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==",
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz",
+      "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==",
       "dependencies": {
+        "@swc/counter": "^0.1.3",
         "tslib": "^2.4.0"
       }
     },
@@ -2956,9 +2962,9 @@
       }
     },
     "node_modules/@types/qs": {
-      "version": "6.9.14",
-      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.14.tgz",
-      "integrity": "sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==",
+      "version": "6.9.15",
+      "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz",
+      "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==",
       "dev": true
     },
     "node_modules/@types/react": {
@@ -2989,9 +2995,9 @@
       }
     },
     "node_modules/@types/react-virtualized": {
-      "version": "9.21.29",
-      "resolved": "https://registry.npmjs.org/@types/react-virtualized/-/react-virtualized-9.21.29.tgz",
-      "integrity": "sha512-+ODVQ+AyKngenj4OPpg43Hz4B9Rdjuz1Naxu9ypNc3Cjo0WVZTYhqXfF/Nm38i8PV/YXECRIl4mTAZK5hq2B+g==",
+      "version": "9.21.30",
+      "resolved": "https://registry.npmjs.org/@types/react-virtualized/-/react-virtualized-9.21.30.tgz",
+      "integrity": "sha512-4l2TFLQ8BCjNDQlvH85tU6gctuZoEdgYzENQyZHpgTHU7hoLzYgPSOALMAeA58LOWua8AzC6wBivPj1lfl6JgQ==",
       "dev": true,
       "dependencies": {
         "@types/prop-types": "*",
@@ -3130,9 +3136,9 @@
       }
     },
     "node_modules/@upstash/redis": {
-      "version": "1.29.0",
-      "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.29.0.tgz",
-      "integrity": "sha512-kbO5fgMAeUzErnA/SOtaSbAa0dguYhhBT4MZHJ1O8gVl4iK754aC9+rIYY5hsp4nlxeCGfnIDkWpof991c9jjA==",
+      "version": "1.30.0",
+      "resolved": "https://registry.npmjs.org/@upstash/redis/-/redis-1.30.0.tgz",
+      "integrity": "sha512-bkxl2n7qls27hONXNyItM6ccRT0NBJ8c6zl83wx+TlODpYXehgeiUJTkcTaXLp7hk+zy6Mqpzpeo3GX+wZSvBw==",
       "dependencies": {
         "crypto-js": "^4.2.0"
       }
@@ -3692,9 +3698,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001606",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001606.tgz",
-      "integrity": "sha512-LPbwnW4vfpJId225pwjZJOgX1m9sGfbw/RKJvw/t0QhYOOaTXHvkjVGFGPpvwEzufrjvTlsULnVTxdy4/6cqkg==",
+      "version": "1.0.30001611",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001611.tgz",
+      "integrity": "sha512-19NuN1/3PjA3QI8Eki55N8my4LzfkMCRLgCVfrl/slbSAchQfV0+GwjPrK3rq37As4UCLlM/DHajbKkAqbv92Q==",
       "funding": [
         {
           "type": "opencollective",
@@ -3890,9 +3896,9 @@
       }
     },
     "node_modules/cookies-next/node_modules/@types/node": {
-      "version": "16.18.94",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.94.tgz",
-      "integrity": "sha512-X8q3DoKq8t/QhA0Rk/9wJUajxtXRDiCK+cVaONKLxpsjPhu+xX6uZuEj4UKGLQ4p0obTdFxa0cP/BMvf9mOYZA=="
+      "version": "16.18.96",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.96.tgz",
+      "integrity": "sha512-84iSqGXoO+Ha16j8pRZ/L90vDMKX04QTYMTfYeE1WrjWaZXuchBehGUZEpNgx7JnmlrIHdnABmpjrQjhCnNldQ=="
     },
     "node_modules/copy-to-clipboard": {
       "version": "3.3.3",
@@ -4266,9 +4272,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.728",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.728.tgz",
-      "integrity": "sha512-Ud1v7hJJYIqehlUJGqR6PF1Ek8l80zWwxA6nGxigBsGJ9f9M2fciHyrIiNMerSHSH3p+0/Ia7jIlnDkt41h5cw=="
+      "version": "1.4.744",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.744.tgz",
+      "integrity": "sha512-nAGcF0yeKKfrP13LMFr5U1eghfFSvFLg302VUFzWlcjPOnUYd52yU5x6PBYrujhNbc4jYmZFrGZFK+xasaEzVA=="
     },
     "node_modules/elliptic": {
       "version": "6.5.4",
@@ -5401,9 +5407,9 @@
       "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="
     },
     "node_modules/gsplat": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/gsplat/-/gsplat-1.2.3.tgz",
-      "integrity": "sha512-5ZthImDFbCKpREgz7SBJUnYRd5GvA5V+0aeaBO4B0M5A/Y5fQwcoeYXioJMcmwfTVRGvHi8W1e2g7RqU1XpbxA=="
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/gsplat/-/gsplat-1.2.4.tgz",
+      "integrity": "sha512-YW/qc1KxmkI8VGfVY0avZ6wpcjOijHdXvNkLSy8KniY3R326bYYbePMBlI1VrGQ60dkj4dWnW2k779jKTWNXyQ=="
     },
     "node_modules/has-bigints": {
       "version": "1.0.2",
@@ -6319,12 +6325,12 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="
     },
     "node_modules/next": {
-      "version": "14.1.4",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.1.4.tgz",
-      "integrity": "sha512-1WTaXeSrUwlz/XcnhGTY7+8eiaFvdet5z9u3V2jb+Ek1vFo0VhHKSAIJvDWfQpttWjnyw14kBeq28TPq7bTeEQ==",
+      "version": "14.2.2",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.2.2.tgz",
+      "integrity": "sha512-oGwUaa2bCs47FbuxWMpOoXtBMPYpvTPgdZr3UAo+pu7Ns00z9otmYpoeV1HEiYL06AlRQQIA/ypK526KjJfaxg==",
       "dependencies": {
-        "@next/env": "14.1.4",
-        "@swc/helpers": "0.5.2",
+        "@next/env": "14.2.2",
+        "@swc/helpers": "0.5.5",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
         "graceful-fs": "^4.2.11",
@@ -6338,18 +6344,19 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.1.4",
-        "@next/swc-darwin-x64": "14.1.4",
-        "@next/swc-linux-arm64-gnu": "14.1.4",
-        "@next/swc-linux-arm64-musl": "14.1.4",
-        "@next/swc-linux-x64-gnu": "14.1.4",
-        "@next/swc-linux-x64-musl": "14.1.4",
-        "@next/swc-win32-arm64-msvc": "14.1.4",
-        "@next/swc-win32-ia32-msvc": "14.1.4",
-        "@next/swc-win32-x64-msvc": "14.1.4"
+        "@next/swc-darwin-arm64": "14.2.2",
+        "@next/swc-darwin-x64": "14.2.2",
+        "@next/swc-linux-arm64-gnu": "14.2.2",
+        "@next/swc-linux-arm64-musl": "14.2.2",
+        "@next/swc-linux-x64-gnu": "14.2.2",
+        "@next/swc-linux-x64-musl": "14.2.2",
+        "@next/swc-win32-arm64-msvc": "14.2.2",
+        "@next/swc-win32-ia32-msvc": "14.2.2",
+        "@next/swc-win32-x64-msvc": "14.2.2"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
+        "@playwright/test": "^1.41.2",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "sass": "^1.3.0"
@@ -6358,6 +6365,9 @@
         "@opentelemetry/api": {
           "optional": true
         },
+        "@playwright/test": {
+          "optional": true
+        },
         "sass": {
           "optional": true
         }
@@ -6596,9 +6606,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.36.0",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.36.0.tgz",
-      "integrity": "sha512-AtYrhhWY64LhB9P6f3H0nV8nTSaQJ89mWPnfNU5CnYg81zlYaV8nkyO+aTNfprdqP/9xv10woNNUgefXINT4Dg==",
+      "version": "4.38.1",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.1.tgz",
+      "integrity": "sha512-nmSKE9O2piuoh9+AgDqwGHojIFSxToQ2jJqwaxjbzz2ebdD5LYY9s+bMe25b18t4QEgvtgW70JfK8BU3xf5dRw==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -6971,9 +6981,9 @@
       }
     },
     "node_modules/qs": {
-      "version": "6.12.0",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.0.tgz",
-      "integrity": "sha512-trVZiI6RMOkO476zLGaBIzszOdFPnCCXHPG9kn0yuS1uz6xdVxPfZdB3vUig9pxPFDM9BRAgz/YUIVQ1/vuiUg==",
+      "version": "6.12.1",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.1.tgz",
+      "integrity": "sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==",
       "dependencies": {
         "side-channel": "^1.0.6"
       },
@@ -7981,11 +7991,11 @@
       }
     },
     "node_modules/tailwind-merge": {
-      "version": "2.2.2",
-      "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.2.2.tgz",
-      "integrity": "sha512-tWANXsnmJzgw6mQ07nE3aCDkCK4QdT3ThPMCzawoYA2Pws7vSTCvz3Vrjg61jVUGfFZPJzxEP+NimbcW+EdaDw==",
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.3.0.tgz",
+      "integrity": "sha512-vkYrLpIP+lgR0tQCG6AP7zZXCTLc1Lnv/CCRT3BqJ9CZ3ui2++GPaGb1x/ILsINIMSYqqvrpqjUFsMNLlW99EA==",
       "dependencies": {
-        "@babel/runtime": "^7.24.0"
+        "@babel/runtime": "^7.24.1"
       },
       "funding": {
         "type": "github",

package.json

@@ -12,7 +12,7 @@
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",
     "@jcoreio/async-throttle": "^1.6.0",
-    "@mediapipe/tasks-vision": "^0.10.12",
+    "@mediapipe/tasks-vision": "^0.10.13-rc.20240419",
     "@photo-sphere-viewer/core": "^5.7.2",
     "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2",
     "@photo-sphere-viewer/gyroscope-plugin": "^5.7.2",

src/app/api/resolvers/image/route.ts

@@ -4,7 +4,6 @@ import queryString from "query-string"
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
-import { getContentType } from "@/lib/data/getContentType"
 
 export async function GET(req: NextRequest) {
 
@@ -22,8 +21,8 @@ let prompt = ""
   // console.log("calling await newRender")
 
   let render = await newRender({
-    prompt,
-    negativePrompt: "blurry, cropped, bad quality",
+    prompt: `${prompt}, cinematic, photo, sublime, pro quality, sharp, crisp, beautiful, impressive, amazing, high quality, 4K`,
+    negativePrompt: "logo, text, ui, hud, interface, buttons, ad, signature, copyright, blurry, cropped, bad quality",
     nbFrames: 1,
     nbSteps: 8,
     width: 1024,

src/app/api/resolvers/video/route.ts

@@ -22,10 +22,10 @@ let prompt = ""
   // console.log("calling await newRender")
 
   let render = await newRender({
-    prompt,
-    negativePrompt: "blurry, cropped, bad quality",
+    prompt: `${prompt}, cinematic, photo, sublime, pro quality, sharp, crisp, beautiful, impressive, amazing, high quality, 4K`,
+    negativePrompt: "logo, text, ui, hud, interface, buttons, ad, signature, copyright, blurry, cropped, bad quality",
     nbFrames: 1,
-    nbSteps: 4,
+    nbSteps: 8,
     width: 1024,
     height: 576,
     turbo: true,

src/components/interface/latent-engine/core/drawSegmentation.ts

@@ -1,36 +1,61 @@
-import { MPMask } from "@mediapipe/tasks-vision"
+import { MPMask } from "@mediapipe/tasks-vision";
+
+interface DrawSegmentationOptions {
+  mask?: MPMask;
+  canvas?: HTMLCanvasElement;
+  backgroundImage?: HTMLImageElement;
+  fillStyle?: string;
+}
 
 /**
- * Draw segmentation result
+ * Draw segmentation result with enhancements.
  */
-export function drawSegmentation(mask?: MPMask, canvas?: HTMLCanvasElement) {
-
-  if (!mask) { throw new Error("drawSegmentation failed: empty mask") }
+export function drawSegmentation(options: DrawSegmentationOptions): HTMLCanvasElement {
+  const {
+    mask,
+    canvas,
+    backgroundImage,
+    fillStyle = "rgba(255, 255, 255, 1.0)"
+  } = options;
 
-  if (!canvas) { throw new Error("drawSegmentation failed: cannot access the canvas") }
+  if (!canvas) {
+    throw new Error("drawSegmentation failed: cannot access the canvas");
+  }
 
-  const width = mask.width;
-  const height = mask.height;
-  const maskData = mask.getAsFloat32Array();
+  const width = mask?.width || 0;
+  const height = mask?.height || 0;
 
-  canvas.width = width;
-  canvas.height = height;
+  canvas.width = width || canvas.width;
+  canvas.height = height || canvas.height;
 
   console.log("drawSegmentation: drawing..")
 
   const ctx = canvas.getContext("2d")
+  if (!ctx) {
+    throw new Error("drawSegmentation failed: cannot access the 2D context")
+  }
 
-  if (!ctx) { throw new Error("drawSegmentation failed: cannot access the 2D context") }
-
-  ctx.fillStyle = "#00000000";
+  ctx.fillStyle = "#00000000"; // Maintain transparent background if no image provided
   ctx.fillRect(0, 0, width, height);
-  ctx.fillStyle = "rgba(18, 181, 203, 0.7)";
-
-  maskData.forEach((category: number, index: number, array: Float32Array) => {
-    if (Math.round(category * 255.0) === 0) {
-      const x = (index + 1) % width;
-      const y = (index + 1 - x) / width;
-      ctx.fillRect(x, y, 1, 1);
-    }
-  })
+
+  // Draw the background image if provided, otherwise default to transparent background.
+  if (backgroundImage) {
+    ctx.drawImage(backgroundImage, 0, 0, width, height);
+  }
+
+  if (mask) {
+  
+    ctx.fillStyle = fillStyle;
+    const maskData = mask.getAsFloat32Array();
+
+    maskData.forEach((category: number, index: number) => {
+      if (Math.round(category * 255.0) === 0) {
+        const x = index % width;
+        const y = Math.floor(index / width);
+        ctx.fillRect(x, y, 1, 1);
+      }
+    });
+  }
+
+  return canvas;
 }

src/components/interface/latent-engine/core/engine.tsx

@@ -11,9 +11,6 @@ import { ContentLayer } from "../components/content-layer"
 import { MediaInfo } from "@/types/general"
 import { getMockClap } from "@/lib/clap/getMockClap"
 import { serializeClap } from "@/lib/clap/serializeClap"
-import { blobToDataUri } from "@/app/api/utils/blobToDataUri"
-import { InteractiveSegmentationCanvas } from "@/lib/on-device-ai/getInteractiveSegmentationCanvas"
-import { InteractiveSegmenterResult } from "@mediapipe/tasks-vision"
 
 function LatentEngine({
   media,

src/components/interface/latent-engine/resolvers/resolveSegment.ts

@@ -13,10 +13,8 @@ export async function resolveSegment(segment: ClapSegment, clap: ClapProject): P
   if (segment.category === "interface") {
     latentComponentResolver = interfaceResolver
   } else if (segment.category === "video") {
-
-    // for backend performance reason (generative video is slow)
-    // we do not support "true" video for now
-    // latentComponentResolver = videoResolver
+    latentComponentResolver = videoResolver
+  } else if (segment.category === "storyboard") {
     latentComponentResolver = imageResolver
   }
 

src/components/interface/latent-engine/store/useLatentEngine.ts

@@ -14,6 +14,7 @@ import { parseClap } from "@/lib/clap/parseClap"
 import { InteractiveSegmenterResult, MPMask } from "@mediapipe/tasks-vision"
 import { segmentFrame } from "@/lib/on-device-ai/segmentFrameOnClick"
 import { drawSegmentation } from "../core/drawSegmentation"
+import { filterImage } from "@/lib/on-device-ai/filterImage"
 
 export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
   width: 1024,
@@ -144,7 +145,19 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       if (debug) {
         console.log(`processClickOnSegment: callling drawSegmentation`)
       }
-      drawSegmentation(result.categoryMask, segmentationElement)
+
+      const canvasMask: HTMLCanvasElement = drawSegmentation({
+        mask: result.categoryMask,
+        canvas: segmentationElement,
+        backgroundImage: imageElement,
+        fillStyle: "rgba(255, 255, 255, 1.0)"
+      })
+      // TODO: read the canvas te determine on what the user clicked
+
+      if (debug) {
+        console.log(`processClickOnSegment: filtering the original image`)
+      }
+      // filterImage(imageElement, canvasMask)
 
       if (debug) {
         console.log("processClickOnSegment: TODO call data.close() to free the memory!")
@@ -278,13 +291,24 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       if (get().isPlaying) {
         // console.log(`runSimulationLoop: rendering video content layer..`)
         // we only grab the first one
+    
         const videoLayer = (await resolveSegments(clap, "video", 1)).at(0)
 
         if (get().isPlaying) {
+
           set({
             videoLayer
           })
 
+          const { videoElement, imageElement, segmentationElement } = get()
+
+          const canvas = drawSegmentation({
+             // no mask means this will effectively clear the canvas
+            canvas: segmentationElement,
+            backgroundImage: imageElement,
+          })
+          
+
           console.log(`runSimulationLoop: rendered video content layer`)
         }
       }

src/components/interface/stream-tag/index.tsx

@@ -15,12 +15,14 @@ export function StreamTag({
   const isInteractive = streamType === "interactive"
   const isLive = streamType === "live"
   const isStatic = !isInteractive && !isLive
+  /*
   console.log("debug:", {
     streamType,
     isInteractive,
     isLive,
     isStatic
   })
+  */
 
   return (
     <div className={cn(`

src/lib/clap/getMockClap.ts

@@ -6,10 +6,11 @@ let defaultSegmentDurationInMs = 2000
 
 // const demoPrompt = "closeup of Queen angelfish, bokeh"
 // const demoPrompt = "portrait of a man tv news anchor, pierre-jean-hyves, serious, bokeh"
-const demoPrompt = "dogs and cats, playing in garden, balls, trees"
+// const demoPrompt = "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing"
+const demoPrompt = "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing"
 
 export function getMockClap({
-  prompt =demoPrompt,
+  prompt = demoPrompt,
   showDisclaimer = true,
 }: {
   prompt?: string

src/lib/on-device-ai/classifyFrame.ts

@@ -0,0 +1,52 @@
+import { FilesetResolver, ImageClassifier, ImageClassifierResult } from "@mediapipe/tasks-vision"
+
+export type InteractiveImageClassifier = (videoFrame: TexImageSource, x: number, y: number) => Promise<ImageClassifierResult>
+
+const getInteractiveImageClassifier = async (): Promise<InteractiveImageClassifier> => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
+  );
+
+  const imageClassifier = await ImageClassifier.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.tflite`
+    },
+    runningMode: "VIDEO",
+  });
+
+  const segmenter: InteractiveImageClassifier = (
+    videoFrame: TexImageSource,
+    x: number,
+    y: number
+  ): Promise<ImageClassifierResult> => {
+    return new Promise((resolve, reject) => {
+      imageClassifier.classify(
+        videoFrame
+        // TODO: there is a "region of interest field" we could use
+      )
+    })
+  }
+
+  return segmenter
+}
+
+
+const globalState: { classifier?: InteractiveImageClassifier } = {};
+
+(async () => {
+  globalState.classifier = globalState.classifier || (await getInteractiveImageClassifier())
+})();
+
+export async function classifyFrame(frame: TexImageSource, x: number, y: number): Promise<ImageClassifierResult> {
+  console.log("classifyFrame: loading classifier..")
+  globalState.classifier = globalState.classifier || (await getInteractiveImageClassifier())
+
+  console.log("classifyFrame: segmenting..")
+  return globalState.classifier(frame, x, y)
+}
+
+// to run:
+
+// see doc:
+// https://developers.google.com/mediapipe/solutions/vision/image_segmenter/web_js#video
+// imageSegmenter.segmentForVideo(video, startTimeMs, callbackForVideo);

src/lib/on-device-ai/filterImage.ts

@@ -0,0 +1,55 @@
+/**
+ * Applies a mask to an image using canvas blending modes to avoid explicit pixel iteration.
+ * 
+ * @param image The source image as an HTMLImageElement.
+ * @param maskCanvas The canvas element containing a mask.
+ * @returns A new Promise resolving to a headless canvas element containing the masked image.
+ */
+export async function filterImage(
+  image?: HTMLImageElement,
+  maskCanvas?: HTMLCanvasElement
+): Promise<HTMLCanvasElement> {
+  return new Promise((resolve, reject) => {
+
+    // Create a new canvas to construct the final image
+    const resultCanvas = document.createElement("canvas");
+
+    if (!image) {
+      // reject(`missing image (must be a HTMLImageElement)`)
+      resolve(resultCanvas)
+      return
+    }
+
+    resultCanvas.width = image.width;
+    resultCanvas.height = image.height;
+
+    if (!maskCanvas) {
+      // reject(`missing image (must be a HTMLImageElement)`)
+      resolve(resultCanvas)
+      return
+    }
+
+    const ctx = resultCanvas.getContext("2d");
+    if (!ctx) {
+        reject(new Error("Failed to get 2D context"));
+        return;
+    }
+
+    // Draw the original image
+    ctx.drawImage(image, 0, 0, image.width, image.height);
+
+    // Set blending mode to 'destination-in' to keep only the content that overlaps the mask
+    ctx.globalCompositeOperation = "destination-in";
+    ctx.drawImage(maskCanvas, 0, 0, image.width, image.height);
+
+    // Reset composite operation to default
+    ctx.globalCompositeOperation = "source-over";
+
+    // Draw white rectangle where the mask is opaque
+    ctx.globalCompositeOperation = "destination-over";
+    ctx.fillStyle = "white";
+    ctx.fillRect(0, 0, image.width, image.height);
+
+    resolve(resultCanvas);
+  });
+}

src/lib/on-device-ai/getInteractiveSegmentationCanvas.tsx

@@ -1,37 +0,0 @@
-import { useRef } from "react"
-import { segmentFrame } from "./segmentFrameOnClick"
-import { ImageSource, InteractiveSegmenterResult } from "@mediapipe/tasks-vision"
-
-
-export function InteractiveSegmentationCanvas({
-  src,
-  onClick,
-}: {
-  src?: ImageSource
-  onClick?: (results: InteractiveSegmenterResult ) => void
-}) {
-  const segmentationClickRef = useRef<HTMLDivElement>(null)
-  return (
-    <div 
-      ref={segmentationClickRef}
-      onClick={(event) => {
-        if (!segmentationClickRef.current || !src || !onClick) { return }
-        
-        const box = segmentationClickRef.current.getBoundingClientRect()
-
-        const px = event.clientX
-        const py = event.clientY
-
-        const x = px / box.width
-        const y = py / box.height
-
-        const fn = async () => {
-          const results: InteractiveSegmenterResult = await segmentFrame(src, x, y);
-          onClick(results)
-        }
-        fn()
-
-    }}>
-    </div>
-  )
-}

src/lib/on-device-ai/getSegmentationCanvas.tsx

@@ -1,6 +1,6 @@
 
 import React from "react"
-import { ImageSegmenterResult, ImageSource } from "@mediapipe/tasks-vision"
+import { ImageSegmenterResult } from "@mediapipe/tasks-vision"
 
 import { segmentFrame } from "./segmentFrame"
 
@@ -10,7 +10,7 @@ export async function getSegmentationCanvas({
   width,
   height
 }: {
-  frame: ImageSource;
+  frame: TexImageSource;
   timestamp: number;
   width: number;
   height: number;

src/lib/on-device-ai/identifyFrame.ts

@@ -1,11 +1,10 @@
 import {
   FilesetResolver,
   ObjectDetector,
-  ObjectDetectorResult,
-  ImageSource
+  ObjectDetectorResult
 } from "@mediapipe/tasks-vision"
 
-export type VideoObjectDetector = (videoFrame: ImageSource, timestamp: number) => Promise<ObjectDetectorResult>
+export type VideoObjectDetector = (videoFrame: TexImageSource, timestamp: number) => Promise<ObjectDetectorResult>
 
 const getObjectDetector = async (): Promise<VideoObjectDetector> => {
   const vision = await FilesetResolver.forVisionTasks(
@@ -20,7 +19,7 @@ const getObjectDetector = async (): Promise<VideoObjectDetector> => {
     runningMode: "VIDEO"
   });
 
-  const detector: VideoObjectDetector = async (videoFrame: ImageSource, timestamp: number): Promise<ObjectDetectorResult> => {
+  const detector: VideoObjectDetector = async (videoFrame: TexImageSource, timestamp: number): Promise<ObjectDetectorResult> => {
     const result = objectDetector.detectForVideo(videoFrame, timestamp)
     return result
   }
@@ -35,7 +34,7 @@ const globalState: { detector?: VideoObjectDetector } = {};
   globalState.detector = globalState.detector || (await getObjectDetector())
 })();
 
-export async function identifyFrame(frame: ImageSource, timestamp: number): Promise<ObjectDetectorResult> {
+export async function identifyFrame(frame: TexImageSource, timestamp: number): Promise<ObjectDetectorResult> {
   console.log("identifyFrame: loading segmenter..")
   globalState.detector = globalState.detector || (await getObjectDetector())
 

src/lib/on-device-ai/segmentFrame.ts

@@ -1,6 +1,6 @@
-import { FilesetResolver, ImageSegmenter, ImageSegmenterResult, ImageSource } from "@mediapipe/tasks-vision"
+import { FilesetResolver, ImageSegmenter, ImageSegmenterResult } from "@mediapipe/tasks-vision"
 
-export type VideoSegmenter = (videoFrame: ImageSource, timestamp: number) => Promise<ImageSegmenterResult>
+export type VideoSegmenter = (videoFrame: TexImageSource, timestamp: number) => Promise<ImageSegmenterResult>
 
 const getSegmenter = async (): Promise<VideoSegmenter> => {
   const vision = await FilesetResolver.forVisionTasks(
@@ -24,7 +24,7 @@ const getSegmenter = async (): Promise<VideoSegmenter> => {
     runningMode: "VIDEO"
   });
 
-  const segmenter: VideoSegmenter = (videoFrame: ImageSource, timestamp: number): Promise<ImageSegmenterResult> => {
+  const segmenter: VideoSegmenter = (videoFrame: TexImageSource, timestamp: number): Promise<ImageSegmenterResult> => {
     return new Promise((resolve, reject) => {
       imageSegmenter.segmentForVideo(videoFrame, timestamp, (results) => {
         resolve(results)
@@ -42,7 +42,7 @@ const globalState: { segmenter?: VideoSegmenter } = {};
   globalState.segmenter = globalState.segmenter || (await getSegmenter())
 })();
 
-export async function segmentFrame(frame: ImageSource, timestamp: number): Promise<ImageSegmenterResult> {
+export async function segmentFrame(frame: TexImageSource, timestamp: number): Promise<ImageSegmenterResult> {
   console.log("segmentFrame: loading segmenter..")
   globalState.segmenter = globalState.segmenter || (await getSegmenter())
 

src/lib/on-device-ai/segmentFrameOnClick.ts

@@ -1,6 +1,6 @@
-import { FilesetResolver, InteractiveSegmenter, InteractiveSegmenterResult, ImageSource } from "@mediapipe/tasks-vision"
+import { FilesetResolver, InteractiveSegmenter, InteractiveSegmenterResult } from "@mediapipe/tasks-vision"
 
-export type InteractiveVideoSegmenter = (videoFrame: ImageSource, x: number, y: number) => Promise<InteractiveSegmenterResult>
+export type InteractiveVideoSegmenter = (videoFrame: TexImageSource, x: number, y: number) => Promise<InteractiveSegmenterResult>
 
 const getInteractiveSegmenter = async (): Promise<InteractiveVideoSegmenter> => {
   const vision = await FilesetResolver.forVisionTasks(
@@ -17,7 +17,7 @@ const getInteractiveSegmenter = async (): Promise<InteractiveVideoSegmenter> =>
   });
 
   const segmenter: InteractiveVideoSegmenter = (
-    videoFrame: ImageSource,
+    videoFrame: TexImageSource,
     x: number,
     y: number
   ): Promise<InteractiveSegmenterResult> => {
@@ -43,7 +43,7 @@ const globalState: { segmenter?: InteractiveVideoSegmenter } = {};
   globalState.segmenter = globalState.segmenter || (await getInteractiveSegmenter())
 })();
 
-export async function segmentFrame(frame: ImageSource, x: number, y: number): Promise<InteractiveSegmenterResult> {
+export async function segmentFrame(frame: TexImageSource, x: number, y: number): Promise<InteractiveSegmenterResult> {
   console.log("segmentFrame: loading segmenter..")
   globalState.segmenter = globalState.segmenter || (await getInteractiveSegmenter())