handle file embedding

package-lock.json

@@ -15,11 +15,13 @@
                 "@types/react": "18.2.6",
                 "@types/react-dom": "18.2.4",
                 "@xenova/transformers": "^2.1.1",
+                "dexie": "^3.2.4",
                 "eslint": "8.40.0",
                 "eslint-config-next": "13.4.2",
                 "fs-extra": "^11.1.1",
                 "langchain": "^0.0.90",
                 "next": "13.4.2",
+                "pdfjs-dist": "^3.7.107",
                 "react": "18.2.0",
                 "react-dom": "18.2.0",
                 "react-dropzone": "^14.2.3",
@@ -987,6 +989,26 @@
             "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz",
             "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw=="
         },
+        "node_modules/@mapbox/node-pre-gyp": {
+            "version": "1.0.10",
+            "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.10.tgz",
+            "integrity": "sha512-4ySo4CjzStuprMwk35H5pPbkymjv1SF3jGLj6rAHp/xT/RF7TL7bd9CTm1xDY49K2qF7jmR/g7k+SkLETP6opA==",
+            "optional": true,
+            "dependencies": {
+                "detect-libc": "^2.0.0",
+                "https-proxy-agent": "^5.0.0",
+                "make-dir": "^3.1.0",
+                "node-fetch": "^2.6.7",
+                "nopt": "^5.0.0",
+                "npmlog": "^5.0.1",
+                "rimraf": "^3.0.2",
+                "semver": "^7.3.5",
+                "tar": "^6.1.11"
+            },
+            "bin": {
+                "node-pre-gyp": "bin/node-pre-gyp"
+            }
+        },
         "node_modules/@next/env": {
             "version": "13.4.2",
             "resolved": "https://registry.npmjs.org/@next/env/-/env-13.4.2.tgz",
@@ -1583,6 +1605,12 @@
                 "onnxruntime-node": "1.14.0"
             }
         },
+        "node_modules/abbrev": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+            "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+            "optional": true
+        },
         "node_modules/acorn": {
             "version": "8.8.2",
             "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz",
@@ -1602,6 +1630,18 @@
                 "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
             }
         },
+        "node_modules/agent-base": {
+            "version": "6.0.2",
+            "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+            "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+            "optional": true,
+            "dependencies": {
+                "debug": "4"
+            },
+            "engines": {
+                "node": ">= 6.0.0"
+            }
+        },
         "node_modules/ajv": {
             "version": "6.12.6",
             "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -1639,6 +1679,25 @@
                 "url": "https://github.com/chalk/ansi-styles?sponsor=1"
             }
         },
+        "node_modules/aproba": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz",
+            "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
+            "optional": true
+        },
+        "node_modules/are-we-there-yet": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
+            "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
+            "optional": true,
+            "dependencies": {
+                "delegates": "^1.0.0",
+                "readable-stream": "^3.6.0"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/argparse": {
             "version": "2.0.1",
             "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
@@ -2061,6 +2120,56 @@
                 }
             ]
         },
+        "node_modules/canvas": {
+            "version": "2.11.2",
+            "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz",
+            "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==",
+            "hasInstallScript": true,
+            "optional": true,
+            "dependencies": {
+                "@mapbox/node-pre-gyp": "^1.0.0",
+                "nan": "^2.17.0",
+                "simple-get": "^3.0.3"
+            },
+            "engines": {
+                "node": ">=6"
+            }
+        },
+        "node_modules/canvas/node_modules/decompress-response": {
+            "version": "4.2.1",
+            "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
+            "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
+            "optional": true,
+            "dependencies": {
+                "mimic-response": "^2.0.0"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/canvas/node_modules/mimic-response": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+            "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
+            "optional": true,
+            "engines": {
+                "node": ">=8"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/canvas/node_modules/simple-get": {
+            "version": "3.1.1",
+            "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz",
+            "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==",
+            "optional": true,
+            "dependencies": {
+                "decompress-response": "^4.2.0",
+                "once": "^1.3.1",
+                "simple-concat": "^1.0.0"
+            }
+        },
         "node_modules/chalk": {
             "version": "4.1.2",
             "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -2123,6 +2232,15 @@
                 "simple-swizzle": "^0.2.2"
             }
         },
+        "node_modules/color-support": {
+            "version": "1.1.3",
+            "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz",
+            "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==",
+            "optional": true,
+            "bin": {
+                "color-support": "bin.js"
+            }
+        },
         "node_modules/combined-stream": {
             "version": "1.0.8",
             "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -2149,6 +2267,12 @@
             "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
             "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="
         },
+        "node_modules/console-control-strings": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
+            "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==",
+            "optional": true
+        },
         "node_modules/convert-source-map": {
             "version": "1.9.0",
             "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz",
@@ -2343,6 +2467,12 @@
                 "node": ">=0.4.0"
             }
         },
+        "node_modules/delegates": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+            "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+            "optional": true
+        },
         "node_modules/detect-libc": {
             "version": "2.0.1",
             "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz",
@@ -2351,6 +2481,14 @@
                 "node": ">=8"
             }
         },
+        "node_modules/dexie": {
+            "version": "3.2.4",
+            "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.2.4.tgz",
+            "integrity": "sha512-VKoTQRSv7+RnffpOJ3Dh6ozknBqzWw/F3iqMdsZg958R0AS8AnY9x9d1lbwENr0gzeGJHXKcGhAMRaqys6SxqA==",
+            "engines": {
+                "node": ">=6.0"
+            }
+        },
         "node_modules/dir-glob": {
             "version": "3.0.1",
             "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@@ -3288,6 +3426,30 @@
                 "node": ">=14.14"
             }
         },
+        "node_modules/fs-minipass": {
+            "version": "2.1.0",
+            "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
+            "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
+            "optional": true,
+            "dependencies": {
+                "minipass": "^3.0.0"
+            },
+            "engines": {
+                "node": ">= 8"
+            }
+        },
+        "node_modules/fs-minipass/node_modules/minipass": {
+            "version": "3.3.6",
+            "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+            "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+            "optional": true,
+            "dependencies": {
+                "yallist": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/fs.realpath": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -3337,6 +3499,26 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/gauge": {
+            "version": "3.0.2",
+            "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
+            "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
+            "optional": true,
+            "dependencies": {
+                "aproba": "^1.0.3 || ^2.0.0",
+                "color-support": "^1.1.2",
+                "console-control-strings": "^1.0.0",
+                "has-unicode": "^2.0.1",
+                "object-assign": "^4.1.1",
+                "signal-exit": "^3.0.0",
+                "string-width": "^4.2.3",
+                "strip-ansi": "^6.0.1",
+                "wide-align": "^1.1.2"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/gensync": {
             "version": "1.0.0-beta.2",
             "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -3579,6 +3761,12 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/has-unicode": {
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
+            "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==",
+            "optional": true
+        },
         "node_modules/hoist-non-react-statics": {
             "version": "3.3.2",
             "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz",
@@ -3588,6 +3776,19 @@
                 "react-is": "^16.7.0"
             }
         },
+        "node_modules/https-proxy-agent": {
+            "version": "5.0.1",
+            "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+            "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+            "optional": true,
+            "dependencies": {
+                "agent-base": "6",
+                "debug": "4"
+            },
+            "engines": {
+                "node": ">= 6"
+            }
+        },
         "node_modules/human-signals": {
             "version": "4.3.1",
             "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz",
@@ -3800,6 +4001,15 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/is-fullwidth-code-point": {
+            "version": "3.0.0",
+            "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
+            "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
+            "optional": true,
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/is-glob": {
             "version": "4.0.3",
             "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
@@ -4577,6 +4787,52 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/minipass": {
+            "version": "5.0.0",
+            "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+            "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+            "optional": true,
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/minizlib": {
+            "version": "2.1.2",
+            "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
+            "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==",
+            "optional": true,
+            "dependencies": {
+                "minipass": "^3.0.0",
+                "yallist": "^4.0.0"
+            },
+            "engines": {
+                "node": ">= 8"
+            }
+        },
+        "node_modules/minizlib/node_modules/minipass": {
+            "version": "3.3.6",
+            "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+            "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+            "optional": true,
+            "dependencies": {
+                "yallist": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/mkdirp": {
+            "version": "1.0.4",
+            "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+            "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+            "optional": true,
+            "bin": {
+                "mkdirp": "bin/cmd.js"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/mkdirp-classic": {
             "version": "0.5.3",
             "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
@@ -4627,6 +4883,12 @@
             "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
             "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
         },
+        "node_modules/nan": {
+            "version": "2.17.0",
+            "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz",
+            "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==",
+            "optional": true
+        },
         "node_modules/nanoid": {
             "version": "3.3.6",
             "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz",
@@ -4747,6 +5009,21 @@
             "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.12.tgz",
             "integrity": "sha512-QzsYKWhXTWx8h1kIvqfnC++o0pEmpRQA/aenALsL2F4pqNVr7YzcdMlDij5WBnwftRbJCNJL/O7zdKaxKPHqgQ=="
         },
+        "node_modules/nopt": {
+            "version": "5.0.0",
+            "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
+            "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==",
+            "optional": true,
+            "dependencies": {
+                "abbrev": "1"
+            },
+            "bin": {
+                "nopt": "bin/nopt.js"
+            },
+            "engines": {
+                "node": ">=6"
+            }
+        },
         "node_modules/normalize-range": {
             "version": "0.1.2",
             "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz",
@@ -4780,6 +5057,18 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/npmlog": {
+            "version": "5.0.1",
+            "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
+            "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
+            "optional": true,
+            "dependencies": {
+                "are-we-there-yet": "^2.0.0",
+                "console-control-strings": "^1.1.0",
+                "gauge": "^3.0.0",
+                "set-blocking": "^2.0.0"
+            }
+        },
         "node_modules/num-sort": {
             "version": "2.1.0",
             "resolved": "https://registry.npmjs.org/num-sort/-/num-sort-2.1.0.tgz",
@@ -5146,6 +5435,27 @@
                 "node": ">=8"
             }
         },
+        "node_modules/path2d-polyfill": {
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/path2d-polyfill/-/path2d-polyfill-2.0.1.tgz",
+            "integrity": "sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA==",
+            "optional": true,
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/pdfjs-dist": {
+            "version": "3.7.107",
+            "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.7.107.tgz",
+            "integrity": "sha512-D0pZgz4u8VNjCCXfNCjzeFcenxKFGh2RnGlLIvDHgpT+ZvOlsBlHugpOfmcLQyKk7csHBhPalf+mCM6thiclSQ==",
+            "engines": {
+                "node": ">=18"
+            },
+            "optionalDependencies": {
+                "canvas": "^2.11.2",
+                "path2d-polyfill": "^2.0.1"
+            }
+        },
         "node_modules/picocolors": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
@@ -5790,6 +6100,12 @@
                 "node": ">=10"
             }
         },
+        "node_modules/set-blocking": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+            "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==",
+            "optional": true
+        },
         "node_modules/shallowequal": {
             "version": "1.1.0",
             "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
@@ -5955,6 +6271,26 @@
                 "safe-buffer": "~5.2.0"
             }
         },
+        "node_modules/string-width": {
+            "version": "4.2.3",
+            "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+            "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+            "optional": true,
+            "dependencies": {
+                "emoji-regex": "^8.0.0",
+                "is-fullwidth-code-point": "^3.0.0",
+                "strip-ansi": "^6.0.1"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/string-width/node_modules/emoji-regex": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+            "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+            "optional": true
+        },
         "node_modules/string.prototype.matchall": {
             "version": "4.0.8",
             "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.8.tgz",
@@ -6174,6 +6510,23 @@
                 "node": ">=6"
             }
         },
+        "node_modules/tar": {
+            "version": "6.1.15",
+            "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.15.tgz",
+            "integrity": "sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A==",
+            "optional": true,
+            "dependencies": {
+                "chownr": "^2.0.0",
+                "fs-minipass": "^2.0.0",
+                "minipass": "^5.0.0",
+                "minizlib": "^2.1.1",
+                "mkdirp": "^1.0.3",
+                "yallist": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/tar-fs": {
             "version": "2.1.1",
             "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
@@ -6200,6 +6553,15 @@
                 "node": ">=6"
             }
         },
+        "node_modules/tar/node_modules/chownr": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+            "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+            "optional": true,
+            "engines": {
+                "node": ">=10"
+            }
+        },
         "node_modules/text-table": {
             "version": "0.2.0",
             "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -6497,6 +6859,15 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/wide-align": {
+            "version": "1.1.5",
+            "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz",
+            "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==",
+            "optional": true,
+            "dependencies": {
+                "string-width": "^1.0.2 || 2 || 3 || 4"
+            }
+        },
         "node_modules/word-wrap": {
             "version": "1.2.3",
             "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",

package.json

@@ -15,11 +15,13 @@
         "@types/react": "18.2.6",
         "@types/react-dom": "18.2.4",
         "@xenova/transformers": "^2.1.1",
+        "dexie": "^3.2.4",
         "eslint": "8.40.0",
         "eslint-config-next": "13.4.2",
         "fs-extra": "^11.1.1",
         "langchain": "^0.0.90",
         "next": "13.4.2",
+        "pdfjs-dist": "^3.7.107",
         "react": "18.2.0",
         "react-dom": "18.2.0",
         "react-dropzone": "^14.2.3",

src/components/DashButton.tsx

@@ -0,0 +1,19 @@
+import React from 'react';
+import { MouseEventHandler, ReactElement } from 'react';
+
+interface Props {
+  handleClick: MouseEventHandler<HTMLButtonElement>;
+  children: ReactElement;
+}
+const DashButton = ({ handleClick, children }: Props) => {
+  return (
+    <button
+      className="border-dashed border-2 border-black rounded-lg py-2 px-4 text-black hover:bg-black hover:text-white transition-all duration-300 ease-in-out"
+      onClick={handleClick}
+    >
+      {children}
+    </button>
+  );
+};
+
+export { DashButton };

src/components/FileLoader.jsx

@@ -0,0 +1,70 @@
+import { useState } from 'react';
+import { db } from '@/utils/db-client';
+import { DashButton } from './DashButton'
+import { FileEmbedder } from './FileEmbedder';
+import * as PDFJS from 'pdfjs-dist/build/pdf';
+
+PDFJS.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${PDFJS.version}/pdf.worker.min.js`;
+
+export default class Pdf {
+  static async getPageText(pdf, pageNo) {
+    const page = await pdf.getPage(pageNo);
+    const tokenizedText = await page.getTextContent();
+    const pageText = tokenizedText.items.map((token) => token.str).join('');
+    return pageText;
+  }
+
+  static async getPDFText(source) {
+    const pdf = await PDFJS.getDocument(source).promise;
+    const maxPages = pdf.numPages;
+    const pageTextPromises = [];
+    for (let pageNo = 1; pageNo <= maxPages; pageNo += 1) {
+      pageTextPromises.push(Pdf.getPageText(pdf, pageNo));
+    }
+    const pageTexts = await Promise.all(pageTextPromises);
+    return pageTexts.join(' ');
+  }
+}
+
+export const FileLoader = () => {
+  const [files, setFiles] = useState();
+  const [uploadStatus, setUploadStatus] = useState("Embed");
+  const handleEmbed = (files) => {
+    setFiles(files)
+  };
+
+  return (
+    <>
+      <FileEmbedder handleEmbed={handleEmbed} />
+      <DashButton
+        handleClick={async () => {
+          if (files && files.length) {
+            const file = files[0];
+            let text;
+            const blob = new Blob([file], { type: 'text/plain' });
+            if (file.type === "application/pdf") {
+              text = await Pdf.getPDFText(URL.createObjectURL(blob));
+            } else {
+              text = URL.createObjectURL(blob);
+            }
+            const response = await fetch('/api/docHandle', {
+              method: 'POST',
+              body: JSON.stringify({ text }),
+            });
+            const docChat = await response.json();
+            console.log('handleDocs-getModels: ', docChat);
+            const id = await db.docs.add({
+              fileName: file.name || '',
+              fileSourceData: pdfText,
+              model: docChat,
+            });
+            setUploadStatus("Embedding Completed");
+            console.log('handleDocs-setToDb', id);
+          }
+        }}
+      >
+        <div>{uploadStatus}</div>
+      </DashButton>
+    </>
+  );
+};

src/components/FileLoader.tsx

@@ -1,29 +0,0 @@
-import { useState } from 'react';
-import { MouseEventHandler, ReactElement } from 'react';
-
-import { FileEmbedder } from './FileEmbedder';
-
-export const FileLoader = () => {
-  const [files, setFiles] = useState<File[]>()
-  const handleEmbed = (files: File[]) => {
-    setFiles(files)
-    console.log(files);
-  };
-
-  const handleClick: MouseEventHandler<HTMLButtonElement> = (e) => {
-    console.log(e);
-  };
-
-  return (
-    <>
-      <FileEmbedder handleEmbed={handleEmbed} />
-      <button
-        onClick={() => {
-          console.log('sure');
-        }}
-      >
-        <div>Embed</div>
-      </button>
-    </>
-  );
-};

src/pages/api/docChat.ts

@@ -0,0 +1,58 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+import { storesDir } from '@/utils/file-handler';
+
+import { makeChain } from '@/utils/make-chain';
+import { XenovaTransformersEmbeddings } from '../../embed/hf';
+import { HNSWLib } from 'langchain/vectorstores/hnswlib';
+
+let vectorStore: HNSWLib;
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse,
+) {
+  const { prompt, messages } = req.body;
+
+  if (!prompt) {
+    return res.status(400).json({ message: 'No question in the request' });
+  }
+  // OpenAI recommends replacing newlines with spaces for best results
+  const sanitizedQuestion = prompt.trim().replaceAll('\n', ' ');
+
+  res.writeHead(200, {
+    'Content-Type': 'text/event-stream',
+    'Cache-Control': 'no-cache, no-transform',
+    Connection: 'keep-alive',
+  });
+
+  const sendData = (data: string) => {
+    res.write(`${data}\n\n`);
+  };
+
+  // load store
+  if (!vectorStore) {
+    vectorStore = await HNSWLib.load(storesDir, new XenovaTransformersEmbeddings());
+  }
+  //create chain
+  const chain = makeChain(vectorStore, (token: string) => {
+    // skipping stremaing for now
+    // sendData(JSON.stringify({ data: token }));
+  });
+
+  try {
+    //Ask a question
+    const response = await chain.call({
+      question: sanitizedQuestion,
+      chat_history: messages || [],
+    });
+
+    console.log('response', response);
+    sendData(response.text);
+  } catch (error) {
+    console.log('error', error);
+  } finally {
+    // sendData('[DONE]');
+    res.end();
+  }
+}

src/utils/db-client.js

@@ -0,0 +1,13 @@
+// import { IDocMeta } from '@/types';
+import Dexie, { Table } from 'dexie';
+
+export class MyDocDb extends Dexie {
+  constructor() {
+    super('myDocDb');
+    this.version(1).stores({
+      docs: '++id, folderName, fileName, fileSourceData, model',
+    });
+  }
+}
+
+export const db = new MyDocDb();

src/utils/make-chain.ts

@@ -0,0 +1,57 @@
+import { CallbackManager } from 'langchain/callbacks';
+import { ConversationalRetrievalQAChain } from 'langchain/chains';
+import { OpenAIChat } from 'langchain/llms';
+import { PromptTemplate } from 'langchain/prompts';
+import { BufferMemory } from "langchain/memory";
+import { HNSWLib } from 'langchain/vectorstores/hnswlib';
+
+export const defaultPrompts = {
+  CONDENSE_PROMPT: `Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question.
+
+  Chat History:
+  {chat_history}
+  Follow Up Input: {question}
+  Standalone question:`,
+  QA_PROMPT: `You are an AI assistant providing helpful advice. You are given the following extracted parts of a long document and a question. Provide a conversational answer based on the context provided.
+  You should only provide hyperlinks that reference the context below. Do NOT make up hyperlinks.
+  If you can't find the answer in the context below, just say "Hmm, I'm not sure." Don't try to make up an answer.
+  If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.
+
+  Question: {question}
+  =========
+  {context}
+  =========
+  Answer:`,
+};
+
+const CONDENSE_PROMPT = PromptTemplate.fromTemplate(
+  defaultPrompts.CONDENSE_PROMPT,
+);
+
+const QA_PROMPT = PromptTemplate.fromTemplate(defaultPrompts.QA_PROMPT);
+
+export const makeChain = (
+  vectorstore: HNSWLib,
+  onTokenStream?: (token: string) => void,
+) => {
+
+  const model = new OpenAIChat({
+    temperature: 0.8,
+    modelName: "OpenAIModelID.GPT_3_5",
+    streaming: false,
+    callbackManager: onTokenStream
+      ? CallbackManager.fromHandlers({
+          async handleLLMNewToken(token) {
+            onTokenStream(token);
+          },
+        })
+      : undefined,
+  })
+
+  return ConversationalRetrievalQAChain.fromLLM(
+    model, vectorstore.asRetriever(),
+    { 
+      memory: new BufferMemory({
+        memoryKey: "chat_history", // Must be set to "chat_history"
+      }), qaTemplate: defaultPrompts.QA_PROMPT, questionGeneratorTemplate: defaultPrompts.CONDENSE_PROMPT })
+};

tsconfig.json

@@ -18,6 +18,6 @@
         "@/*": ["./src/*"]
       }
     },
-    "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "src/components/ChatV2.jsx", "src/pages/index.jsx"],
+    "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "src/components/ChatV2.jsx", "src/pages/index.jsx", "src/utils/db-client.js", "src/components/FileLoader.jsx"],
     "exclude": ["node_modules"]
 }