update to tokenizers 0.12.1 and use 'debug' file for small model

.gitignore

@@ -1,3 +1,3 @@
-use_normal_tokenizers
+debug
 __pycache__
 incoder-6B

modules/app.py

@@ -8,19 +8,19 @@ import logging
 logging.basicConfig(level=logging.INFO)
 import modules.cloud_logging
 
+import tokenizers
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import json
+import pprint
+
 # needs to be imported *before* transformers
-if os.path.exists('use_normal_tokenizers'):
-    import tokenizers
+if os.path.exists('debug'):
     BIG_MODEL = False
     CUDA = False
 else:
-    import tokenizers_patch
     BIG_MODEL = True
     CUDA = True
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import json
-import pprint
 
 # from flask import Flask, request, render_template
 # from flask_cors import CORS

requirements.txt

@@ -2,6 +2,6 @@ fastapi==0.74.*
 requests==2.27.*
 torch==1.11.*
 uvicorn[standard]==0.17.*
-https://dl.fbaipublicfiles.com/codemodels/tokenizers-0.11.6-cp38-cp38-linux_x86_64.whl
+tokenizers==0.12.1
 git+https://github.com/huggingface/transformers.git@b18dfd95e1f60ae65a959a7b255fc06522170d1b
 google-cloud-logging