update code

app.py

@@ -17,11 +17,12 @@ import whisper
 import fire
 import gradio as gr
 import gradio.themes.base as ThemeBase
-from gradio.themes.utils import colors, fonts, sizes
-import os
+from gradio.themes.utils import colors, sizes
+from gradio.components.image_editor import Brush
 import sys
 
 sys.path.append(os.getcwd())
+
 from cllm.agents.builtin import plans
 from cllm.services.general.api import remote_logging
 from cllm.agents import container, FILE_EXT
@@ -199,6 +200,38 @@ class InteractionLoop:
             return self.add_audio(history, audio, role=role, append=append)
         return history, ""
 
+    def add_sketch(self, user_state, history, sketch):
+        if user_state.get("resources", None) is None:
+            user_state["resources"] = OrderedDict()
+
+        if sketch is None or "layers" not in sketch:
+            return user_state, None, history, None
+
+        mask = None
+        for layer in sketch["layers"]:
+            alpha = layer[:, :, 3:] // 255
+            if mask is None:
+                mask = np.ones_like(layer[:, :, :3]) * 255
+            mask = mask * (1 - alpha) + layer[:, :, :3] * alpha
+
+        ext = "png"
+        r_type = "scribble"
+        new_filename = self._gen_new_name(r_type, ext)
+        saved_path = get_real_path(new_filename)
+        if ext in FILE_EXT["image"]:
+            Image.fromarray(mask).save(saved_path, "png")
+            user_state["sketch_image"] = new_filename
+
+        logger.info(f"add file: {saved_path}")
+        user_state["resources"][new_filename] = r_type
+        history, _ = self.add_text(history, (saved_path,), role="human", append=False)
+        history, _ = self.add_text(
+            history, f"Recieved file {new_filename}", role="assistant", append=False
+        )
+        memory = convert_dict_to_frame(user_state["resources"])
+
+        return user_state, history, memory
+
     def add_text(self, history, text, role="assistant", append=False):
         if history is None:
             return history, ""
@@ -326,10 +359,7 @@ class InteractionLoop:
             for result_per_step, executed_solutions, wrapped_outputs in results:
                 tool_name = json.dumps(result_per_step[0], ensure_ascii=False)
                 args = json.dumps(result_per_step[1], ensure_ascii=False)
-                if isinstance(result_per_step[2], Exception):
-                    ret = f"Internal error: {result_per_step[2]}"
-                else:
-                    ret = json.dumps(result_per_step[2], ensure_ascii=False)
+                ret = json.dumps(result_per_step[2], ensure_ascii=False)
                 history, _ = self.add_text(
                     history,
                     f"Call **{tool_name}:**<br>&nbsp;&nbsp;&nbsp;&nbsp;**Args**: {plain2md(args)}<br>&nbsp;&nbsp;&nbsp;&nbsp;**Ret**: {plain2md(ret)}",
@@ -552,37 +582,43 @@ def app(controller="cllm.agents.tog.Controller", https=False, **kwargs):
 
             with gr.Column(scale=5):
                 with gr.Tabs():
-                    with gr.Tab("Mask Input"):
-                        image_mask = gr.components.Image(
-                            sources="upload",
+                    with gr.Tab("Click"):
+                        click_image = gr.Image(
+                            sources=["upload", "clipboard"],
                             interactive=True,
                             type="filepath",
                         )
-                        # with gr.Row():
-                        #     mask_submit_btn = gr.Button("Segment", variant="primary")
                         with gr.Row():
-                            image_submit_btn = gr.Button("Upload", variant="primary")
+                            click_image_submit_btn = gr.Button(
+                                "Upload", variant="primary"
+                            )
+                        gr.Examples(
+                            [
+                                osp.join("./assets/resources", item)
+                                for item in builtin_resources.keys()
+                                if item.endswith(".png")
+                            ],
+                            inputs=[click_image],
+                            label="File Examples",
+                        )
+
+                    with gr.Tab("Draw"):
+                        sketch = gr.Sketchpad(
+                            sources=(), brush=Brush(colors=["#000000"])
+                        )
+                        with gr.Row():
+                            sketch_submit_btn = gr.Button("Upload", variant="primary")
 
                     with gr.Tab("Plan"):
                         planbot = gr.JSON(elem_classes="json")
 
                     with gr.Tab("Memory"):
                         memory_table = gr.DataFrame(
-                            # value=convert_dict_to_frame(builtin_resources),
                             label="Memory",
                             headers=["Resource", "Type"],
                             row_count=5,
                             wrap=True,
                         )
-                gr.Examples(
-                    [
-                        osp.join("./assets/resources", item)
-                        for item in builtin_resources.keys()
-                        if item.endswith(".png")
-                    ],
-                    inputs=[image_mask],
-                    label="File Examples",
-                )
 
         chatbot.like(
             loop.vote,
@@ -595,14 +631,8 @@ def app(controller="cllm.agents.tog.Controller", https=False, **kwargs):
                 chatbot,
             ],
         )
-        reply_inputs = [user_state, image_mask, chatbot, planbot]
-        reply_outputs = [
-            user_state,
-            # image_mask,
-            chatbot,
-            memory_table,
-            # planbot,
-        ]
+        reply_inputs = [user_state, click_image, chatbot, planbot]
+        reply_outputs = [user_state, chatbot, memory_table]
 
         add_text = [
             partial(loop.add_text, role="human"),
@@ -633,7 +663,7 @@ def app(controller="cllm.agents.tog.Controller", https=False, **kwargs):
         upload_btn.upload(
             loop.add_file,
             inputs=[user_state, chatbot, upload_btn],
-            outputs=[user_state, image_mask, chatbot, memory_table],
+            outputs=[user_state, click_image, chatbot, memory_table],
         )
         record.click(
             on_switch_input,
@@ -641,7 +671,7 @@ def app(controller="cllm.agents.tog.Controller", https=False, **kwargs):
             [state_input, text, audio],
         )
 
-        image_mask.select(
+        click_image.select(
             loop.save_point, [user_state, chatbot], [user_state, chatbot]
         ).then(loop.plan, reply_inputs, reply_inputs).then(
             loop.execute, reply_inputs, reply_inputs
@@ -649,22 +679,28 @@ def app(controller="cllm.agents.tog.Controller", https=False, **kwargs):
             loop.reply, [user_state, chatbot], reply_outputs
         )
 
-        image_mask.upload(
+        click_image.upload(
             loop.add_file,
-            inputs=[user_state, chatbot, image_mask],
-            outputs=[user_state, image_mask, chatbot, memory_table],
+            inputs=[user_state, chatbot, click_image],
+            outputs=[user_state, click_image, chatbot, memory_table],
         )
-        image_submit_btn.click(
+        click_image_submit_btn.click(
             loop.add_file,
-            inputs=[user_state, chatbot, image_mask],
-            outputs=[user_state, image_mask, chatbot, memory_table],
+            inputs=[user_state, chatbot, click_image],
+            outputs=[user_state, click_image, chatbot, memory_table],
+        )
+
+        sketch_submit_btn.click(
+            loop.add_sketch,
+            inputs=[user_state, chatbot, sketch],
+            outputs=[user_state, chatbot, memory_table],
         )
 
     if https:
         demo.queue().launch(
             server_name="0.0.0.0",
-            # ssl_certfile="./certificate/cert.pem",
-            # ssl_keyfile="./certificate/key.pem",
+            ssl_certfile="./certificate/cert.pem",
+            ssl_keyfile="./certificate/key.pem",
             ssl_verify=False,
             show_api=False,
             allowed_paths=[

builtin_plan.json

@@ -1,15 +1 @@
-{
-    "you know what I want": [
-        [
-            {
-                "tool_name": "text_to_image",
-                "inputs": {
-                    "text": "a dog"
-                },
-                "outputs": [
-                    "image"
-                ]
-            }
-        ]
-    ]
-}
+{}

cllm/agents/tog/interpretor.py

@@ -94,13 +94,14 @@ class Interpretor:
                 zip(action.outputs, tool_outputs)
             ):
                 memory[arg_name] = arg_value
-                if arg_value is None:
+                if arg_value is None or isinstance(arg_value, Exception):
                     outputs.append(arg_value)
+                    arg_value = f"{arg_value}"
                     wrapped_outputs.append(
                         auto_type(
                             arg_name,
                             DataType.TEXT,
-                            None,
+                            arg_value,
                         )
                     )
                     continue

cllm/agents/tog/responser.py

@@ -56,7 +56,7 @@ def generate_response(user_input, solution, output_files):
     logger.info("##### Response Generation #####")
     logger.info(content)
 
-    chat = ChatOpenAI(model_name="gpt-3.5-turbo-1106")
+    chat = ChatOpenAI(model_name="gpt-3.5-turbo-16k")
     messages = [SystemMessage(content=content)]
     output = chat(messages)
     logger.info(output)

cllm/services/nlp/api.py

@@ -34,7 +34,7 @@ __ALL__ = [
 ]
 
 
-HOST = os.environ.get("CLLM_SERVICES_HOST", "localhost")
+HOST = "localhost"
 PORT = os.environ.get("CLLM_SERVICES_PORT", 10056)
 
 
@@ -63,7 +63,7 @@ def question_answering_with_context(context: str, question: str, **kwargs):
 
 
 def openai_chat_model(input_msg: str, **kwargs):
-    chat = ChatOpenAI()
+    chat = ChatOpenAI(model_name="gpt-3.5-turbo-16k")
     chat_log = []
     default_sys_msg = "Your name is ControlLLM, an AI-powered assistant developed by OpenGVLab from Shanghai AI Lab. You need to respond to user requests based on the following information."
     sys_msg = kwargs.get("sys_msg", default_sys_msg)

cllm/services/nlp/llms/chat_models.py

@@ -1,4 +1,5 @@
 import os
+from turtle import mode
 import openai
 import requests
 from typing import (
@@ -39,6 +40,8 @@ class ChatOpenAI:
         self.model_kwargs = model_kwargs
         self.api_key = os.environ.get("OPENAI_API_KEY", openai_api_key)
         self.base_url = os.environ.get("OPENAI_BASE_URL", openai_base_url)
+        # openai.api_key = self.api_key
+        # openai.base_url = self.base_url
 
     def __call__(self, messages: List[BaseMessage], **kwargs):
         stream = kwargs.get("stream", False)
@@ -210,7 +213,7 @@ class ChatLLAMA2(SimpleChatModel):
 
 
 if __name__ == "__main__":
-    chat = ChatOpenAI()
+    chat = ChatOpenAI(model_name="gpt-3.5-turbo-1106")
     msg = [
         SystemMessage(content="You are a helpful assistant."),
         HumanMessage(content="Hello!"),