diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..13d4ac58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +.idea/ \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100644 index 00000000..3358bcd4 --- /dev/null +++ b/run.sh @@ -0,0 +1,7 @@ +python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:4, + Text2Image_cuda:1,Image2Canny_cpu,CannyText2Image_cuda:0, + Image2Depth_cpu,DepthText2Image_cuda:5,VisualQuestionAnswering_cuda:2, + InstructPix2Pix_cuda:2,Image2Scribble_cpu,ScribbleText2Image_cuda:2, + Image2Seg_cpu,SegText2Image_cuda:6,Image2Pose_cpu,PoseText2Image_cuda:6, + Image2Hed_cpu,HedText2Image_cuda:3,Image2Normal_cpu, + NormalText2Image_cuda:3,Image2Line_cpu,LineText2Image_cuda:7" \ No newline at end of file diff --git a/visual_chatgpt.py b/visual_chatgpt.py index e9614894..5e59a28d 100644 --- a/visual_chatgpt.py +++ b/visual_chatgpt.py @@ -1,5 +1,8 @@ import os +from dotenv import load_dotenv +load_dotenv() import gradio as gr +gr.close_all() import random import torch import cv2 @@ -44,7 +47,7 @@ ``` Thought: Do I need to use a tool? Yes Action: the action to take, should be one of [{tool_names}] -Action Input: the input to the action +Action Input: the input to the action (in English) Observation: the result of the action ``` @@ -57,7 +60,8 @@ """ VISUAL_CHATGPT_SUFFIX = """You are very strict to the filename correctness and will never fake a file name if it does not exist. -You will remember to provide the image file name loyally if it's provided in the last tool observation. +You will remember to provide the image file name loyally if it's provided in the last tool observation. +When using tools, for all the input except the filename (e.g. object, text, user description, and question), you can only describe them in English even if the Human may ask you in other languages (like Chinese). Begin! @@ -223,7 +227,7 @@ def __init__(self, device): description="useful when you want to remove and object or something from the photo " "from its description or location. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the object need to be removed. ") + "representing the image_path and the object (in English) need to be removed. ") def inference_remove(self, inputs): image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) return self.inference_replace(f"{image_path},{to_be_removed_txt},background") @@ -232,7 +236,7 @@ def inference_remove(self, inputs): description="useful when you want to replace an object from the object description or " "location with another object from its description. " "The input to this tool should be a comma separated string of three, " - "representing the image_path, the object to be replaced, the object to be replaced with ") + "representing the image_path, the object to be replaced (in English), the object to be replaced with (in English) ") def inference_replace(self, inputs): image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",") original_image = Image.open(image_path) @@ -263,7 +267,7 @@ def __init__(self, device): description="useful when you want to the style of the image to be like the text. " "like: make it look like a painting. or make it like a robot. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the text. ") + "representing the image_path and the text (in English). ") def inference(self, inputs): """Change style of image.""" print("===>Starting InstructPix2Pix Inference") @@ -292,7 +296,7 @@ def __init__(self, device): @prompts(name="Generate Image From User Input Text", description="useful when you want to generate an image from a user input text and save it to a file. " "like: generate an image of an object or something, or generate an image that includes some objects. " - "The input to this tool should be a string, representing the text used to generate image. ") + "The input to this tool should be a string, representing the text (in English) used to generate image. ") def inference(self, text): image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png") prompt = text + ', ' + self.a_prompt @@ -368,7 +372,7 @@ def __init__(self, device): " like: generate a real image of a object or something from this canny image," " or generate a new real image of a object or something from this edge image. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description. ") + "representing the image_path and the user description (in English). ") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -426,7 +430,7 @@ def __init__(self, device): "like: generate a real image of a object or something from this straight line image, " "or generate a new real image of a object or something from this straight lines. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description. ") + "representing the image_path and the user description (in English). ") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -484,7 +488,7 @@ def __init__(self, device): "like: generate a real image of a object or something from this soft hed boundary image, " "or generate a new real image of a object or something from this hed boundary. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -540,7 +544,7 @@ def __init__(self, device): description="useful when you want to generate a new real image from both the user description and " "a scribble image or a sketch image. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -598,7 +602,7 @@ def __init__(self, device): "like: generate a real image of a human from this human pose image, " "or generate a new real image of a human from this pose. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -702,7 +706,7 @@ def __init__(self, device): "like: generate a real image of a object or something from this segmentation image, " "or generate a new real image of a object or something from these segmentations. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -761,7 +765,7 @@ def __init__(self, device): "like: generate a real image of a object or something from this depth image, " "or generate a new real image of a object or something from the depth map. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -832,7 +836,7 @@ def __init__(self, device): "like: generate a real image of a object or something from this normal map, " "or generate a new real image of a object or something from the normal map. " "The input to this tool should be a comma separated string of two, " - "representing the image_path and the user description") + "representing the image_path and the user description (in English)") def inference(self, inputs): image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) image = Image.open(image_path) @@ -860,7 +864,7 @@ def __init__(self, device): @prompts(name="Answer Question About The Image", description="useful when you need an answer for a question based on an image. " "like: what is the background color of the last image, how many cats in this figure, what is in this figure. " - "The input to this tool should be a comma separated string of two, representing the image_path and the question") + "The input to this tool should be a comma separated string of two, representing the image_path and the question (in English)") def inference(self, inputs): image_path, question = inputs.split(",")[0], ','.join(inputs.split(',')[1:]) raw_image = Image.open(image_path).convert('RGB') @@ -1066,4 +1070,4 @@ def run_image(self, image, state, txt): clear.click(bot.memory.clear) clear.click(lambda: [], None, chatbot) clear.click(lambda: [], None, state) - demo.launch(server_name="0.0.0.0", server_port=1015) + demo.launch(server_name=os.getenv("HOST_NAME"), server_port=int(os.getenv("HOST_PORT")))