Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Other Languages like Chinese #299

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Support Other Languages like Chinese #299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7d090bf
add gitignore
Mar 29, 2023
ec00912
support Chinese
Mar 29, 2023
2f51440
close gr
Mar 29, 2023
60e537f
debug
Mar 29, 2023
b484de8
debug
Mar 29, 2023
03c103f
update prompt
Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
.env
.idea/
7 changes: 7 additions & 0 deletions run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
python visual_chatgpt.py --load "ImageCaptioning_cuda:0,ImageEditing_cuda:4,
Text2Image_cuda:1,Image2Canny_cpu,CannyText2Image_cuda:0,
Image2Depth_cpu,DepthText2Image_cuda:5,VisualQuestionAnswering_cuda:2,
InstructPix2Pix_cuda:2,Image2Scribble_cpu,ScribbleText2Image_cuda:2,
Image2Seg_cpu,SegText2Image_cuda:6,Image2Pose_cpu,PoseText2Image_cuda:6,
Image2Hed_cpu,HedText2Image_cuda:3,Image2Normal_cpu,
NormalText2Image_cuda:3,Image2Line_cpu,LineText2Image_cuda:7"
36 changes: 20 additions & 16 deletions visual_chatgpt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import os
from dotenv import load_dotenv
load_dotenv()
import gradio as gr
gr.close_all()
import random
import torch
import cv2
Expand Down Expand Up @@ -44,7 +47,7 @@
```
Thought: Do I need to use a tool? Yes
Action: the action to take, should be one of [{tool_names}]
Action Input: the input to the action
Action Input: the input to the action (in English)
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the key point

Observation: the result of the action
```

Expand All @@ -57,7 +60,8 @@
"""

VISUAL_CHATGPT_SUFFIX = """You are very strict to the filename correctness and will never fake a file name if it does not exist.
You will remember to provide the image file name loyally if it's provided in the last tool observation.
You will remember to provide the image file name loyally if it's provided in the last tool observation.
When using tools, for all the input except the filename (e.g. object, text, user description, and question), you can only describe them in English even if the Human may ask you in other languages (like Chinese).

Begin!

Expand Down Expand Up @@ -223,7 +227,7 @@ def __init__(self, device):
description="useful when you want to remove and object or something from the photo "
"from its description or location. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the object need to be removed. ")
"representing the image_path and the object (in English) need to be removed. ")
def inference_remove(self, inputs):
image_path, to_be_removed_txt = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
return self.inference_replace(f"{image_path},{to_be_removed_txt},background")
Expand All @@ -232,7 +236,7 @@ def inference_remove(self, inputs):
description="useful when you want to replace an object from the object description or "
"location with another object from its description. "
"The input to this tool should be a comma separated string of three, "
"representing the image_path, the object to be replaced, the object to be replaced with ")
"representing the image_path, the object to be replaced (in English), the object to be replaced with (in English) ")
def inference_replace(self, inputs):
image_path, to_be_replaced_txt, replace_with_txt = inputs.split(",")
original_image = Image.open(image_path)
Expand Down Expand Up @@ -263,7 +267,7 @@ def __init__(self, device):
description="useful when you want to the style of the image to be like the text. "
"like: make it look like a painting. or make it like a robot. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the text. ")
"representing the image_path and the text (in English). ")
def inference(self, inputs):
"""Change style of image."""
print("===>Starting InstructPix2Pix Inference")
Expand Down Expand Up @@ -292,7 +296,7 @@ def __init__(self, device):
@prompts(name="Generate Image From User Input Text",
description="useful when you want to generate an image from a user input text and save it to a file. "
"like: generate an image of an object or something, or generate an image that includes some objects. "
"The input to this tool should be a string, representing the text used to generate image. ")
"The input to this tool should be a string, representing the text (in English) used to generate image. ")
def inference(self, text):
image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
prompt = text + ', ' + self.a_prompt
Expand Down Expand Up @@ -368,7 +372,7 @@ def __init__(self, device):
" like: generate a real image of a object or something from this canny image,"
" or generate a new real image of a object or something from this edge image. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description. ")
"representing the image_path and the user description (in English). ")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -426,7 +430,7 @@ def __init__(self, device):
"like: generate a real image of a object or something from this straight line image, "
"or generate a new real image of a object or something from this straight lines. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description. ")
"representing the image_path and the user description (in English). ")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -484,7 +488,7 @@ def __init__(self, device):
"like: generate a real image of a object or something from this soft hed boundary image, "
"or generate a new real image of a object or something from this hed boundary. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -540,7 +544,7 @@ def __init__(self, device):
description="useful when you want to generate a new real image from both the user description and "
"a scribble image or a sketch image. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -598,7 +602,7 @@ def __init__(self, device):
"like: generate a real image of a human from this human pose image, "
"or generate a new real image of a human from this pose. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -702,7 +706,7 @@ def __init__(self, device):
"like: generate a real image of a object or something from this segmentation image, "
"or generate a new real image of a object or something from these segmentations. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -761,7 +765,7 @@ def __init__(self, device):
"like: generate a real image of a object or something from this depth image, "
"or generate a new real image of a object or something from the depth map. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -832,7 +836,7 @@ def __init__(self, device):
"like: generate a real image of a object or something from this normal map, "
"or generate a new real image of a object or something from the normal map. "
"The input to this tool should be a comma separated string of two, "
"representing the image_path and the user description")
"representing the image_path and the user description (in English)")
def inference(self, inputs):
image_path, instruct_text = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
image = Image.open(image_path)
Expand Down Expand Up @@ -860,7 +864,7 @@ def __init__(self, device):
@prompts(name="Answer Question About The Image",
description="useful when you need an answer for a question based on an image. "
"like: what is the background color of the last image, how many cats in this figure, what is in this figure. "
"The input to this tool should be a comma separated string of two, representing the image_path and the question")
"The input to this tool should be a comma separated string of two, representing the image_path and the question (in English)")
def inference(self, inputs):
image_path, question = inputs.split(",")[0], ','.join(inputs.split(',')[1:])
raw_image = Image.open(image_path).convert('RGB')
Expand Down Expand Up @@ -1066,4 +1070,4 @@ def run_image(self, image, state, txt):
clear.click(bot.memory.clear)
clear.click(lambda: [], None, chatbot)
clear.click(lambda: [], None, state)
demo.launch(server_name="0.0.0.0", server_port=1015)
demo.launch(server_name=os.getenv("HOST_NAME"), server_port=int(os.getenv("HOST_PORT")))