飞桨AI Studio可以玩多模态了?MiniGPT4实战演练!
MiniGPT4是基于GPT3的改进版本,它的参数量比GPT3少了一个数量级,但是在多项自然语言处理任务上的表现却不逊于GPT3。项目作者以MiniGPT4-7B作为实战演练项目。
创作者:衍哲
一键fork
fork该项目并运行,运行环境建议至少选择A100(40G)及以上配置
安装相关模块
1
2os
3os
4os
5
import
os2os
.system(
"pip install --pre --upgrade paddlenlp -f https://www.paddlepaddle.org.cn/whl/paddlenlp.html") # 安装nlp分支最新包
3os
.system(
"pip install paddlepaddle-gpu==0.0.0.post112 -f https://www.paddlepaddle.org.cn/whl/linux/gpu/develop.html")
4os
.system(
"pip install tqdm")
5
!pip install ipywidgets
引用相关模块
1
2
3
4import
5from
6import
7from
8import
9from
10import
11from
12from
%%capture
2
os.environ[
"CUDA_VISIBLE_DEVICES"] =
"0"3
os.environ[
"FLAGS_use_cuda_managed_memory"] =
"true"4import
requests
5from
PIL
import Image
6import
gradio
as gr
7from
tqdm
import tqdm
8import
ipywidgets
as widgets
9from
IPython.display
import display
10import
csv
11from
itertools
import islice
12from
paddlenlp.transformers
import MiniGPT4ForConditionalGeneration, MiniGPT4Processor
下载miniGPT4权重或配置文件
1!mkdir minigpt4
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
%%capture
2
os.system(
"wget -O minigpt4/model_config.json https://bj.bcebos.com/v1/ai-studio-online/924ed883c17b4b8b88b4a1f98e24d34b3b00160ac9bd4b3ba478aff6974e0e9d?responseContentDisposition=attachment%3B%20filename%3Dmodel_config.json ")
3
!wget -O ./minigpt4/model_state.pdparams https:
//bj.bcebos.com/v1/ai-studio-online/
18bd53eaa2854263ba31fb4d75f31a5f0d38421a6da64525bff6da230389fc36?responseContentDisposition=attachment%3B%20filename%3Dmodel_state.pdparams
4
!wget -O ./minigpt4/generation_config.json https:
//bj.bcebos.com/v1/ai-studio-online/f0b2129d6a934a97abcaa139ac1f28e33a6940004c7a4c859737f282640cf332?responseContentDisposition=attachment%3B%20filename%3Dgeneration_config.json
5
!wget -O ./minigpt4/preprocessor_config.json https:
//bj.bcebos.com/v1/ai-studio-online/
748c332837d34f389d762f487470b1a7221edd36ccb5484b913bd2d3855ee9f6?responseContentDisposition=attachment%3B%20filename%3Dpreprocessor_config.json
6
!wget -O ./minigpt4/sentencepiece.bpe.model https:
//bj.bcebos.com/v1/ai-studio-online/
0139a1bfcdf84058b77cea4631837340ea94f5fcc37445929a3414f05d07579b?responseContentDisposition=attachment%3B%20filename%3Dsentencepiece.bpe.model
7
!wget -O ./minigpt4/special_tokens_map.json https:
//bj.bcebos.com/v1/ai-studio-online/
90b16a96d4f94200ab417b39dcf3bce4ddef5885625c4d0c8e70b3f659cb6993?responseContentDisposition=attachment%3B%20filename%3Dspecial_tokens_map.json
8
!wget -O ./minigpt4/tokenizer.json https:
//bj.bcebos.com/v1/ai-studio-online/e877a685eb86499cb87e1c4cbf85353856506d12e9a841a292e780aa4a9e188a?responseContentDisposition=attachment%3B%20filename%3Dtokenizer.json
9
!wget -O ./minigpt4/tokenizer_config.json https:
//bj.bcebos.com/v1/ai-studio-online/f93064db167c4075b1f86d6878cac9303fb8df418f7a42a7900785a6e188cc44?responseContentDisposition=attachment%3B%20filename%3Dtokenizer_config.json
10
--
2023-
07-
2710:
54:
29-- https:
//bj.bcebos.com/v1/ai-studio-online/
924ed883c17b4b8b88b4a1f98e24d34b3b00160ac9bd4b3ba478aff6974e0e9d?responseContentDisposition=attachment%3B%20filename%3Dmodel_config.json
11
Resolving bj.bcebos.com (bj.bcebos.com)...
182.61.
200.195,
182.61.
200.229,
2409:
8c04:
1001:
1002:
0:ff:b001:
368a
12
Connecting to bj.bcebos.com (bj.bcebos.com)|
182.61.
200.195|:
443... connected.
13
HTTP request sent, awaiting response...
200 OK
14
Length:
5628 (
5.5K) [application/octet-stream]
15
Saving to:
'minigpt4/model_config.json'实例化miniGPT4模型和处理器
1
2
3
4
model_path ='./minigpt4'
2
model = MiniGPT4ForConditionalGeneration.from_pretrained(model_path)
3
model.eval()
4
processor = MiniGPT4Processor.from_pretrained(model_path)
模型推理
输入图像url+prompt(单张图片+单轮对话)
另有本地上传图像形式,请进入项目查看
1defpredict_per_url_prompt(url=None,text=None):
2if
3
4
5if
6
7
8
9
10
11
12
13"max_length"
14"num_beams"
15"top_p"
16"repetition_penalty"
17"length_penalty"
18"temperature"
19"decode_strategy"
20"eos_token_id"
21
22
23
24return
2if
url==
None:
3
url =
"https://paddlenlp.bj.bcebos.com/data/images/mugs.png"4
image = Image.open(requests.get(url, stream=
True).raw)
5if
text==
None:
6
text =
"describe this image"7
8
prompt =
"Give the following image: <Img>ImageContent</Img>. You will be able to see the image once I provide it to you. Please answer my questions.###Human: <Img><ImageHere></Img> <TextHere>###Assistant:"9
10
inputs = processor([image], text, prompt)
11
12
generate_kwargs = {
13"max_length"
:
300,
14"num_beams"
:
1,
15"top_p"
:
1.0,
16"repetition_penalty"
:
1.0,
17"length_penalty"
:
0,
18"temperature"
:
1,
19"decode_strategy"
:
"greedy_search",
20"eos_token_id"
: [[
835], [
2277,
29937]],
21
}
22
outputs = model.generate(**inputs, **generate_kwargs)
23
msg = processor.batch_decode(outputs[
0])
24return
msg[
0][
0:
-5]
将图像上传到本地后的file_path+prompt(多张图片+单轮对话)
def predict_dir_and_one_prompt_out_list(dir_path=None,text=None):
2 import
os 3assertos.
path.isdir(dir_path),
print(
'请输入文件夹路径,而不是图像路径')
4output = []
5for per_image_name
in tqdm (
os.listdir(dir_path)):
6 image = Image.
open(
os.
path.join(dir_path,per_image_name))
7if text== None:
8 text =
"describe this image" 9else:
10 text = text
1112 prompt =
"Give the following image: <Img>ImageContent</Img>. You will be able to see the image once I provide it to you. Please answer my questions.###Human: <Img><ImageHere></Img> <TextHere>###Assistant:"1314 inputs = processor([image], text, prompt)
1516 generate_kwargs = {
17"max_length":
300,
18"num_beams":
1,
19"top_p":
1.0,
20"repetition_penalty":
1.0,
21"length_penalty":
0,
22"temperature":
1,
23"decode_strategy":
"greedy_search",
24"eos_token_id":
[[835], [2277, 29937]],
25 }
26 outputs = model.generate(**inputs, **generate_kwargs)
27 msg = processor.batch_decode(outputs[
0])
28output.append(msg[
0][
0:
-5])
29returnoutput输出:这张图片显示了一个女性角色,穿着红色和白色的服装,手持一根金色的剑。她的头发是白色的,眼睛是红色的。她站在一张草地上,手持剑的柄子。这个角色看起来像是一个英雄,她的服装和装备显示出她的力量和勇气
predict_per_url_prompt(url=
'https://ai-studio-static-online.cdn.bcebos.com/d283b05404bd44b69b9be868fddb67616296858284bf4ad587e29432de66e930',text=
"描述这张图片,使用中文")
2'这张图片显示了一个女性角色,穿着红色和白色的服装,手持一根金色的剑。她的头发是白色的,眼睛是红色的。她站在一张草地上,手持剑的柄子。这个角色看起来像是一个英雄,她的服装和装备显示出她的力量和勇气'更多玩法,可一键fork该项目进行模型微调。
点击底部阅读原文即可立即体验更多大模型应用。
关注【飞桨PaddlePaddle】公众号
获取更多技术内容~
阅读原文
最新评论
推荐文章
作者最新文章
你可能感兴趣的文章
Copyright Disclaimer: The copyright of contents (including texts, images, videos and audios) posted above belong to the User who shared or the third-party website which the User shared from. If you found your copyright have been infringed, please send a DMCA takedown notice to [email protected]. For more detail of the source, please click on the button "Read Original Post" below. For other communications, please send to [email protected].
版权声明:以上内容为用户推荐收藏至CareerEngine平台,其内容(含文字、图片、视频、音频等)及知识版权均属用户或用户转发自的第三方网站,如涉嫌侵权,请通知[email protected]进行信息删除。如需查看信息来源,请点击“查看原文”。如需洽谈其它事宜,请联系[email protected]。
版权声明:以上内容为用户推荐收藏至CareerEngine平台,其内容(含文字、图片、视频、音频等)及知识版权均属用户或用户转发自的第三方网站,如涉嫌侵权,请通知[email protected]进行信息删除。如需查看信息来源,请点击“查看原文”。如需洽谈其它事宜,请联系[email protected]。