Single Inference

[insafim]

I have written a code for single inference. But I am not getting the expected output.

User Query: Please explain the video in detail. Koala Response: What query are you answering?

Here is the code:

`""" Adapted from: https://github.com/Vision-CAIR/MiniGPT-4/blob/main/demo.py

Koala Single Video Inference """ import argparse import os import random import numpy as np import torch import torch.backends.cudnn as cudnn from koala.common.config import Config from koala.common.dist_utils import get_rank from koala.common.registry import registry from koala.conversation.conversation_video import Chat, Conversation, default_conversation, SeparatorStyle, conv_llava_llama_2 import decord decord.bridge.set_bridge('torch') import copy

Imports modules for registration

from koala.datasets.builders import from koala.models import from koala.processors import from koala.runners import from koala.tasks import *

def parse_args(): parser = argparse.ArgumentParser(description="Koala Inference") parser.add_argument("--cfg-path", default="./eval_configs/conversation_demo.yaml", help="Path to configuration file.") parser.add_argument("--video-path", default="/share/softwares/uzair/datasets/LoViQA/test_video3_converted/excercise_trick.mp4", help="Path to the input video file.") parser.add_argument("--gpu-id", type=int, default=0, help="Specify the GPU to load the model.") parser.add_argument("--model_type", type=str, default='vicuna', help="Specify LLM.")

parser.add_argument('--pretrained_weight_path', type=str, default="./koala_model.pth", help='Path to pretrained weight path')

parser.add_argument('--num_frames_per_clip', type=int, default=16, help='Specify how many frames to use per clip')
parser.add_argument('--num_segments', type=int, default=4, help='Specify number of video segments')
parser.add_argument('--hierarchical_agg_function', type=str, default="without-top-final-global-prompts-region-segment-full-dis-spatiotemporal-prompts-attn-early-attn-linear-learned", help='Specify function to merge global and clip visual representations')
parser.add_argument("--num-beams", type=int, default=1, help="Beam search numbers.")
parser.add_argument("--temperature", type=float, default=1.0, help="Sampling temperature.")

parser.add_argument("--prompt", type=str, default="Please explain the video in detail.", help="User query for video inference.")

parser.add_argument(
    "--options",
    nargs="+",
    help="override some settings in the used config, the key-value pair "
    "in xxx=yyy format will be merged into config file (deprecate), "
    "change to --cfg-options instead.",
)
args = parser.parse_args()
return args

========================================

Model Initialization

========================================

print('Initializing Chat') args = parse_args() cfg = Config(args)

model_config = cfg.model_cfg model_config.device_8bit = args.gpu_id model_cls = registry.get_model_class(model_config.arch) model = model_cls.from_config(model_config).to('cuda:{}'.format(args.gpu_id))

model.num_frames_per_clip = args.num_frames_per_clip # model.num_segments = args.num_segments # model.hierarchical_agg_function = args.hierarchical_agg_function # model.global_region_embed_weight = None #

model.initialize_visual_agg_function() #*

best_checkpoint = torch.load(args.pretrained_weight_path, map_location='cpu')['model_state_dict'] # pretrained_dict = {} # for k, v in best_checkpoint.items(): # pretrained_dict[k.replace('module.', '')] = v #

model_dict = model.state_dict() # model_dict.update(pretrained_dict) # model.load_state_dict(model_dict) # model.cuda().eval() #

vis_processor_cfg = cfg.datasets_cfg.cc_sbu_align.vis_processor.train

vis_processor_cfg = cfg.datasets_cfg.webvid.vis_processor.train vis_processor = registry.get_processor_class(vis_processor_cfg.name).from_config(vis_processor_cfg) chat = Chat(model, vis_processor, device='cuda:{}'.format(args.gpu_id)) print('Initialization Finished')

========================================

Prepare conversation

if args.model_type == 'vicuna': chat_state = default_conversation.copy() else: chat_state = conv_llava_llama_2.copy()

Upload the single video

chat_state.system = "You are able to understand the visual content that the user provides. Follow the instructions carefully and explain your answers in detail." img_list = [] llm_message = chat.upload_video_without_audio(video_path, chat_state, img_list) #VIDEO_PATH

chat_state_seperate = copy.deepcopy(chat_state) user_question = args.prompt #TEXT_INPUT chat.ask(user_question, chat_state_seperate)

llm_message = chat.answer(conv=chat_state_seperate, img_list=img_list, num_beams=args.num_beams, temperature=args.temperature, max_new_tokens=300, max_length=2000)[0]

Display the response

print("User Query:", args.prompt) print("Koala Response:", llm_message) `

Can you please help me to correct this.

[rxtan2]

@insafim I have uploaded the correct version of demo.py now. I realized that it was the older version that I pushed previously. Please let me know if you face any issues.