Demo does not work.

[MykhailoPoleshchukLitslink]

Hello. Demo returns the following error message when I try to load the model. Also other model from model zoo can't be imported and throw different errors model, vis_processors, _ = load_model_and_preprocess( name="blip2_vicuna_instruct_malmm", model_type="vicuna7b", is_eval=True, device=device )

---

HTTPError Traceback (most recent call last) File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/utils/_errors.py:304, in hf_raise_for_status(response, endpoint_name) 303 try: --> 304 response.raise_for_status() 305 except HTTPError as e:

File ~/test_ma/lib/python3.10/site-packages/requests/models.py:1021, in Response.raise_for_status(self) 1020 if http_error_msg: -> 1021 raise HTTPError(http_error_msg, response=self)

HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/llm/vicuna-7b/resolve/main/tokenizer_config.json

The above exception was the direct cause of the following exception:

RepositoryNotFoundError Traceback (most recent call last) File ~/test_ma/lib/python3.10/site-packages/transformers/utils/hub.py:398, in cached_file(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs) 396 try: 397 # Load from URL or cache if already cached --> 398 resolved_file = hf_hub_download( 399 path_or_repo_id, 400 filename, 401 subfolder=None if len(subfolder) == 0 else subfolder, 402 repo_type=repo_type, 403 revision=revision, 404 cache_dir=cache_dir, 405 user_agent=user_agent, 406 force_download=force_download, 407 proxies=proxies, 408 resume_download=resume_download, 409 token=token, 410 local_files_only=local_files_only, 411 ) 412 except GatedRepoError as e:

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:119, in validate_hf_hub_args.._inner_fn(*args, *kwargs) 117 kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.name, has_token=has_token, kwargs=kwargs) --> 119 return fn(args, **kwargs)

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/file_download.py:1403, in hf_hub_download(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, local_dir_use_symlinks, user_agent, force_download, force_filename, proxies, etag_timeout, resume_download, token, local_files_only, headers, legacy_cache_layout, endpoint) 1401 elif isinstance(head_call_error, RepositoryNotFoundError) or isinstance(head_call_error, GatedRepoError): 1402 # Repo not found or gated => let's raise the actual error -> 1403 raise head_call_error 1404 else: 1405 # Otherwise: most likely a connection issue or Hub downtime => let's warn the user

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/file_download.py:1261, in hf_hub_download(repo_id, filename, subfolder, repo_type, revision, library_name, library_version, cache_dir, local_dir, local_dir_use_symlinks, user_agent, force_download, force_filename, proxies, etag_timeout, resume_download, token, local_files_only, headers, legacy_cache_layout, endpoint) 1260 try: -> 1261 metadata = get_hf_file_metadata( 1262 url=url, 1263 token=token, 1264 proxies=proxies, 1265 timeout=etag_timeout, 1266 library_name=library_name, 1267 library_version=library_version, 1268 user_agent=user_agent, 1269 ) 1270 except EntryNotFoundError as http_error: 1271 # Cache the non-existence of the file and raise

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:119, in validate_hf_hub_args.._inner_fn(*args, *kwargs) 117 kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.name, has_token=has_token, kwargs=kwargs) --> 119 return fn(args, **kwargs)

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/file_download.py:1674, in get_hf_file_metadata(url, token, proxies, timeout, library_name, library_version, user_agent, headers) 1673 # Retrieve metadata -> 1674 r = _request_wrapper( 1675 method="HEAD", 1676 url=url, 1677 headers=headers, 1678 allow_redirects=False, 1679 follow_relative_redirects=True, 1680 proxies=proxies, 1681 timeout=timeout, 1682 ) 1683 hf_raise_for_status(r)

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/file_download.py:369, in _request_wrapper(method, url, follow_relative_redirects, params) 368 if follow_relative_redirects: --> 369 response = _request_wrapper( 370 method=method, 371 url=url, 372 follow_relative_redirects=False, 373 params, 374 ) 376 # If redirection, we redirect only relative paths. 377 # This is useful in case of a renamed repository.

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/file_download.py:393, in _request_wrapper(method, url, follow_relative_redirects, params) 392 response = get_session().request(method=method, url=url, params) --> 393 hf_raise_for_status(response) 394 return response

File ~/test_ma/lib/python3.10/site-packages/huggingface_hub/utils/_errors.py:352, in hf_raise_for_status(response, endpoint_name) 344 message = ( 345 f"{response.status_code} Client Error." 346 + "\n\n" (...) 350 " make sure you are authenticated." 351 ) --> 352 raise RepositoryNotFoundError(message, response) from e 354 elif response.status_code == 400:

RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-6630b645-1ec06be83da8eeef3f5100f8;89556c43-061d-4725-b3d8-2f9b21850e2d)

Repository Not Found for url: https://huggingface.co/llm/vicuna-7b/resolve/main/tokenizer_config.json. Please make sure you specified the correct repo_id and repo_type. If you are trying to access a private or gated repo, make sure you are authenticated. Invalid username or password.

The above exception was the direct cause of the following exception:

OSError Traceback (most recent call last) Cell In[8], line 2 1 # we associate a model with its preprocessors to make it easier for inference. ----> 2 model, visprocessors, = load_model_and_preprocess( 3 name="blip2_vicuna_instruct_malmm", model_type="vicuna7b", is_eval=True, device=device 4 )

File ~/MA-LMM/lavis/models/init.py:198, in load_model_and_preprocess(name, model_type, is_eval, device) 195 model_cls = registry.get_model_class(name) 197 # load model --> 198 model = model_cls.from_pretrained(model_type=model_type) 200 if is_eval: 201 model.eval()

File ~/MA-LMM/lavis/models/base_model.py:69, in BaseModel.from_pretrained(cls, model_type) 59 """ 60 Build a pretrained model from default configuration file, specified by model_type. 61 (...) 66 - model (nn.Module): pretrained or finetuned model, depending on the configuration. 67 """ 68 model_cfg = OmegaConf.load(cls.default_config_path(model_type)).model ---> 69 model = cls.from_config(model_cfg) 71 return model

File ~/MA-LMM/lavis/models/blip2_models/blip2_vicuna_instruct.py:797, in Blip2VicunaInstruct_MALMM.from_config(cls, cfg) 793 apply_lemmatizer = cfg.get("apply_lemmatizer", False) 795 qformer_text_input = cfg.get("qformer_text_input", True) --> 797 model = cls( 798 vit_model=vit_model, 799 img_size=img_size, 800 drop_path_rate=drop_path_rate, 801 use_grad_checkpoint=use_grad_checkpoint, 802 vit_precision=vit_precision, 803 freeze_vit=freeze_vit, 804 num_query_token=num_query_token, 805 llm_model=llm_model, 806 prompt=prompt, 807 max_txt_len=max_txt_len, 808 max_output_txt_len=max_output_txt_len, 809 apply_lemmatizer=apply_lemmatizer, 810 qformer_text_input=qformer_text_input, 811 memory_bank_length=memory_bank_length, 812 num_frames=num_frames, 813 ) 815 # if qformer_text_input: 816 # # Hard-coded to load from BLIP-2 stage-1 pre-trained model (not ideal) 817 # model.load_from_pretrained( 818 # url_or_filename="https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/blip2_pretrained.pth" 819 # ) 821 model.load_checkpoint_from_config(cfg)

File ~/MA-LMM/lavis/models/blip2_models/blip2_vicuna_instruct.py:75, in Blip2VicunaInstruct_MALMM.init(self, vit_model, img_size, drop_path_rate, use_grad_checkpoint, vit_precision, freeze_vit, num_query_token, llm_model, prompt, max_txt_len, max_output_txt_len, apply_lemmatizer, qformer_text_input, memory_bank_length, num_frames) 72 self.Qformer.resize_token_embeddings(len(self.tokenizer)) 73 self.Qformer.cls = None ---> 75 self.llm_tokenizer = LlamaTokenizer.from_pretrained(llm_model, use_fast=False, truncation_side="left") 76 self.llm_model = LlamaForCausalLM.from_pretrained( 77 llm_model, torch_dtype=torch.float16 78 ) 79 self.llm_tokenizer.add_special_tokens({'pad_token': '[PAD]'})

File ~/test_ma/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:2010, in PreTrainedTokenizerBase.from_pretrained(cls, pretrained_model_name_or_path, cache_dir, force_download, local_files_only, token, revision, trust_remote_code, *init_inputs, **kwargs) 2007 if "tokenizer_file" in vocab_files: 2008 # Try to get the tokenizer config to see if there are versioned tokenizer files. 2009 fast_tokenizer_file = FULL_TOKENIZER_FILE -> 2010 resolved_config_file = cached_file( 2011 pretrained_model_name_or_path, 2012 TOKENIZER_CONFIG_FILE, 2013 cache_dir=cache_dir, 2014 force_download=force_download, 2015 resume_download=resume_download, 2016 proxies=proxies, 2017 token=token, 2018 revision=revision, 2019 local_files_only=local_files_only, 2020 subfolder=subfolder, 2021 user_agent=user_agent, 2022 _raise_exceptions_for_gated_repo=False, 2023 _raise_exceptions_for_missing_entries=False, 2024 _raise_exceptions_for_connection_errors=False, 2025 _commit_hash=commit_hash, 2026 ) 2027 commit_hash = extract_commit_hash(resolved_config_file, commit_hash) 2028 if resolved_config_file is not None:

File ~/test_ma/lib/python3.10/site-packages/transformers/utils/hub.py:421, in cached_file(path_or_repo_id, filename, cache_dir, force_download, resume_download, proxies, token, revision, local_files_only, subfolder, repo_type, user_agent, _raise_exceptions_for_gated_repo, _raise_exceptions_for_missing_entries, _raise_exceptions_for_connection_errors, _commit_hash, **deprecated_kwargs) 416 raise EnvironmentError( 417 "You are trying to access a gated repo.\nMake sure to have access to it at " 418 f"https://huggingface.co/{path_or_repo_id}.\n{str(e)}" 419 ) from e 420 except RepositoryNotFoundError as e: --> 421 raise EnvironmentError( 422 f"{path_or_repo_id} is not a local folder and is not a valid model identifier " 423 "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a token " 424 "having permission to this repo either by logging in with huggingface-cli login or by passing " 425 "token=<your_token>" 426 ) from e 427 except RevisionNotFoundError as e: 428 raise EnvironmentError( 429 f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists " 430 "for this model name. Check the model page at " 431 f"'https://huggingface.co/{path_or_repo_id}' for available revisions." 432 ) from e

OSError: llm/vicuna-7b is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' If this is a private repository, make sure to pass a token having permission to this repo either by logging in with huggingface-cli login or by passing token=<your_token>

[MykhailoPoleshchukLitslink]

Also with model, vis_processors, _ = load_model_and_preprocess( name="blip_classification", model_type="base", is_eval=True, device=device ) I get

AssertionError Traceback (most recent call last) Cell In[9], line 4 1 # model, visprocessors, = load_model_and_preprocess( 2 # name="blip_classification", model_type="base", is_eval=True, device=device 3 # ) ----> 4 model, visprocessors, = load_model_and_preprocess( 5 name="blip_classification", model_type="base", is_eval=True, device=device 6 )

File ~/MA-LMM/lavis/models/init.py:198, in load_model_and_preprocess(name, model_type, is_eval, device) 195 model_cls = registry.get_model_class(name) 197 # load model --> 198 model = model_cls.from_pretrained(model_type=model_type) 200 if is_eval: 201 model.eval()

File ~/MA-LMM/lavis/models/base_model.py:69, in BaseModel.from_pretrained(cls, model_type) 59 """ 60 Build a pretrained model from default configuration file, specified by model_type. 61 (...) 66 - model (nn.Module): pretrained or finetuned model, depending on the configuration. 67 """ 68 model_cfg = OmegaConf.load(cls.default_config_path(model_type)).model ---> 69 model = cls.from_config(model_cfg) 71 return model

File ~/MA-LMM/lavis/models/blip_models/blip_classification.py:158, in BlipClassification.from_config(cls, cfg) 155 alpha = cfg.get("alpha", 0.4) 156 max_txt_len = cfg.get("max_txt_len", 40) --> 158 assert num_classes > 1, "Invalid number of classes provided, found {}".format( 159 num_classes 160 ) 162 model = cls( 163 image_encoder=image_encoder, 164 text_encoder=text_encoder, (...) 169 max_txt_len=max_txt_len, 170 ) 172 # load pre-trained weights

AssertionError: Invalid number of classes provided, found -1

[mvsoom]

Did you manually download Vicuna 7b v1.1? See https://github.com/boheumd/MA-LMM/issues/7#issuecomment-2084991618