Model Misclassification (All Images Classified as NSFW)

[mehranfvs]

Description :

I've been using the "MobileVLMv2-1.7B" model for a task where I need to classify images as either "NSFW" or "SFW". However, the model consistently classifies all images as NSFW, regardless of their actual content. This behavior persists even with various prompt modifications and different image inputs.

Reproduce: prompt = "Is this picture sfw or nsfw?\nAnswer the question using a single word of nsfw or sfw"

Expected Behavior: The model should classify images accurately as SFW or NSFW based on the content.

Actual Behavior: All images are classified as NSFW, even those that are clearly SFW.

I made some changes to the script including encapsulating functionalities within a class and ... However, the overall workflow and logic of the code remain consistent with the original version. https://github.com/Meituan-AutoML/MobileVLM/blob/main/scripts/inference.py

my code :

import sys import os import torch import argparse from PIL import Image from pathlib import Path from time import time import transformers import warnings

  transformers.logging.set_verbosity_error()
  warnings.filterwarnings('ignore')

  sys.path.append(str(Path(__file__).parent.parent.resolve()))
  from mobilevlm.model.mobilevlm import load_pretrained_model
  from mobilevlm.conversation import conv_templates, SeparatorStyle
  from mobilevlm.utils import disable_torch_init, process_images, tokenizer_image_token, KeywordsStoppingCriteria
  from mobilevlm.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN

  class inference_once():

      def __init__(self):
          pass

      def load_model(self,model_path,load_8bit=False,load_4bit=False):
          model_name = model_path.split('/')[-1]
          self.tokenizer, self.model, self.image_processor, context_len = load_pretrained_model(model_path,
                                                                                                load_8bit,
                                                                                                load_4bit)
          disable_torch_init()

      def load_image(self,image_file):
          images = [(Image.open(image_file).convert("RGB")).resize((512, 512), Image.ANTIALIAS)]
          self.images_tensor = process_images(images,
                                         self.image_processor,
                                         self.model.config).to(self.model.device,
                                                              dtype=torch.float16)

      def infer(self,args):
          conv = conv_templates[args.conv_mode].copy()
          conv.append_message(conv.roles[0], DEFAULT_IMAGE_TOKEN + "\n" + args.prompt)
          conv.append_message(conv.roles[1], None)

          prompt = conv.get_prompt()
          stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2

          # Input
          self.input_ids = (tokenizer_image_token(prompt,
                                             self.tokenizer,
                                             IMAGE_TOKEN_INDEX,
                                             return_tensors="pt").unsqueeze(0).cuda())

          self.stopping_criteria = KeywordsStoppingCriteria([stop_str],
                                                       self.tokenizer,
                                                       self.input_ids)

          # Inference
          with torch.inference_mode():
              output_ids = self.model.generate(
                  self.input_ids,
                  images= self.images_tensor,
                  do_sample= True if args.temperature > 0 else False,
                  temperature= args.temperature,
                  top_p= args.top_p,
                  num_beams= args.num_beams,
                  max_new_tokens= args.max_new_tokens,
                  use_cache=True,
                  stopping_criteria=[self.stopping_criteria])

          # Result-Decode
          input_token_len = self.input_ids.shape[1]
          n_diff_input_output = (self.input_ids != output_ids[:, :input_token_len]).sum().item()

          if n_diff_input_output > 0:
              print(f"[Warning] {n_diff_input_output} output_ids are not the same as the input_ids")

          outputs = self.tokenizer.batch_decode(output_ids[:, input_token_len:],
                                                  skip_special_tokens=True)[0]
          outputs = outputs.strip()
          if outputs.endswith(stop_str):
              outputs = outputs[: -len(stop_str)]
          # print(f"🚀 {model_name}: {outputs.strip()}\n")
          return outputs

  if __name__ == "__main__":
      model_path = "/MobileVLM_V2-1.7B"
      image_folder = "/samples"
      prompt = "Is this picture sfw or nsfw?\nAnswer the question using a single word of nsfw or sfw"

      mvlm2 = inference_once()
      s_l = time()
      mvlm2.load_model(model_path)
      e_l = time()- s_l
      print(f"load time : {e_l:.2f} seconds")

      images = os.listdir(image_folder)
      for img in images:
          img_path = os.path.join(image_folder, img)
          print(img)

          s = time()

          args = type('Args', (), {
          "prompt": prompt,
          "conv_mode": "v1",
          "temperature": 0, 
          "top_p": None,
          "num_beams": 1,
          "max_new_tokens": 512
          })()

          mvlm2.load_image(img_path)
          output = mvlm2.infer(args)

          e = time()-s

          print("\033[92m", 50 * "-")
          print(f"time : {e:.2f} seconds")
          print(f"🚀 output {img} : {output.strip()}")
          print(50 * "-", "\033[0m")

output :