Error in pytorch_yolo.py

[c0052742]

I encountered an error while trying to run the ART_attack.py script for my YOLOv5 object detection model. The error message reads: The code:

model = torch.hub.load('ultralytics/yolov5', 'custom', path='runs/train/exp/weights/best.pt')
model.eval()
config = {
            "attack_losses": ["loss_classifier", "loss_box_reg", "loss_objectness", "loss_rpn_box_reg"],}
estimator = PyTorchYolo(
    model=model, device_type="cuda", input_shape=(3, 640, 640), clip_values=(0, 255), attack_losses=config["attack_losses"]
)
test_image_folder = 'data/KITTI/test/images_resized'
output_folder = 'adversarial_images'
image_filenames = os.listdir(test_image_folder)
image_filenames = [filename for filename in image_filenames if filename.endswith('.png')]

for filename in image_filenames:
    image_path = os.path.join(test_image_folder, filename)
    image = np.asarray(Image.open(image_path).resize((640, 640)))
    img_reshape = image.transpose((2, 0, 1))
    im = np.stack([img_reshape], axis=0).astype(np.float32)
    x = im.copy()
    fgm = FastGradientMethod(estimator=estimator, eps=0.01, targeted=False)

    adversarial_image_fgm = fgm.generate(x=x, y=None)

    pgd = ProjectedGradientDescent(estimator=estimator, eps=0.01, eps_step=0.005, max_iter=100, targeted=False)

    adversarial_image_pgd = pgd.generate(x=x)

    carlini = CarliniL2Method(estimator=estimator, confidence=0.0, targeted=False, learning_rate=0.01, binary_search_steps=10,
                              max_iter=10, initial_const=0.01, clip_min=0.0, clip_max=1.0)
    adversarial_image_carlini = carlini.generate(x=x)

    adversarial_image_fgm = adversarial_image_fgm.squeeze(0).permute(1, 2, 0).numpy()
    adversarial_image_pgd = adversarial_image_pgd.squeeze(0).permute(1, 2, 0).numpy()
    adversarial_image_carlini = adversarial_image_carlini.squeeze(0).permute(1, 2, 0).numpy()

    # Convert pixel values from [0,1] to [0,255] and cast to uint8
    adversarial_image_fgm = np.uint8(adversarial_image_fgm * 255)
    adversarial_image_pgd = np.uint8(adversarial_image_pgd * 255)
    adversarial_image_carlini = np.uint8(adversarial_image_carlini * 255)

    if not os.path.exists(output_folder):/
      os.makedirs(output_folder)
    Image.fromarray(adversarial_image_fgm).save(os.path.join(output_folder, 'fgm_' + filename))
    Image.fromarray(adversarial_image_pgd).save(os.path.join(output_folder, 'pgd_' + filename))
    Image.fromarray(adversarial_image_carlini).save(os.path.join(output_folder, 'carlini_' + filename))

The error: "Traceback (most recent call last): File "c:\dis\dissertation - Copy\yolov5\ART_attack.py", line 52, in <module> adversarial_image_fgm = fgm.generate(x=x, y=None) File "C:\Users\c0052742\AppData\Roaming\Python\Python310\site-packages\art\attacks\evasion\fast_gradient.py", line 316, in generate adv_x_best = self._compute( File "C:\Users\c0052742\AppData\Roaming\Python\Python310\site-packages\art\attacks\evasion\fast_gradient.py", line 540, in _compute perturbation = self._compute_perturbation(batch, batch_labels, mask_batch, decay, momentum) File "C:\Users\c0052742\AppData\Roaming\Python\Python310\site-packages\art\attacks\evasion\fast_gradient.py", line 395, in _compute_perturbation grad = self.estimator.loss_gradient(x, y) * (1 - 2 * int(self.targeted)) File "C:\Users\c0052742\AppData\Roaming\Python\Python310\site-packages\art\estimators\object_detection\pytorch_yolo.py", line 377, in loss_gradient loss = output[loss_name] TypeError: list indices must be integers or slices, not str"

The error seems to be related to the line of code that tries to access an element in a list using a string as an index. Specifically, the "loss_name" variable is a string that is being used to access an element in the "output" list, which is causing the error.

I am not sure what is causing this issue and how to resolve it. Any help would be appreciated. I have looked into other similar issue like #1796 but I can't seem to fix it while using YOLOv5.

[beat-buesser]

Hi @c0052742 Thank you for using ART for YOLO!

@kieranfraser What do you think?

[kieranfraser]

Thanks @beat-buesser,

Hi @c0052742, thanks for reaching out!

We currently have a notebook here that demonstrates how to set up YOLOv5 for an attack with ART.

Also, just to note that the CarliniL2Method attack you have implemented will fail as it supports classification estimators and not object detectors. FGM and PGD should work.

If more interested in the individual loss components, as opposed the total, they differ slightly between YOLOv3 and v5 (which you can see here) - they're implemented in this example below which should help.

Let us know how you get on!

import torch
import numpy as np

from art.attacks.evasion.fast_gradient import FastGradientMethod
from art.attacks.evasion import ProjectedGradientDescent
from art.estimators.object_detection import PyTorchYolo

import yolov5
from yolov5.utils.loss import ComputeLoss

import requests
from PIL import Image
from io import BytesIO

response = requests.get('https://ultralytics.com/images/zidane.jpg')
img = np.asarray(Image.open(BytesIO(response.content)).resize((640, 640)))
img_reshape = img.transpose((2, 0, 1))
image = np.stack([img_reshape], axis=0).astype(np.float32)
x = image.copy()

config = {"attack_losses": ["loss_total", "loss_box", "loss_obj", "loss_cls"],}

class Yolo(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model
        self.model.hyp = {'box': 0.05,
                        'obj': 1.0,
                        'cls': 0.5,
                        'anchor_t': 4.0,
                        'cls_pw': 1.0,
                        'obj_pw': 1.0,
                        'fl_gamma': 0.0
                        }
        self.compute_loss = ComputeLoss(self.model.model.model)

    def forward(self, x, targets=None):
        if self.training:
            outputs = self.model.model.model(x)
            loss, loss_items = self.compute_loss(outputs, targets)
            loss_components_dict = {"loss_total": loss,
                                    "loss_box": loss_items[0],
                                    "loss_obj": loss_items[1],
                                    "loss_cls": loss_items[2]}
            return loss_components_dict
        else:
            return self.model(x)

model = yolov5.load('yolov5s.pt')

model = Yolo(model)

detector = PyTorchYolo(model=model,
                    device_type='cpu',
                    input_shape=(3, 640, 640),
                    clip_values=(0, 255), 
                    attack_losses=config["attack_losses"])

fgm = FastGradientMethod(estimator=detector, eps=0.01, targeted=False)
pgd = ProjectedGradientDescent(estimator=detector, eps_step=0.005, max_iter=100, targeted=False)

adversarial_image_fgm = fgm.generate(x=x, y=None)
adversarial_image_pgd = pgd.generate(x=x)