Memory leak issue with maskrcnn

[ankitsharma07]

❓ Questions and Help

I am running maskrcnn_benchmark in the server and facing an issue of memory leak. I have made a flask application which when gets a request utilizes 800 MB of memory. On making another request the memory starts accumulating from 800 MB and goes further. On sending multiple requests simultaneously the program occupies all the allocated memory and exits.

I am not able to figure out where the memory is leaking. I am attaching some code snippets where I am using maskrcnn in my program.

def process_maskrcnn(self,front,side):
        # try:
        self.obj_maskrcnn.loader()
        frontrcnnmask,frontmask = self.obj_maskrcnn.run_maskrcnn(front,'f')
        sidercnnmask,sidemask = self.obj_maskrcnn.run_maskrcnn(side,'s')
        cv2.imwrite("frontmaskrcnn.jpg",frontrcnnmask)
        cv2.imwrite("sidemaskrcnn.jpg",sidercnnmask)
        gc.collect()
        return True,frontrcnnmask,sidercnnmask,frontmask,sidemask

def run_maskrcnn(self, image,pose):
        if pose == 'f':
            predictions = self.compute_prediction(image)
            top_predictions = self.select_top_predictions(predictions)
            result = image.copy()
            maskimg,maskimgall = self.overlay_mask(result, top_predictions)
            # kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
            # maskimg = cv2.dilate(maskimg, kernel)
            # maskimgall = cv2.dilate(maskimgall, kernel)
            if cfg.debug == 'True':
                cv2.imwrite(self.outputdir + '/front.jpg', maskimg)
            return maskimg, maskimgall
        if pose == 's':
            predictions = self.compute_prediction(image)
            top_predictions = self.select_top_predictions(predictions)
            result = image.copy()
            maskimg, maskimgall = self.overlay_mask(result, top_predictions)
            kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15, 15))
            maskimg = cv2.dilate(maskimg, kernel)
            maskimgall = cv2.dilate(maskimgall, kernel)
            if cfg.debug == 'True':
                cv2.imwrite(self.outputdir + '/side.jpg', maskimg)
            return maskimg, maskimgall

def compute_prediction(self, original_image):
        image = self.transforms(original_image)
        image_list = to_image_list(image, self.cfg.DATALOADER.SIZE_DIVISIBILITY)
        image_list = image_list.to(self.device)
        with torch.no_grad():
            predictions = self.model(image_list)
        predictions = [o.to(self.cpu_device) for o in predictions]
        prediction = predictions[0]
        height, width = original_image.shape[:-1]
        prediction = prediction.resize((width, height))
        if prediction.has_field("mask"):
            masks = prediction.get_field("mask")
            masks = self.masker([masks], [prediction])[0]
            prediction.add_field("mask", masks)

def select_top_predictions(self, predictions):
        scores = predictions.get_field("scores")
        keep = torch.nonzero(scores > self.confidence_threshold).squeeze(1)
        predictions = predictions[keep]
        scores = predictions.get_field("scores")
        _, idx = scores.sort(0, descending=True)
        return predictions[idx]```

[sanli0]

I have the same problem. Have you solved your problem?

[ankitsharma07]

@sanli0 What OS are you running maskrcnn_benchmark?

[sanli0]

centos

[ankitsharma07]

I used something like this to resolve the issue:

def process_maskrcnn(self, image):
        try:
            manager = multiprocessing.Manager()
            return_dict = manager.dict()

            def worker_process(return_dic, image):
                imagercnnmask,  imagemask = self.obj_maskrcnn.run_maskrcnn(image)
                return_dic['1'] = True
                return_dic['2'] = imagercnnmask
                return_dic['3'] = imagemask
            p = Process(target=worker_process, args=(return_dict, image))
            p.start()
            p.join()
            return return_dict.values()

        except:
            return False, None, None

[ankitsharma07]

@sanli0 Did it solve your problem?