search

friendgooeun / javacv

Automatically exported from code.google.com/p/javacv
GNU General Public License v2.0
0 stars 0 forks source link

Memory Leak with my Face Detection Implementation #158

Closed GoogleCodeExporter closed 8 years ago

GoogleCodeExporter commented 8 years ago 
What steps will reproduce the problem?
1. Trying to detect 3000 faces with the code below

This main method parses Images of a Folder (TEST_DIR: containing 2700 positives 
and 300 negatives).
I iterate over the positives and negatives and detect the faces (with method 
detectFacesWithoutRotating)
on the images. 

        private static final int MIN_NEIGHBOURS = 3;
    private static final double SCALE_FACTOR = 1.1;
    private static final int FLAGS = 0;
    private static final String CASCADE_FILE = "haarcascade_frontalface_alt_tree.xml";

        public static void main(String[] args) {

        File dir = new File(TEST_DIR);
        File resultDir = new File(TEST_DIR + "/results");
        resultDir.mkdir();

        if (!dir.isDirectory()) {
            throw new IllegalArgumentException("Please provide a directory");
        }

        File[] positives = dir.listFiles(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                if (name.startsWith("positives_") && name.endsWith("jpg"))
                    return true;
                return false;
            }
        });

        File[] negatives = dir.listFiles(new FilenameFilter() {
            @Override
            public boolean accept(File dir, String name) {
                if (name.startsWith("negatives") && name.endsWith("jpg"))
                    return true;
                return false;
            }
        });

        logger.info("{} positives found", positives.length);
        logger.info("{} negatives found", negatives.length);

        ViolaJonesFaceDetector faceDetector = new ViolaJonesFaceDetector();
        logger.info("Face detector initialized");

        DetectionReport report = new DetectionReport(positives.length,
                negatives.length, resultDir.getAbsolutePath() + "/report.txt",
                resultDir.getAbsolutePath());

        report.startTimer();

        StringBuilder resultPath, fileName;

        for (int i = 0; i < positives.length; i++) {
            long start = System.currentTimeMillis();
            File file = positives[i];
            BufferedImage image = null;
            try {
                image = ImageIO.read(file);
            } catch (IOException e1) {
                logger.error("Error reading image: {}", file.getName());
            }

            logger.info("File: {} under inspection", file.getName());
            if (image != null) {
                logger.info("Detecting with cascade: {}, minRectangles: {}, scaleFactor: {}, cannyPruning: {}",new Object[] { CASCADE_FILE.split("/")[9],                       MIN_NEIGHBOURS, SCALE_FACTOR, FLAGS });
                FaceDetectionResult result = detect(faceDetector, image);
                Face[] faces = result.getFaces();
                fileName = new StringBuilder(file.getName());
                if (faces.length != 0) {
                    report.incrementTruePositives();
                    report.incrementFacesFound(faces.length);
                    logger.info("{} faces found", faces.length);

                    resultPath = new StringBuilder().append(
                            resultDir.getAbsolutePath()).append("/").append(
                            "true").append("_").append(fileName);
                    try {
                        ImageIO.write(ImageUtils.paintFacesOnImage(result
                                .getRotatedImg(), faces), "jpg", new File(
                                resultPath.toString()));
                    } catch (IOException e) {
                        logger.error("Error writing image: {}", resultPath);
                    }

                } else {
                    report.incrementFalseNegatives();
                    resultPath = new StringBuilder().append(
                            resultDir.getAbsolutePath()).append("/").append(
                            "false").append("_negatives_").append(
                            file.getName().split("_")[1]);

                    logger.info("No face found");
                    try {
                        ImageIO.write(image, "jpg", new File(resultPath
                                .toString()));
                    } catch (IOException e) {
                        logger.error("Error writing image: {}", resultPath);
                    }

                }
                report.addDetTime(file.getName(), Long.valueOf(System
                        .currentTimeMillis()
                        - start));
            }

        }

        for (int j = 0; j < negatives.length; j++) {
            long start = System.currentTimeMillis();

            File file = negatives[j];
            BufferedImage image = null;
            try {
                image = ImageIO.read(file);
            } catch (IOException e1) {
                logger.error("Error reading image: {}", file.getName());
            }

            logger.info("File: {} under inspection", file.getName());

            if (image != null) {
                logger.info("Detecting with cascade: {}, minRectangles: {}, scaleFactor: {}, cannyPruning: {}",new Object[] { CASCADE_FILE.split("/")[9],                               MIN_NEIGHBOURS, SCALE_FACTOR, FLAGS });

                FaceDetectionResult result = detect(faceDetector, image);
                Face[] faces = result.getFaces();
                fileName = new StringBuilder(file.getName());
                if (faces.length != 0) {
                    report.incrementFalsePositives();
                    report.incrementFacesFound(faces.length);

                    logger.info("{} faces found", faces.length);

                    resultPath = new StringBuilder().append(
                            resultDir.getAbsolutePath()).append("/").append(
                            "false").append("_positives_").append(
                            file.getName().split("_")[1]);

                    try {
                        ImageIO.write(ImageUtils.paintFacesOnImage(result
                                .getRotatedImg(), faces), "jpg", new File(
                                resultPath.toString()));
                    } catch (IOException e) {
                        logger.error("Error writing image: {}", resultPath);
                    }

                } else {
                    report.incrementTrueNegatives();
                    resultPath = new StringBuilder().append(
                            resultDir.getAbsolutePath()).append("/").append(
                            "true").append("_").append(fileName);

                    logger.info("No face found");
                    try {
                        ImageIO.write(image, "jpg", new File(resultPath
                                .toString()));
                    } catch (IOException e) {
                        logger.error("Error writing image: {}", resultPath);
                    }

                }
                report.addDetTime(file.getName(), Long.valueOf(System
                        .currentTimeMillis()
                        - start));
            }

        }

        report.stopTimer();
        report.printReport();

    }

Here is Method detect(..) calling the detection function:

private static FaceDetectionResult detect(
            ViolaJonesFaceDetector faceDetector, BufferedImage image) {
        return faceDetector.detectFacesWithoutRotating(image, CASCADE_FILE,
                SCALE_FACTOR, MIN_NEIGHBOURS, FLAGS);
    }

Important Parts of Class ViolaJonesFaceDetector:

        private CvHaarClassifierCascade frontalFaceCasc;
    private CvHaarClassifierCascade profileFaceCasc;
    private CvMemStorage storage;

    /**
     * Init the detector with a certain cascade file
     * 
     * @param cascade
     */
    public void init(String cascade) {
        // check for null
        if (cascade == null) {
            String error = "cascade must not be null";
            logger.error(error);
            throw new IllegalArgumentException(error);
        }

        // Preload the opencv_objdetect module to work around a known bug.
        Loader.load(opencv_objdetect.class);

        // Objects allocated with a create*() or clone() factory method are
        // automatically released
        // by the garbage collector, but may still be explicitly released by
        // calling release().
        // You shall NOT call cvReleaseImage(), cvReleaseMemStorage(), etc. on
        // objects allocated this way.
        if (this.storage == null) {
            this.storage = CvMemStorage.create();
        } else {
            cvClearMemStorage(this.storage);
        }
        // We instantiate a classifier cascade to be used for detection,
        // using the cascade definition.
        if (this.frontalFaceCasc == null
                || !cascade.equals(FRONTALFACE_CASCADE_FILE)) {
            this.frontalFaceCasc = new CvHaarClassifierCascade(cvLoad(cascade));
        }
        // instantiate classifier profileface used as a last step
        if (this.profileFaceCasc == null) {
            this.profileFaceCasc = new CvHaarClassifierCascade(
                    cvLoad(PROFILEFACE_CASCADE_FILE));
        }
        if (this.frontalFaceCasc.isNull()) {
            logger.error("Error loading classifier file {}", cascade);
        }
        if (this.profileFaceCasc.isNull()) {
            logger.error("Error loading classifier file {}",
                    PROFILEFACE_CASCADE_FILE);
        }
    }

        /**
     * DONOTUSE ANYMORE
     * 
     * @param image
     * @param cascade
     * @param scaleFactor
     * @param minNeighbours
     * @param flags
     * @return
     */
    public FaceDetectionResult detectFacesWithoutRotating(BufferedImage image,
            String cascade, double scaleFactor, int minNeighbours, int flags) {
        // init cascade
        init(cascade);

        // create IplImage form BufferedImage
        IplImage original = IplImage.createFrom(image);

        IplImage grayImage = null;

        if (original.nChannels() >= 3) {
            // We need a grayscale image in order to do the recognition, so we
            // create a new image of the same size as the original one.
            grayImage = IplImage.create(image.getWidth(), image.getHeight(),
                    IPL_DEPTH_8U, 1);
            // We convert the original image to grayscale.
            cvCvtColor(original, grayImage, CV_BGR2GRAY);
        } else {
            grayImage = original.clone();
        }

        // if set equalize the image
        if (EQUALIZE_HISTOGRAM) {
            IplImage grayImageEqualized = IplImage.create(original.width(),
                    original.height(), IPL_DEPTH_8U, 1);
            grayImageEqualized = IplImageUtils.equalizeHist(grayImage);
            grayImage.release();
            grayImage = null;
            grayImage = grayImageEqualized.clone();
            grayImageEqualized.release();
            grayImageEqualized = null;
        }

        // We detect the faces.
        CvSeq faces = cvHaarDetectObjects(grayImage, this.frontalFaceCasc,
                this.storage, scaleFactor, minNeighbours, flags);

        Face[] resultFaces = fillFaceArrayWithCvSeq(faces);

        // clear storage and images after detection
        cvClearMemStorage(faces.storage());
        cvClearMemStorage(this.storage);

        // call garbage collector to be sure
        System.gc();

        return new FaceDetectionResult(resultFaces, image, image, false, 0.0f);
    }

    /**
     * Create a list of Faces out of the CvSeq class which contains the
     * CvRectangles defining the faces borders
     * 
     * @param faces
     *            Sequence of faces
     * @return List of the detected faces. If no face is found it returns an
     *         empty Face List.
     */
    private Face[] fillFaceArrayWithCvSeq(CvSeq faces) {
        if (faces.total() == 0) {
            return new Face[0];
        } else {
            Face[] result = new Face[faces.total()];
            // We iterate over the discovered faces and add them to our
            // Face-object list
            for (int i = 0; i < faces.total(); i++) {
                CvRect r = new CvRect(cvGetSeqElem(faces, i));
                result[i] = new Face(r.x(), r.y(), r.width(), r.height());
            }

            return result;
        }
    }

2. It seems that this problem only appears if i use the cascade-file: 
"haarcascade_frontalface_alt_tree.xml".
".._alt.xml" etc. still make memory grow but its not overflowing.  
3. I read other issues regarding memory leaks and tried to apply the hints 
(espacially IplImage.clone()/create() and createFrom() + releasing again hints) 
on my problem but maybe i did something wrong)
4. I start this code using eclipse

What is the expected output? What do you see instead?

I run the detection and the java process starts consuming more and more memory 
until it kills my eclipse application. After that the java process is still 
there using the same amount of memory.

What version of the product are you using? On what operating system?

Eclipse 3.5
java-6-openjdk
opencv 2.3.1
ubuntu 64 bit

Please provide any additional information below.

Original issue reported on code.google.com by cesma...@googlemail.com on 23 Feb 2012 at 1:17

GoogleCodeExporter commented 8 years ago 
Would it possible to verify that this does not happen when using OpenCV 
directly in C/C++, without JavaCV? OpenCV has quite a few leaks itself, and it 
looks like one of those...

Original comment by samuel.a...@gmail.com on 23 Feb 2012 at 1:42

GoogleCodeExporter commented 8 years ago 
Hi I nailed it.. First of all I made some googling trying to reimplement my 
program in c/c++ as you suggested. I came back to the original face detection 
homepage of opencv found an interesting paragraph: 
http://opencv.willowgarage.com/wiki/FaceDetection :

"The above code (function detect and draw) has a serious memory leak when run 
in an infinite for loop for real time face detection. Please add 
"cvReleaseMemStorage(&storage);" after you release the temp image in the detect 
and draw function. The cvClearMemStorage only clears the previous set of values 
but does not free memory. I did not modify the initial code as it is simply not 
mine. I have also cleared "faces" using cvClearSeq in my process of trying to 
find the memory leak as the program would quickly crash for 640x480 res 30fps 
video. I am very thankful to the original programmer for the code. Please let 
me know if this helps. --AbhinayE"

So I tried to use cvReleaseMemStorage() instead of cvClearMemStorage() in my 
program. The first two detections were successful but then I got an internal 
native code error. Head of the error file:

#
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGSEGV (0xb) at pc=0x00007f86bd733813, pid=29982, tid=140216804407040
#
# JRE version: 6.0_20-b20
# Java VM: OpenJDK 64-Bit Server VM (19.0-b09 mixed mode linux-amd64 compressed 
oops)
# Derivative: IcedTea6 1.9.10
# Distribution: Ubuntu 10.04.1 LTS, package 6b20-1.9.10-0ubuntu1~10.04.2
# Problematic frame:
# C  [libopencv_core.so.2.3.1+0x58813]
#
# If you would like to submit a bug report, please include
# instructions how to reproduce the bug and visit:
#   https://bugs.launchpad.net/ubuntu/+source/openjdk-6/
# The crash happened outside the Java Virtual Machine in native code.
# See problematic frame for where to report the bug.
#
........

You should be able to reproduce this by using a detection loop and instead of 
clearing the storage with cvClearMemStorage() clearing it with 
cvReleaseMemStorage().

Ok now my solution: I finally used storage.release() as I do for images. This 
solution is somehow obvious since you explain in your README that the 
counterpart for create() is release() but all the examples I found use 
cvClearMemStorage().

Original comment by cesma...@googlemail.com on 23 Feb 2012 at 2:53

GoogleCodeExporter commented 8 years ago 
cvClearMemStorage() does not release any system memory... Read OpenCV's 
documentation about it:
http://opencv.willowgarage.com/documentation/c/dynamic_structures.html
The new C++ API tends to do without this little system, and instead rely on STL 
data structures, without properly evaluating the differences in performance IMO

Anyway, calling create() and release() all the time fixes the leak?

Original comment by samuel.a...@gmail.com on 23 Feb 2012 at 11:45

GoogleCodeExporter commented 8 years ago 
Hi Samuel,
Good to know, thanks for the advice. Yeah it fixed all the leaks. The final 
detection code looks like this:

public FaceDetectionResult detectFacesWithoutRotating(BufferedImage image,
            double scaleFactor, int minNeighbours, int flags) {

        if (this.frontalFaceCasc == null) {
            init(FRONTALFACE_CASCADE_FILE);
        }

        CvMemStorage tempStorage = CvMemStorage.create();

        // create IplImage form BufferedImage
        IplImage original = IplImage.createFrom(image);

        IplImage grayImage = null;

        if (original.nChannels() >= 3) {
            // We need a grayscale image in order to do the recognition, so we
            // create a new image of the same size as the original one.
            grayImage = IplImage.create(image.getWidth(), image.getHeight(),
                    IPL_DEPTH_8U, 1);
            // We convert the original image to grayscale.
            cvCvtColor(original, grayImage, CV_BGR2GRAY);
        } else {
            grayImage = original.clone();
        }

        // if set equalize the image
        if (EQUALIZE_HISTOGRAM) {
            IplImage grayImageEqualized = IplImage.create(original.width(),
                    original.height(), IPL_DEPTH_8U, 1);
            grayImageEqualized = IplImageUtils.equalizeHist(grayImage);
            grayImage.release();
            grayImage = null;
            grayImage = grayImageEqualized.clone();
            grayImageEqualized.release();
            grayImageEqualized = null;
        }

        // We detect the faces.
        CvSeq faces = cvHaarDetectObjects(grayImage, this.frontalFaceCasc,
                tempStorage, scaleFactor, minNeighbours, flags);

        Face[] resultFaces = fillFaceArrayWithCvSeq(faces);

        // clear storage and images after detection
        tempStorage.release();

        return new FaceDetectionResult(resultFaces, image, image, false, 0.0f);
    }

    /**
     * Create a list of Faces out of the CvSeq class which contains the
     * CvRectangles defining the faces borders
     * 
     * @param faces
     *            Sequence of faces
     * @return List of the detected faces. If no face is found it returns an
     *         empty Face List.
     */
    private Face[] fillFaceArrayWithCvSeq(CvSeq faces) {
        if (faces.total() == 0) {
            return new Face[0];
        } else {
            Face[] result = new Face[faces.total()];
            // We iterate over the discovered faces and add them to our
            // Face-object list
            for (int i = 0; i < faces.total(); i++) {
                CvRect r = new CvRect(cvGetSeqElem(faces, i));
                result[i] = new Face(r.x(), r.y(), r.width(), r.height());
            }

            return result;
        }
    }

I appreciate your help, Cheers

Original comment by cesma...@googlemail.com on 24 Feb 2012 at 1:02

GoogleCodeExporter commented 8 years ago 
Great, thanks for the feedback!

Original comment by samuel.a...@gmail.com on 24 Feb 2012 at 2:21