Big Project Plan Trimester #3 - Ethan Tran

[realethantran]

Build upon previous work

So far, our backend has...

• [x] A working AI model that makes predictions on what sign the user is making with their hands (ASL Alphabet and Custom signs)
• [x] Data storage for each frame from the backend (what we use to make predictions)

To be finished - Hand Recognition Generalization

Improve accuracy of the model (further training of the model for accuracy) OpenCV's bounding box capabilities rely solely on geometric properties and the pixels taken from the frontend in the bounded area. Through Generalization, the hand's appearance in different environments such as lighting.

Timeline

• [x] Research generalization, understand how it works and why it's implementation will allow for adaptability in the machine learning model
• [x] Code for Data Augmentation of original frames I am working on code that takes each frame and creates new data based off the original frame in order to account for variability. This would allow the model to be trained on each frame in different lighting scenarios and allow for an accurate generalization/prediction.

    // method to create variations of an image
    private BufferedImage[] createAugmentedImages(BufferedImage originalImage) throws IOException {
        BufferedImage[] augmentedImages = new BufferedImage[9]; // array size remains 9 for 9 variations
        augmentedImages[0] = originalImage; // first image remains unchanged
        augmentedImages[1] = changeBrightness(originalImage, 0.8f); // decrease brightness by 20%
        augmentedImages[2] = changeBrightness(originalImage, 1.2f); // increase brightness by 20%
        augmentedImages[3] = addNoise(originalImage, 0.1); // add low level of noise
        augmentedImages[4] = addNoise(originalImage, 0.2); // add higher level of noise
        augmentedImages[5] = adjustHue(originalImage); // randomly adjust hue
        augmentedImages[6] = rotateImage(originalImage, 30); // rotate image by 30 degrees
        augmentedImages[7] = flipImage(originalImage, true); // flip image horizontally
        augmentedImages[8] = flipImage(originalImage, false); // flip image vertically

        // save each augmented image to a file using i+1 as filename
        for (int i = 0; i < augmentedImages.length; i++) {
            saveImage(augmentedImages[i], "augmented_image_" + (i + 1) + ".png");
        }

        return augmentedImages;
    }    

    // method to change brightness of an image
    private BufferedImage changeBrightness(BufferedImage image, float factor) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // adjust brightness for each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = (int)Math.min(255, ((rgba >> 16) & 0xff) * factor);
                int green = (int)Math.min(255, ((rgba >> 8) & 0xff) * factor);
                int blue = (int)Math.min(255, (rgba & 0xff) * factor);
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to add noise to an image
    private BufferedImage addNoise(BufferedImage image, double noiseLevel) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // add noise to each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = (int)Math.min(255, Math.max(0, (int)(((rgba >> 16) & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                int green = (int)Math.min(255, Math.max(0, (int)(((rgba >> 8) & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                int blue = (int)Math.min(255, Math.max(0, (int)((rgba & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to adjust hue of an image
    private BufferedImage adjustHue(BufferedImage image) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // adjust hue for each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = Math.min(255, (int)(((rgba >> 16) & 0xff) * (0.8 + Math.random() * 0.4)));
                int green = Math.min(255, (int)(((rgba >> 8) & 0xff) * (0.8 + Math.random() * 0.4)));
                int blue = Math.min(255, (int)((rgba & 0xff) * (0.8 + Math.random() * 0.4)));
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to rotate an image
    private BufferedImage rotateImage(BufferedImage image, double angle) {
        // create a transformation matrix for rotating the image
        AffineTransform tx = new AffineTransform();
        tx.rotate(Math.toRadians(angle), image.getWidth() / 2.0, image.getHeight() / 2.0);

        // apply the transformation to the image
        AffineTransformOp op = new AffineTransformOp(tx, AffineTransformOp.TYPE_BILINEAR);
        return op.filter(image, null);
    }

    // method to flip an image horizontally or vertically
    private BufferedImage flipImage(BufferedImage image, boolean horizontal) {
        // create a transformation matrix for flipping the image
        AffineTransform tx = new AffineTransform();
        if (horizontal) {
            tx.scale(-1, 1); // flip horizontally
            tx.translate(-image.getWidth(), 0);
        } else {
            tx.scale(1, -1); // flip vertically
            tx.translate(0, -image.getHeight());
        }

        // apply the transformation to the image
        AffineTransformOp op = new AffineTransformOp(tx, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);
        return op.filter(image, null);
    }

• [x] Implement Generalization to the model itself After completing the code that will take into account the variations ie. lighting, horizontal, tilt and form the next step is to utilize the mnist data from the model and use this in the model which will make the predictions

    private List<List<List<Integer>>> generateAugmentedFrames(String imageData) throws IOException {
        byte[] imageBytes = Base64.getDecoder().decode(imageData);
        ByteArrayInputStream bis = new ByteArrayInputStream(imageBytes);
        BufferedImage originalImage = ImageIO.read(bis);

        List<List<List<Integer>>> mnistDataList = new ArrayList<>();
        BufferedImage[] augmentedImages = createAugmentedImages(originalImage);

        for (BufferedImage image : augmentedImages) {
            mnistDataList.add(convertToMNIST(image));
        }

        return mnistDataList;
    }

[realethantran]

AugmentedFrames Key Features

Code for Data Augmentation of original frames

  // method to change brightness of an image
    private BufferedImage changeBrightness(BufferedImage image, float factor) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // adjust brightness for each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = (int)Math.min(255, ((rgba >> 16) & 0xff) * factor);
                int green = (int)Math.min(255, ((rgba >> 8) & 0xff) * factor);
                int blue = (int)Math.min(255, (rgba & 0xff) * factor);
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to add noise to an image
    private BufferedImage addNoise(BufferedImage image, double noiseLevel) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // add noise to each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = (int)Math.min(255, Math.max(0, (int)(((rgba >> 16) & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                int green = (int)Math.min(255, Math.max(0, (int)(((rgba >> 8) & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                int blue = (int)Math.min(255, Math.max(0, (int)((rgba & 0xff) * (1 + (Math.random() - 0.5) * noiseLevel))));
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to adjust hue of an image
    private BufferedImage adjustHue(BufferedImage image) {
        BufferedImage newImage = new BufferedImage(image.getWidth(), image.getHeight(), image.getType());
        Graphics2D g = newImage.createGraphics();
        g.drawImage(image, 0, 0, null);
        g.dispose();

        // adjust hue for each pixel
        for (int y = 0; y < newImage.getHeight(); y++) {
            for (int x = 0; x < newImage.getWidth(); x++) {
                int rgba = newImage.getRGB(x, y);
                int alpha = (rgba >> 24) & 0xff;
                int red = Math.min(255, (int)(((rgba >> 16) & 0xff) * (0.8 + Math.random() * 0.4)));
                int green = Math.min(255, (int)(((rgba >> 8) & 0xff) * (0.8 + Math.random() * 0.4)));
                int blue = Math.min(255, (int)((rgba & 0xff) * (0.8 + Math.random() * 0.4)));
                newImage.setRGB(x, y, (alpha << 24) | (red << 16) | (green << 8) | blue);
            }
        }
        return newImage;
    }

    // method to rotate an image
    private BufferedImage rotateImage(BufferedImage image, double angle) {
        // create a transformation matrix for rotating the image
        AffineTransform tx = new AffineTransform();
        tx.rotate(Math.toRadians(angle), image.getWidth() / 2.0, image.getHeight() / 2.0);

        // apply the transformation to the image
        AffineTransformOp op = new AffineTransformOp(tx, AffineTransformOp.TYPE_BILINEAR);
        return op.filter(image, null);
    }

    // method to flip an image horizontally or vertically
    private BufferedImage flipImage(BufferedImage image, boolean horizontal) {
        // create a transformation matrix for flipping the image
        AffineTransform tx = new AffineTransform();
        if (horizontal) {
            tx.scale(-1, 1); // flip horizontally
            tx.translate(-image.getWidth(), 0);
        } else {
            tx.scale(1, -1); // flip vertically
            tx.translate(0, -image.getHeight());
        }

        // apply the transformation to the image
        AffineTransformOp op = new AffineTransformOp(tx, AffineTransformOp.TYPE_NEAREST_NEIGHBOR);
        return op.filter(image, null);
    }

Implementation in the backend

Predictions in the backend for each augmented image

Key Errors

• MySQL error

data too long for column 'frame' at row 1

• This issue was prevalent because when sending the data for frames the MNIST data was very large
• To fix this issue, I increased the VARCHAR length so that I can insert more data into the column