Hi. I am modifying the cigar-10 program to utilize captured images from the onboard camera instead of the sample input. And it is not working now, as it predicts a cat in each image. I don't know what the issue is, but it probably has something to do with how I capture the image, process it, or load it into the CNN, or all of them simultaneously.
I have shared the code with you guys, and ANY HELP would be highly appreciated!
Any general understanding is also very much welcome, as I would like to deploy other models as well, such as a tinyissimoYOLO model or something similar.
Fingers crossed. Cheers!
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include "mxc.h"
#include "fcr_regs.h"
#include "icc.h"
#include "led.h"
#include "tmr.h"
#include "dma.h"
#include "cnn.h"
#include "weights.h"
#include "camera.h"
#define IMAGE_SIZE_X (32)
#define IMAGE_SIZE_Y (32)
#define CAMERA_FREQ (5 * 1000 * 1000)
const char classes[CNN_NUM_OUTPUTS][11] = {"airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"};
// Classification layer:
static int32_t ml_data[CNN_NUM_OUTPUTS];
static q15_t ml_softmax[CNN_NUM_OUTPUTS];
// CNN inference time variable
volatile uint32_t cnn_time; // Stopwatch
// Buffers for camera image
static uint32_t input_0[IMAGE_SIZE_X * IMAGE_SIZE_Y]; // buffer for camera image
// DMA channel for camera interface
int dma_channel;
int g_dma_channel_tft = 1;
void cnn_load_input(void)
{
memcpy32((uint32_t *)0x50400000, input_0, (IMAGE_SIZE_X * IMAGE_SIZE_Y));
}
void softmax_layer(void)
{
cnn_unload((uint32_t *)ml_data);
cnn_stop();
softmax_q17p14_q15((const q31_t *)ml_data, CNN_NUM_OUTPUTS, ml_softmax);
}
void capture_process_camera(void)
{
uint8_t *raw;
uint32_t imgLen;
uint32_t w, h;
int cnt = 0;
uint8_t r, g, b;
uint8_t *data = NULL;
stream_stat_t *stat;
camera_start_capture_image();
// Get the details of the image from the camera driver.
camera_get_image(&raw, &imgLen, &w, &h);
printf("\nW:%d H:%d L:%d \n", w, h, imgLen);
// Get image line by line
for (int row = 0; row < h; row++)
{
// Wait until camera streaming buffer is full
while ((data = get_camera_stream_buffer()) == NULL)
{
if (camera_is_image_rcv())
{
break;
}
}
for (int k = 0; k < 4 * w; k += 4)
{
// data format: 0x00bbggrr
r = data[k];
g = data[k + 1];
b = data[k + 2];
// skip k+3 because it is not used
// change the range from [0,255] to [-128,127] and store in buffer for CNN
input_0[cnt++] = ((b << 16) | (g << 8) | r) ^ 0x00808080;
}
// LED_Toggle(LED2);
// Release stream buffer
release_camera_stream_buffer();
}
// camera_sleep(1);
stat = get_camera_stream_statistic();
if (stat->overflow_count > 0)
{
printf("OVERFLOW DISP = %d\n", stat->overflow_count);
LED_On(LED2); // Turn on red LED if overflow detected
while (1)
{
}
}
}
int main(void)
{
int i;
int digs, tens;
int ret = 0;
int result[CNN_NUM_OUTPUTS] = {0}; // = {0} ensures that all elements are initialized to 0
int dma_channel;
MXC_Delay(SEC(2)); // Wait for 2 seconds
// Enable camera power
printf("Enable Camera Power...\n");
Camera_Power(POWER_ON);
printf("\nCifar-10 Feather Demo\n");
// Enable cache
MXC_ICC_Enable(MXC_ICC0);
// Switch to 100 MHz clock
MXC_SYS_Clock_Select(MXC_SYS_CLOCK_IPO);
SystemCoreClockUpdate();
printf("Waiting...\n");
// Enable peripheral, enable CNN interrupt, turn on CNN clock
// CNN clock: APB (50 MHz) div 1
cnn_enable(MXC_S_GCR_PCLKDIV_CNNCLKSEL_PCLK, MXC_S_GCR_PCLKDIV_CNNCLKDIV_DIV1);
// Configure P2.5, turn on the CNN Boost, which helps with CNN performance
// cnn_boost_enable(MXC_GPIO2, MXC_GPIO_PIN_5);
printf("Init CNN...\n");
cnn_init(); // Bring state machine into consistent state
cnn_load_weights(); // Load kernels
cnn_load_bias(); // Load bias values
cnn_configure(); // Configure state machine
// Initialize DMA for camera interface
MXC_DMA_Init();
dma_channel = MXC_DMA_AcquireChannel();
// Initialize camera
printf("Init Camera...\n");
camera_init(CAMERA_FREQ);
// Set up camera
ret = camera_setup(IMAGE_SIZE_X, IMAGE_SIZE_Y, PIXFORMAT_RGB888, FIFO_THREE_BYTE, STREAMING_DMA,
dma_channel);
if (ret != STATUS_OK)
{
printf("Error returned from setting up camera. Error %d\n", ret);
return -1;
}
camera_write_reg(0x11, 0x0); // Set the camera to sleep mode
printf("********** Next snapshot in 1 second **********\r\n");
MXC_Delay(SEC(1));
// Enable CNN clock
MXC_SYS_ClockEnable(MXC_SYS_PERIPH_CLOCK_CNN);
while (1)
{
LED_Off(LED1);
LED_Off(LED2);
// Capture image from camera
printf("\nCapture and process camera image.\n");
capture_process_camera();
// Start CNN processing
printf("\n*** CNN Inference Test ***\n");
cnn_start();
// Load camera image into CNN input buffer
printf("Load CNN input...\n");
cnn_load_input();
// Wait for CNN
SCB->SCR &= ~SCB_SCR_SLEEPDEEP_Msk; // SLEEPDEEP=0
while (cnn_time == 0)
__WFI();
// Softmax layer
softmax_layer();
// Print inference time
printf("Time for CNN: %d us\n\n", cnn_time);
// Print classification results
printf("\nClassification results:\n");
for (i = 0; i < CNN_NUM_OUTPUTS; i++)
{
digs = (1000 * ml_softmax[i] + 0x4000) >> 15;
tens = digs % 10;
digs = digs / 10;
result[i] = digs;
printf("[%7d] -> Class %d %11s: %d.%d%%\r\n\n", ml_data[i], i, classes[i], result[i],
tens);
}
printf("\n");
// Wait for 1 second
printf("********** Next snapshot in 1 second **********\r\n");
MXC_Delay(SEC(1));
}
return 0;
}
Hi. I am modifying the cigar-10 program to utilize captured images from the onboard camera instead of the sample input. And it is not working now, as it predicts a cat in each image. I don't know what the issue is, but it probably has something to do with how I capture the image, process it, or load it into the CNN, or all of them simultaneously.
I have shared the code with you guys, and ANY HELP would be highly appreciated! Any general understanding is also very much welcome, as I would like to deploy other models as well, such as a tinyissimoYOLO model or something similar.
Fingers crossed. Cheers!