Camera Read and Write FPS not syncced - Python

[weihan08]

Hi I am using Raspberry Pi 4b+ (4GB RAM) to implement a driver drowsiness detection system. The problem is apparently the processing of image and drowsiness determination is very taxing, so when the camera points at me the FPS reduces to 13 (Normally it is able to handle 30 FPS @ Full HD), whereas when the camera points to anywhere without a face, the FPS rises to 30. I have used threads to optimize the I/O read, but it doesn't seem to improve the FPS.

In the end, I don't mind if the whole video is 13 FPS, but the problem is when writing the video using openCV VideoWriter, I can only initialize the FPS once. So, when the camera FPS is variable but the video write is fixed, the video written will be fluctuating in speed all the time (slow with no face; normal with face).

As you can see from the code, for some reason, initializing the camera framerate to match the Video Write doesn't help. Writing is still not in sync for some reason.

Hope someone can help! Thanks in advance.

#python drowniness_yawn.py --webcam webcam_index

from scipy.spatial import distance as dist
# from imutils.video import VideoStream
from imutils.video.pivideostream import PiVideoStream
from imutils.video import FPS
from imutils import face_utils
from threading import Thread
import numpy as np
import argparse
import imutils
import datetime as dt
import time
import dlib
import cv2
import os

def alarm(msg):
    global alarm_status
    global alarm_status2
    global saying

    while alarm_status:
        print('Eyes Closed')
        saying = True
#         s = 'espeak "'+msg+'"'
#         os.system(s)
        saying = False

    if alarm_status2:
        print('Yawn')
        saying = True
#         s = 'espeak "' + msg + '"'
#         os.system(s)
        saying = False

def eye_aspect_ratio(eye):
    A = dist.euclidean(eye[1], eye[5])
    B = dist.euclidean(eye[2], eye[4])

    C = dist.euclidean(eye[0], eye[3])

    ear = (A + B) / (2.0 * C)

    return ear

def final_ear(shape):
    (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
    (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]

    leftEye = shape[lStart:lEnd]
    rightEye = shape[rStart:rEnd]

    leftEAR = eye_aspect_ratio(leftEye)
    rightEAR = eye_aspect_ratio(rightEye)

    ear = (leftEAR + rightEAR) / 2.0
    return (ear, leftEye, rightEye)

def lip_distance(shape):
    top_lip = shape[50:53]
    top_lip = np.concatenate((top_lip, shape[61:64]))

    low_lip = shape[56:59]
    low_lip = np.concatenate((low_lip, shape[65:68]))

    top_mean = np.mean(top_lip, axis=0)
    low_mean = np.mean(low_lip, axis=0)

    distance = abs(top_mean[1] - low_mean[1])
    return distance

ap = argparse.ArgumentParser()
ap.add_argument("-w", "--webcam", type=int, default=0,
                help="index of webcam on system")
args = vars(ap.parse_args())

EYE_AR_THRESH = 0.3
EYE_AR_CONSEC_FRAMES = 45
YAWN_THRESH = 40
alarm_status = False
alarm_status2 = False
saying = False
COUNTER = 0

print("-> Loading the predictor and detector...")
#detector = dlib.get_frontal_face_detector()
detector = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")    #Faster but less accurate
predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

print("-> Starting Video Stream")
#vs = VideoStream(src=args["webcam"]).start()
vs= PiVideoStream().start()
vs.camera.framerate = 13

# vs= VideoStream(usePiCamera=True, framerate = 15).start()      #For Raspberry Pi
time.sleep(1.0)

fps = FPS().start()  

fourcc = cv2.VideoWriter_fourcc(*"mp4v")
writer = None
(h,w) = (None, None)

while True:
    timestamp = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    frame = vs.read()
    frame = imutils.resize(frame, width=400)
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    fps.update()
    fps.stop()

    NEW_FPS = round(fps.fps(),2)
    print(NEW_FPS)
    if writer is None:
        (height,width,layers)= frame.shape
        writer = cv2.VideoWriter('/home/pi/Desktop/video.mp4', fourcc, 13 , (width,height), True)
    cv2.putText(frame, timestamp,(130, 280), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255), 2)
    #rects = detector(gray, 0)
    rects = detector.detectMultiScale(gray, scaleFactor=1.1, 
        minNeighbors=5, minSize=(30, 30),
        flags=cv2.CASCADE_SCALE_IMAGE)

    #for rect in rects:
    for (x, y, w, h) in rects:
        rect = dlib.rectangle(int(x), int(y), int(x + w),int(y + h))

        shape = predictor(gray, rect)
        shape = face_utils.shape_to_np(shape)

        eye = final_ear(shape)
        ear = eye[0]
        leftEye = eye[1]
        rightEye = eye[2]

        distance = lip_distance(shape)

        leftEyeHull = cv2.convexHull(leftEye)
        rightEyeHull = cv2.convexHull(rightEye)
        cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
        cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)

        lip = shape[48:60]
        cv2.drawContours(frame, [lip], -1, (0, 255, 0), 1)

        if ear < EYE_AR_THRESH:
            COUNTER += 1

            if COUNTER >= EYE_AR_CONSEC_FRAMES:
                if alarm_status == False and saying == False:
                    alarm_status = True
                    t = Thread(target=alarm, args=('wake up sir',))
                    t.deamon = True
                    t.start()

                cv2.putText(frame, "DROWSINESS ALERT!", (10, 30),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

        else:
            COUNTER = 0
            alarm_status = False

        if (distance > YAWN_THRESH):
                cv2.putText(frame, "Yawn Alert", (10, 30),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
                if alarm_status2 == False and saying == False:
                    alarm_status2 = True
                    t = Thread(target=alarm, args=('take some fresh air sir',))
                    t.deamon = True
                    t.start()
        else:
            alarm_status2 = False

        cv2.putText(frame, "EAR: {:.2f}".format(ear), (250, 30),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
        cv2.putText(frame, "YAWN: {:.2f}".format(distance), (250, 60),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

    writer.write(frame)
    cv2.imshow("Frame", frame)
    key = cv2.waitKey(1) & 0xFF

    if key == ord("q"):
        fps.stop()
        print("Elapsed Time: {:.2f}".format(fps.elapsed()))
        print("approx FPS: {:.2f}".format(fps.fps()))
        break

cv2.destroyAllWindows()
vs.stop()
writer.release()