Camera Read and Write FPS not syncced - Python

[weihan08]

Hi I am using Raspberry Pi 4b+ (4GB RAM) to implement a driver drowsiness detection system. The problem is apparently the processing of image and drowsiness determination is very taxing, so when the camera points at me the FPS reduces to 13 (Normally it is able to handle 30 FPS @ Full HD), whereas when the camera points to anywhere without a face, the FPS rises to 30. I have used threads to optimize the I/O read, but it doesn't seem to improve the FPS.

In the end, I don't mind if the whole video is 13 FPS, but the problem is when writing the video using openCV VideoWriter, I can only initialize the FPS once. So, when the camera FPS is variable but the video write is fixed, the video written will be fluctuating in speed all the time (slow with no face; normal with face).

As you can see from the code, for some reason, initializing the camera framerate to match the Video Write doesn't help. Writing is still not in sync for some reason.

Hope someone can help! Thanks in advance.

`#python drowniness_yawn.py --webcam webcam_index

from scipy.spatial import distance as dist

from imutils.video import VideoStream

from imutils.video.pivideostream import PiVideoStream from imutils.video import FPS from imutils import face_utils from threading import Thread import numpy as np import argparse import imutils import datetime as dt import time import dlib import cv2 import os

def alarm(msg): global alarm_status global alarm_status2 global saying

while alarm_status:
    print('Eyes Closed')
    saying = True

s = 'espeak "'+msg+'"'

os.system(s)

    saying = False

if alarm_status2:
    print('Yawn')
    saying = True

s = 'espeak "' + msg + '"'

os.system(s)

    saying = False

def eye_aspect_ratio(eye): A = dist.euclidean(eye[1], eye[5]) B = dist.euclidean(eye[2], eye[4])

C = dist.euclidean(eye[0], eye[3])

ear = (A + B) / (2.0 * C)

return ear

def final_ear(shape): (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]

leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]

leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)

ear = (leftEAR + rightEAR) / 2.0
return (ear, leftEye, rightEye)

def lip_distance(shape): top_lip = shape[50:53] top_lip = np.concatenate((top_lip, shape[61:64]))

low_lip = shape[56:59]
low_lip = np.concatenate((low_lip, shape[65:68]))

top_mean = np.mean(top_lip, axis=0)
low_mean = np.mean(low_lip, axis=0)

distance = abs(top_mean[1] - low_mean[1])
return distance

ap = argparse.ArgumentParser() ap.add_argument("-w", "--webcam", type=int, default=0, help="index of webcam on system") args = vars(ap.parse_args())

EYE_AR_THRESH = 0.3 EYE_AR_CONSEC_FRAMES = 45 YAWN_THRESH = 40 alarm_status = False alarm_status2 = False saying = False COUNTER = 0

print("-> Loading the predictor and detector...")

detector = dlib.get_frontal_face_detector()

detector = cv2.CascadeClassifier("haarcascade_frontalface_default.xml") #Faster but less accurate predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

print("-> Starting Video Stream")

vs = VideoStream(src=args["webcam"]).start()

vs= PiVideoStream().start() vs.camera.framerate = 13

vs= VideoStream(usePiCamera=True, framerate = 15).start() #For Raspberry Pi

time.sleep(1.0)

fps = FPS().start()

fourcc = cv2.VideoWriter_fourcc(*"mp4v") writer = None (h,w) = (None, None)

while True: timestamp = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") frame = vs.read() frame = imutils.resize(frame, width=400) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

fps.update()
fps.stop()

NEW_FPS = round(fps.fps(),2)
print(NEW_FPS)
if writer is None:
    (height,width,layers)= frame.shape
    writer = cv2.VideoWriter('/home/pi/Desktop/video.mp4', fourcc, 13 , (width,height), True)
cv2.putText(frame, timestamp,(130, 280), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 255), 2)
#rects = detector(gray, 0)
rects = detector.detectMultiScale(gray, scaleFactor=1.1, 
    minNeighbors=5, minSize=(30, 30),
    flags=cv2.CASCADE_SCALE_IMAGE)

#for rect in rects:
for (x, y, w, h) in rects:
    rect = dlib.rectangle(int(x), int(y), int(x + w),int(y + h))

    shape = predictor(gray, rect)
    shape = face_utils.shape_to_np(shape)

    eye = final_ear(shape)
    ear = eye[0]
    leftEye = eye[1]
    rightEye = eye[2]

    distance = lip_distance(shape)

    leftEyeHull = cv2.convexHull(leftEye)
    rightEyeHull = cv2.convexHull(rightEye)
    cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)
    cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)

    lip = shape[48:60]
    cv2.drawContours(frame, [lip], -1, (0, 255, 0), 1)

    if ear < EYE_AR_THRESH:
        COUNTER += 1

        if COUNTER >= EYE_AR_CONSEC_FRAMES:
            if alarm_status == False and saying == False:
                alarm_status = True
                t = Thread(target=alarm, args=('wake up sir',))
                t.deamon = True
                t.start()

            cv2.putText(frame, "DROWSINESS ALERT!", (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

    else:
        COUNTER = 0
        alarm_status = False

    if (distance > YAWN_THRESH):
            cv2.putText(frame, "Yawn Alert", (10, 30),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
            if alarm_status2 == False and saying == False:
                alarm_status2 = True
                t = Thread(target=alarm, args=('take some fresh air sir',))
                t.deamon = True
                t.start()
    else:
        alarm_status2 = False

    cv2.putText(frame, "EAR: {:.2f}".format(ear), (250, 30),
                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
    cv2.putText(frame, "YAWN: {:.2f}".format(distance), (250, 60),
                cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)

writer.write(frame)
cv2.imshow("Frame", frame)
key = cv2.waitKey(1) & 0xFF

if key == ord("q"):
    fps.stop()
    print("Elapsed Time: {:.2f}".format(fps.elapsed()))
    print("approx FPS: {:.2f}".format(fps.fps()))
    break

cv2.destroyAllWindows() vs.stop() writer.release()`