split capture very slow

I've got a project where I'm trying to record high-res h264 encoded video while at the same time do some image processing on a second, downsized rgb stream. I'm capturing the hi-res at the highest full-frame resolution supported by V2 camera, 1640x1232 at 30 fps with a baseline profile, which seems to capture at the 30 fps mark (according to timing metrics in my code). However, I'm opening up a second stream with rgb format on splitter_port 2 with a resize of 640x480 and I'm finding at best, with zero extra processing going on, I get ~15 fps out of that stream. Any ideas as to what's going on or what I'm doing wrong?

Seeing this on both a RPi3 and RPi4 with the V2 camera module, both NoIR and IR versions.

import picamera
import time

class H264Output(object):
    def __init__(self):
        #Gather timing metrics
        self.lastTime = time.time()
        self.smoothedFps = 30.0
        self.counter = 0

    def write(self, buf):
        #This works fine, timing at ~30 fps
        nowTime = time.time()
        dt = (nowTime - self.lastTime)
        self.lastTime = nowTime
        self.smoothedFps += 0.05*((1./dt - self.smoothedFps))

        self.counter += 1
        if(self.counter%30 == 0):
            print("H264 FPS:", "%.2f" % self.smoothedFps)

        return len(buf)

class RGBOutput(object):
    def __init__(self):
        #Gather timing metrics
        self.lastTime = time.time()
        self.smoothedFps = 30.0
        self.counter = 0

    def write(self, buf):
        #This is the issue, only seeing ~15 fps at best
        nowTime = time.time()
        dt = (nowTime - self.lastTime)
        self.lastTime = nowTime
        self.smoothedFps += 0.05*((1./dt - self.smoothedFps))

        self.counter += 1
        if(self.counter%30 == 0):
            print("RGB FPS:", "%.2f" % self.smoothedFps)

        return len(buf)

if __name__ == '__main__':
    with picamera.PiCamera() as camera:
        camera.resolution = (1640, 1232)
        camera.framerate = 30
        camera.rotation = 180
        camera.drc_strength = 'high'
        camera.exposure_mode = 'sports'
        camera.video_stabilization = True

        h264output = H264Output()
        camera.start_recording(
            h264output,
            splitter_port=1,
            format='h264',
            profile='baseline',
            intra_period=30,
            bitrate=0,
            quality=30)

        rgb_output = RGBOutput()
        camera.start_recording(
            rgb_output,
            format='rgb',
            splitter_port=2,
            resize=(640,480))

        #Wait for 60 seconds to see fps results
        time.sleep(60)

        camera.stop_recording(splitter_port=1)
        camera.stop_recording(splitter_port=2)

waveform80 / picamera

split capture very slow #668