MPV shader for looking glass portrait

[jakedowns]

hello, thanks for posting your work on this. i'm curious how one might go about modifying the quiltshader for a looking glass portrait?

here's what i've tried and it doesn't quite work

// mpv glsl shader hook for looking glass PORTRAIT
// Usage sample:
// mpv --screen=1 --fs-screen=1 --fs --glsl-shader=quiltshader.glsl --no-keepaspect *.mp4
// or via input.conf:
// CTRL+1 no-osd change-list glsl-shaders set "~~/shaders/lookingglass.glsl"; set dither no; set scale nearest; set deband false; show-text "LookingGlass: enabled"

//!HOOK OUTPUT
//!DESC Looking Glass Quilt renderer
//!BIND HOOKED

float width = 1536.0f;
float height = 2048.0f;
float dpi = 324.0f;
// Calibration values 
// via: https://jakedowns.github.io/looking-glass-calibration.html
float slope = -7.083540916442871;
float pitch = 52.59267044067383; //{pitch};
float center = 0.8167931437492371;
// const float tilt = -0.12039111681976107; //{tilt};
float tilt = height / (width * slope);
float pitch_adjusted = pitch * width / dpi * cos(atan(1.0f, slope));
float subp = 1.0 / (3*width) * pitch_adjusted; //{subp};

// testing interlacing SBS video
const vec2 tiles = vec2(2,1);

vec2 quilt_map(vec2 pos, float a) {
  // Y major positive direction, X minor negative direction
  vec2 tile = vec2(tiles.x-1,0), dir=vec2(-1,1);
  a = fract(a)*tiles.y;
  tile.y += dir.y*floor(a);
  a = fract(a)*tiles.x;
  tile.x += dir.x*floor(a);
  return (tile+pos)/tiles;
}

vec4 hook() {
  vec4 res;
  float a;
  a = (HOOKED_pos.x + HOOKED_pos.y*tilt)*pitch_adjusted - center;
  res.x = HOOKED_tex(quilt_map(HOOKED_pos, a)).x;
  res.y = HOOKED_tex(quilt_map(HOOKED_pos, a+subp)).y;
  res.z = HOOKED_tex(quilt_map(HOOKED_pos, a+2*subp)).z;
  res.w = 1.0;
  return res;
}

[jakedowns]

i've gotten closer, but it's horizontal bands instead of vertical :G

i tested on shader toy, https://www.shadertoy.com/view/stKfRG and it works there, so it must be something special about how mpv renders things

//!HOOK OUTPUT
//!BIND HOOKED
//!DESC LookingGlass
// ! WIDTH 1536
// ! HEIGHT 2048

const bool DEBUG_COLORS = false;

// this is especially made for looking glass portrait.
// TODO: make it more generic / able to support other variations
const float width = 1536.0f;
const float height = 2048.0f;
const float dpi = 324.0f;

// Calibration values 
// via: https://jakedowns.github.io/looking-glass-calibration.html
const float slope = -7.083540916442871;
const float pitch = 52.59267044067383;
const float tilt = height / (width * slope);
const float center = 0.8167931437492371;

const float pitch_adjusted = pitch * width / dpi * cos(atan(1.0f, slope));

const float subp = 1.0f / (3.0f * width) * pitch_adjusted;
const float repeat = 100.0f/3.0f;

vec4 my_sample(float alpha){
    const vec2 pos = vec2(HOOKED_pos);
    const float halfX = pos.x / 2.0f;

    // return HOOKED_texOff(vec2(-alpha,0.0));

    // if(pos.x < 0.25){
    //  return vec4(0.0,1.0,0.0,1.0);
    // }
    // if(pos.y < 0.25){
    //  return vec4(1.0,0.0,0.0,1.0);
    // }

    if(fract(alpha) < .5){
    // if(mod(fract(alpha)*100.0f,repeat) < repeat*0.5){
    // if(pos.x < 0.5){
        // right eye (right half of SBS);
        // if(DEBUG_COLORS)
        return vec4(0.0,0.0,1.0,1.0);
        return HOOKED_tex(vec2(halfX, pos.y));
    }
    // left eye (left half of SBS)
    // return vec4(0.0);
    if(DEBUG_COLORS)
        return vec4(1.0,0.0,0.0,1.0);
    return HOOKED_tex(vec2(0.5 + halfX, pos.y));
}

vec4 hook(){

    vec4 myColor = vec4(0.0,0.0,0.0,0.1);//HOOKED_tex(HOOKED_pos);

    // float alpha = (HOOKED_pos.x + HOOKED_pos.y * -tilt) * pitch_adjusted - center;
    const float alpha = (HOOKED_pos.x + 1.0-HOOKED_pos.y * slope) * pitch_adjusted - center;
    // This makes a perfect red/cyan filter somehow
    // float alpha = gl_FragCoord.x; // + gl_FragCoord.y;

    // we sample 3 times since the r,g,b subpixels for each "original" pixel needs to be additionally shifted by one extra "subpixel" amount per channel to match the unique sub-pixel layout of the LKGP display
    // myColor = my_sample(alpha);
    myColor.r = my_sample(alpha).r;
    myColor.g = my_sample(alpha + subp).g;
    myColor.b = my_sample(alpha + 2.0f * subp).b;

    return myColor;
}