Implement shadow offsets

[famfo]

This fixes #99

Still TODO:

• [ ] ~negative offsets~ - not easy to implement without ugly hacks
• [x] maybe a bit of code cleanup

[WillPower3309]

Good stuff! I'll review this in the evening:)

[lm41]

LGTM

[WillPower3309]

Believe @ErikReider had the solution for the negative offsets, maybe he can paste his diff here

[ErikReider]

Here's a patch that fixes a lot of the present issues :)

diff --git a/include/sway/config.h b/include/sway/config.h
index fd132a9d..ab71efc1 100644
--- a/include/sway/config.h
+++ b/include/sway/config.h
@@ -492,7 +492,7 @@ struct sway_config {
    bool shadows_on_csd_enabled;
    int shadow_blur_sigma;
    float shadow_color[4];
-   float shadow_offset_x, shadow_offset_y;
+   int shadow_offset_x, shadow_offset_y;

    bool blur_enabled;
    bool blur_xray;
diff --git a/include/sway/desktop/fx_renderer/fx_renderer.h b/include/sway/desktop/fx_renderer/fx_renderer.h
index 410e3d94..bfae7b72 100644
--- a/include/sway/desktop/fx_renderer/fx_renderer.h
+++ b/include/sway/desktop/fx_renderer/fx_renderer.h
@@ -207,8 +207,8 @@ void fx_render_border_corner(struct fx_renderer *renderer, const struct wlr_box
        enum corner_location corner_location, int radius, int border_thickness);

 void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *box,
-       const float color[static 4], const float matrix[static 9], int radius,
-       float blur_sigma);
+       const struct wlr_box *inner_box, const float color[static 4],
+       const float matrix[static 9], int corner_radius, float blur_sigma);

 void fx_render_blur(struct fx_renderer *renderer, const float matrix[static 9],
        struct fx_framebuffer **buffer, struct blur_shader *shader, const struct wlr_box *box,
diff --git a/sway/commands/shadow_offset.c b/sway/commands/shadow_offset.c
index e76477ba..070047aa 100644
--- a/sway/commands/shadow_offset.c
+++ b/sway/commands/shadow_offset.c
@@ -12,10 +12,10 @@ struct cmd_results *cmd_shadow_offset(int argc, char **argv) {
    char *err;
    float offset_x = strtof(argv[0], &err);
    float offset_y = strtof(argv[1], &err);
-   if (*err || offset_x < 0.0f || offset_x > 99.9f) {
+   if (*err || offset_x < -99 || offset_x > 99) {
        return cmd_results_new(CMD_INVALID, "x offset float invalid");
    } 
-   if (*err || offset_y < 0.0f || offset_y > 99.9f) {
+   if (*err || offset_y < -99 || offset_y > 99) {
        return cmd_results_new(CMD_INVALID, "y offset float invalid");
    }

diff --git a/sway/config.c b/sway/config.c
index 7be3397e..a2b6eaef 100644
--- a/sway/config.c
+++ b/sway/config.c
@@ -351,8 +351,8 @@ static void config_defaults(struct sway_config *config) {
    config->shadow_enabled = false;
    config->shadows_on_csd_enabled = false;
    config->shadow_blur_sigma = 20.0f;
-   config->shadow_offset_x = 0.0f;
-   config->shadow_offset_y = 0.0f;
+   config->shadow_offset_x = 0;
+   config->shadow_offset_y = 0;
    color_to_rgba(config->shadow_color, 0x0000007F);

    config->blur_enabled = false;
diff --git a/sway/desktop/fx_renderer/fx_renderer.c b/sway/desktop/fx_renderer/fx_renderer.c
index 78953d5c..0a66c67a 100644
--- a/sway/desktop/fx_renderer/fx_renderer.c
+++ b/sway/desktop/fx_renderer/fx_renderer.c
@@ -772,8 +772,8 @@ void fx_render_stencil_mask(struct fx_renderer *renderer, const struct wlr_box *

 // TODO: alpha input arg?
 void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *box,
-       const float color[static 4], const float matrix[static 9],
-       int corner_radius, float blur_sigma) {
+       const struct wlr_box *inner_box, const float color[static 4],
+       const float matrix[static 9], int corner_radius, float blur_sigma) {
    if (box->width == 0 || box->height == 0) {
        return;
    }
@@ -787,17 +787,9 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo

    wlr_matrix_transpose(gl_matrix, gl_matrix);

-   // Init stencil work
-   struct wlr_box inner_box;
-   memcpy(&inner_box, box, sizeof(struct wlr_box));
-   inner_box.x += blur_sigma;
-   inner_box.y += blur_sigma;
-   inner_box.width -= blur_sigma * 2;
-   inner_box.height -= blur_sigma * 2;
-
    fx_renderer_stencil_mask_init();
    // Draw the rounded rect as a mask
-   fx_render_stencil_mask(renderer, &inner_box, matrix, corner_radius);
+   fx_render_stencil_mask(renderer, inner_box, matrix, corner_radius);
    fx_renderer_stencil_mask_close(false);

    // blending will practically always be needed (unless we have a madman
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index c41088ac..ca883144 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -758,8 +758,8 @@ void output_damage_whole_container(struct sway_output *output,

    // Pad the box by 1px, because the width is a double and might be a fraction
    struct wlr_box box = {
-       .x = con->current.x - output->lx - 1 - shadow_sigma,
-       .y = con->current.y - output->ly - 1 - shadow_sigma,
+       .x = con->current.x - output->lx - 1 - shadow_sigma + config->shadow_offset_x,
+       .y = con->current.y - output->ly - 1 - shadow_sigma + config->shadow_offset_y,
        .width = con->current.width + 2 + shadow_sigma * 2,
        .height = con->current.height + 2 + shadow_sigma * 2,
    };
diff --git a/sway/desktop/render.c b/sway/desktop/render.c
index 42a21e52..f6646df3 100644
--- a/sway/desktop/render.c
+++ b/sway/desktop/render.c
@@ -343,22 +343,17 @@ damage_finish:
 // _box.width and .height are expected to be output-buffer-local
 void render_box_shadow(struct sway_output *output, pixman_region32_t *output_damage,
        const struct wlr_box *_box, const float color[static 4], float blur_sigma,
-       float corner_radius, float offset_x, float offset_y) {
+       float corner_radius, int offset_x, int offset_y) {
    struct wlr_output *wlr_output = output->wlr_output;
    struct fx_renderer *renderer = output->renderer;

    struct wlr_box box;
    memcpy(&box, _box, sizeof(struct wlr_box));
-   box.x -= blur_sigma;
-   box.y -= blur_sigma;
+   box.x -= blur_sigma - offset_x;
+   box.y -= blur_sigma - offset_y;
    box.width += 2 * blur_sigma;
    box.height += 2 * blur_sigma;

-   box.x += offset_x;
-   box.y += offset_y;
-   box.width -= offset_x;
-   box.height -= offset_y;
-
    pixman_region32_t damage = create_damage(box, output_damage);

    // don't damage area behind window since we dont render it anyway
@@ -371,6 +366,7 @@ void render_box_shadow(struct sway_output *output, pixman_region32_t *output_dam

    pixman_region32_t inner_damage = create_damage(inner_box, output_damage);
    pixman_region32_subtract(&damage, &damage, &inner_damage);
+   pixman_region32_fini(&inner_damage);

    bool damaged = pixman_region32_not_empty(&damage);
    if (!damaged) {
@@ -381,20 +377,23 @@ void render_box_shadow(struct sway_output *output, pixman_region32_t *output_dam
    wlr_matrix_project_box(matrix, &box, WL_OUTPUT_TRANSFORM_NORMAL, 0,
            wlr_output->transform_matrix);

-   // ensure the box is updated as per the output orientation
-   struct wlr_box transformed_box;
    int width, height;
    wlr_output_transformed_resolution(wlr_output, &width, &height);
-   wlr_box_transform(&transformed_box, &box,
-           wlr_output_transform_invert(wlr_output->transform), width, height);
+   enum wl_output_transform transform = wlr_output_transform_invert(wlr_output->transform);
+   // ensure the shadow_box is updated as per the output orientation
+   struct wlr_box transformed_shadow_box;
+   wlr_box_transform(&transformed_shadow_box, &box, transform, width, height);
+   // ensure the box is updated as per the output orientation
+   struct wlr_box transformed_box;
+   wlr_box_transform(&transformed_box, _box, transform, width, height);

    int nrects;
    pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
    for (int i = 0; i < nrects; ++i) {
        scissor_output(wlr_output, &rects[i]);

-       fx_render_box_shadow(renderer, &transformed_box, color, matrix,
-               corner_radius, blur_sigma);
+       fx_render_box_shadow(renderer, &transformed_shadow_box, &transformed_box,
+               color, matrix, corner_radius, blur_sigma);
    }

 damage_finish:
@@ -494,9 +493,11 @@ static void render_layer_iterator(struct sway_output *output,
    // render shadow
    if (deco_data.shadow && config_should_parameters_shadow()) {
        int corner_radius = deco_data.corner_radius *= output->wlr_output->scale;
+       int offset_x = config->shadow_offset_x * output->wlr_output->scale;
+       int offset_y = config->shadow_offset_y * output->wlr_output->scale;
        scale_box(_box, output->wlr_output->scale);
        render_box_shadow(output, data->damage, _box, config->shadow_color, config->shadow_blur_sigma,
-                   corner_radius, config->shadow_offset_x, config->shadow_offset_y);
+                   corner_radius, offset_x, offset_y);
    }
 }

@@ -891,8 +892,10 @@ static void render_view(struct sway_output *output, pixman_region32_t *damage,
        scale_box(&box, output_scale);
        int scaled_corner_radius = deco_data.corner_radius == 0 ?
                0 : (deco_data.corner_radius + state->border_thickness) * output_scale;
+       int offset_x = config->shadow_offset_x * output->wlr_output->scale;
+       int offset_y = config->shadow_offset_y * output->wlr_output->scale;
        render_box_shadow(output, damage, &box, config->shadow_color, config->shadow_blur_sigma,
-               scaled_corner_radius, config->shadow_offset_x, config->shadow_offset_y);
+               scaled_corner_radius, offset_x, offset_y);
    }

    if (state->border == B_NONE || state->border == B_CSD) {

[ErikReider]

Just realized that the patch switches the variable types to integers which probably isn't optional when multiplying by a fraction (monitor scale) so ignore that part... 😅