From 55bfeb62d8c4dcbd4df0c759d61ad49bdb9d9b49 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 6 Oct 2026 23:11:12 -0700 Subject: [PATCH] Fix Wayland cursor metadata negotiation and visibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accept compositor cursor metadata allocations up to 384 pixels while retaining the 128 pixel wire limit. Preserve cached shapes for position updates, hide invalid cursors, and encode empty sprites as transparent cursors. Validated with Debian 12 ASAN/UBSan and Xvfb integration tests, plus an SPA negotiation reproduction against Mutter’s fixed-size metadata. Assisted-by: gpt-6.1-sol --- guest/linux-tests.c | 17 +++++++++++++---- guest/linux.c | 22 +++++++++++++--------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/guest/linux-tests.c b/guest/linux-tests.c index 66203444f2c40fc795cd37aa54ea792fd13ff53f..e0b55cc069978d6f7e085887aabf737f3bbe9b34 100644 --- a/guest/linux-tests.c +++ b/guest/linux-tests.c @@ -40,10 +40,19 @@ int main(void) { pw_cursor(&buffer); assert(cursor_packet->len == SG_CURSOR_WORDS * 4); cursor->hotspot.x = 1; bitmap->offset = sizeof(*bitmap); pw_cursor(&buffer); assert(pointer[SG_POINTER_VISIBLE]); - cursor->id = 0; pw_cursor(&buffer); assert(pointer[SG_POINTER_VISIBLE]); - cursor->id = 7; bitmap->offset = 0; pw_cursor(&buffer); assert(!pointer[SG_POINTER_VISIBLE]); - cursor->bitmap_offset = 0; pw_cursor(&buffer); assert(!pointer[SG_POINTER_VISIBLE]); - cursor->bitmap_offset = sizeof(*cursor); bitmap->format = 0; pw_cursor(&buffer); assert(!pointer[SG_POINTER_VISIBLE]); + length = cursor_packet->len; + cursor->id = 0; pw_cursor(&buffer); assert(!pointer[SG_POINTER_VISIBLE] && cursor_packet->len == length); + cursor->id = 7; cursor->bitmap_offset = 0; pw_cursor(&buffer); + assert(pointer[SG_POINTER_VISIBLE] && cursor_packet->len == length); + cursor->bitmap_offset = sizeof(*cursor); bitmap->offset = 0; bitmap->format = 0; + pw_cursor(&buffer); + assert(word(cursor_packet->data + SG_CURSOR_WIDTH * 4) == 1); + assert(word(cursor_packet->data + SG_CURSOR_HEIGHT * 4) == 1); + assert(!memcmp(cursor_packet->data + SG_CURSOR_WORDS * 4 + 4, (uint8_t[4]){0}, 4)); + length = cursor_packet->len; cursor->bitmap_offset = 0; pw_cursor(&buffer); + assert(pointer[SG_POINTER_VISIBLE] && cursor_packet->len == length); + cursor->bitmap_offset = sizeof(*cursor); bitmap->offset = sizeof(*bitmap); + pw_cursor(&buffer); assert(cursor_packet->len == length); bitmap->format = SPA_VIDEO_FORMAT_BGRA; bitmap->offset = sizeof(*bitmap); pw_cursor(&buffer); assert(pointer[SG_POINTER_VISIBLE]); uint8_t drain[4096]; while (read(sockets[1], drain, sizeof(drain)) > 0) {} g_byte_array_set_size(cursor_packet, 0); diff --git a/guest/linux.c b/guest/linux.c index fceb80de7b5899087d64e2d67e3e9a49663fe443..dfe263ea5c0f57cb93f3bf8493750945ca004e6b 100644 --- a/guest/linux.c +++ b/guest/linux.c @@ -317,17 +317,19 @@ static void pw_cursor(struct spa_buffer *buffer) { struct spa_meta *meta = spa_buffer_find_meta(buffer, SPA_META_Cursor); if (!meta || meta->size < sizeof(struct spa_meta_cursor)) return; struct spa_meta_cursor cursor; memcpy(&cursor, meta->data, sizeof(cursor)); + position(cursor.position.x, cursor.position.y, cursor.id != 0); if (!cursor.id) return; - position(cursor.position.x, cursor.position.y, pointer[SG_POINTER_VISIBLE]); if (!cursor.bitmap_offset) return; if (cursor.bitmap_offset < sizeof(cursor) || cursor.bitmap_offset > meta->size - sizeof(struct spa_meta_bitmap)) return; struct spa_meta_bitmap bitmap; memcpy(&bitmap, (uint8_t *)meta->data + cursor.bitmap_offset, sizeof(bitmap)); + if (!bitmap.offset || !bitmap.size.width || !bitmap.size.height) { + cursor_shape(1, 1, 0, 0, (uint8_t[4]){0}); return; + } if (!bitmap.format) return; - if (!bitmap.offset) { position(cursor.position.x, cursor.position.y, 0); return; } uint32_t w = bitmap.size.width, h = bitmap.size.height; size_t available = meta->size - cursor.bitmap_offset; - if (!w || !h || w > SG_MAX_CURSOR_SIZE || h > SG_MAX_CURSOR_SIZE || bitmap.stride < (int32_t)w * 4 || bitmap.offset < sizeof(bitmap) || bitmap.offset > available || (size_t)(h - 1) * bitmap.stride + w * 4 > available - bitmap.offset) { - position(cursor.position.x, cursor.position.y, 1); cursor_shape(0, 0, 0, 0, NULL); return; + if (w > SG_MAX_CURSOR_SIZE || h > SG_MAX_CURSOR_SIZE || bitmap.stride < (int32_t)w * 4 || bitmap.offset < sizeof(bitmap) || bitmap.offset > available || (size_t)(h - 1) * bitmap.stride + w * 4 > available - bitmap.offset) { + cursor_shape(0, 0, 0, 0, NULL); return; } unsigned offsets[4]; switch (bitmap.format) { @@ -335,21 +337,19 @@ static void pw_cursor(struct spa_buffer *buffer) { case SPA_VIDEO_FORMAT_BGRA: offsets[0] = 2; offsets[1] = 1; offsets[2] = 0; offsets[3] = 3; break; case SPA_VIDEO_FORMAT_ARGB: offsets[0] = 1; offsets[1] = 2; offsets[2] = 3; offsets[3] = 0; break; case SPA_VIDEO_FORMAT_ABGR: offsets[0] = 3; offsets[1] = 2; offsets[2] = 1; offsets[3] = 0; break; - default: position(cursor.position.x, cursor.position.y, 1); cursor_shape(0, 0, 0, 0, NULL); return; + default: cursor_shape(0, 0, 0, 0, NULL); return; } uint8_t rgba[SG_MAX_CURSOR_SIZE * SG_MAX_CURSOR_SIZE * 4]; const uint8_t *pixels = (uint8_t *)meta->data + cursor.bitmap_offset + bitmap.offset; - int visible = 0; for (uint32_t y = 0; y < h; ++y) for (uint32_t x = 0; x < w; ++x) { const uint8_t *src = pixels + (size_t)y * bitmap.stride + x * 4; uint8_t *dst = rgba + ((size_t)y * w + x) * 4; - unsigned alpha = src[offsets[3]]; dst[3] = (uint8_t)alpha; visible |= alpha != 0; + unsigned alpha = src[offsets[3]]; dst[3] = (uint8_t)alpha; for (unsigned c = 0; c < 3; ++c) { unsigned value = alpha ? (src[offsets[c]] * 255u + alpha / 2) / alpha : 0; dst[c] = (uint8_t)(value > 255 ? 255 : value); } } - position(cursor.position.x, cursor.position.y, visible); cursor_shape(w, h, cursor.hotspot.x, cursor.hotspot.y, rgba); } @@ -401,7 +401,11 @@ static void pw_format(void *unused, uint32_t id, const struct spa_pod *param) { SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd))); params[1] = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Cursor), - SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + SG_MAX_CURSOR_SIZE * SG_MAX_CURSOR_SIZE * 4)); + /* Compositors can require a larger allocation than the transmitted cursor limit. */ + SPA_PARAM_META_size, SPA_POD_CHOICE_RANGE_Int( + sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + SG_MAX_CURSOR_SIZE * SG_MAX_CURSOR_SIZE * 4, + sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + 4, + sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + 384 * 384 * 4)); params[2] = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoTransform), SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_videotransform))); pw_stream_update_params(pw_stream, params, 3); -- 2.54.0