Add 'output render_bit_depth [8|10]' command

This makes it possible to hint to the renderer and backends how many
bits per channel the buffers that the compositor draws windows onto
should have. Renderers and backends may deviate from this if they
do not support the formats with higher bit depth.

include/sway/commands.h

@@ -284,6 +284,7 @@ sway_cmd output_cmd_max_render_time;
 sway_cmd output_cmd_mode;
 sway_cmd output_cmd_modeline;
 sway_cmd output_cmd_position;
+sway_cmd output_cmd_render_bit_depth;
 sway_cmd output_cmd_scale;
 sway_cmd output_cmd_scale_filter;
 sway_cmd output_cmd_subpixel;

include/sway/config.h

@@ -266,6 +266,7 @@ struct output_config {
 	enum wl_output_subpixel subpixel;
 	int max_render_time; // In milliseconds
 	int adaptive_sync;
+	const uint32_t *render_format_order;
 
 	char *background;
 	char *background_option;

sway/commands/output.c

@@ -18,6 +18,7 @@ static const struct cmd_handler output_handlers[] = {
 	{ "modeline", output_cmd_modeline },
 	{ "pos", output_cmd_position },
 	{ "position", output_cmd_position },
+	{ "render_bit_depth", output_cmd_render_bit_depth },
 	{ "res", output_cmd_mode },
 	{ "resolution", output_cmd_mode },
 	{ "scale", output_cmd_scale },

sway/commands/output/render_bit_depth.c

@@ -0,0 +1,40 @@
+#include <drm_fourcc.h>
+#include <strings.h>
+#include "sway/commands.h"
+#include "sway/config.h"
+
+static const uint32_t format_order_8bpc[] = {
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888, 0
+};
+
+static const uint32_t format_order_10bpc[] = {
+	DRM_FORMAT_ABGR2101010, DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_XBGR2101010, DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_ARGB8888, DRM_FORMAT_XRGB8888,
+	0,
+};
+
+struct cmd_results *output_cmd_render_bit_depth(int argc, char **argv) {
+	if (!config->handler_context.output_config) {
+		return cmd_results_new(CMD_FAILURE, "Missing output config");
+	}
+	if (!argc) {
+		return cmd_results_new(CMD_INVALID, "Missing bit depth argument.");
+	}
+
+	if (strcmp(*argv, "8") == 0) {
+		config->handler_context.output_config->render_format_order =
+			format_order_8bpc;
+	} else if (strcmp(*argv, "10") == 0) {
+		config->handler_context.output_config->render_format_order =
+			format_order_10bpc;
+	} else {
+		return cmd_results_new(CMD_INVALID,
+			"Invalid bit depth. Must be either 8 or 10 .");
+	}
+
+	config->handler_context.leftovers.argc = argc - 1;
+	config->handler_context.leftovers.argv = argv + 1;
+	return NULL;
+}
+

sway/config/output.c

@@ -67,6 +67,7 @@ struct output_config *new_output_config(const char *name) {
 	oc->subpixel = WL_OUTPUT_SUBPIXEL_UNKNOWN;
 	oc->max_render_time = -1;
 	oc->adaptive_sync = -1;
+	oc->render_format_order = NULL;
 	return oc;
 }
 
@@ -113,6 +114,9 @@ void merge_output_config(struct output_config *dst, struct output_config *src) {
 	if (src->adaptive_sync != -1) {
 		dst->adaptive_sync = src->adaptive_sync;
 	}
+	if (src->render_format_order != NULL) {
+		dst->render_format_order = src->render_format_order;
+	}
 	if (src->background) {
 		free(dst->background);
 		dst->background = strdup(src->background);
@@ -430,6 +434,20 @@ static void queue_output_config(struct output_config *oc,
 			oc->adaptive_sync);
 		wlr_output_enable_adaptive_sync(wlr_output, oc->adaptive_sync == 1);
 	}
+
+	if (oc && oc->render_format_order != NULL) {
+		char buf[256];
+		size_t start = 0;
+		const uint32_t *format = oc->render_format_order;
+		while (*format && start < sizeof(buf)) {
+			start += snprintf(buf + start, sizeof(buf) - start,
+					  "0x%08x ", *format);
+			format++;
+		}
+		sway_log(SWAY_DEBUG, "Set %s render format preference order to %s",
+			wlr_output->name, buf);
+		wlr_output_set_render_format_preference_order(wlr_output, oc->render_format_order);
+	}
 }
 
 bool apply_output_config(struct output_config *oc, struct sway_output *output) {

sway/meson.build

@@ -188,6 +188,7 @@ sway_sources = files(
 	'commands/output/max_render_time.c',
 	'commands/output/mode.c',
 	'commands/output/position.c',
+	'commands/output/render_bit_depth.c',
 	'commands/output/scale.c',
 	'commands/output/scale_filter.c',
 	'commands/output/subpixel.c',

sway/sway-output.5.scd

@@ -157,6 +157,20 @@ must be separated by one space. For example:
 	adaptive sync can improve latency, but can cause flickering on some
 	hardware.
 
+*output* <name> render_bit_depth 8|10
+	Controls the color channel bit depth at which frames are rendered; the
+	default is 8 bits per channel.
+
+	Setting higher values will not have an effect if hardware and software lack
+	support for such bit depths. Successfully increasing the render bit depth
+	will not necessarily increase the bit depth of the frames sent to a display.
+	An increased render bit depth may provide smoother rendering of gradients,
+	and screenshots which can more precisely store the colors of programs
+	which display high bit depth colors.
+
+	Warning: this can break screenshot/screencast programs which have not been
+	updated to work with higher bit depths.
+
 # SEE ALSO
 
 *sway*(5) *sway-input*(5)