videout: Add support for disabling chroma keying

And make vo_enable_colorkey() return 0 on success and a negative error
code otherwise.

Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

1 files changed, 37 insertions, 12 deletions

diff --git a/videoout.c b/videoout.c
index df580c0..f3d3a22 100644
--- a/videoout.c
+++ b/videoout.c
@@ -136,37 +136,62 @@ int vo_enable_colorkey(struct videoout *vo, unsigned int val)
 	struct v4l2_format fmt;
 	int ret;
 
+	ret = ioctl(vo->dev->fd, VIDIOC_G_FBUF, &framebuffer);
+	if (ret < 0) {
+		perror("VIDIOC_G_FBUF");
+		return -errno;
+	}
+
+	if (!(framebuffer.capability & V4L2_FBUF_CAP_CHROMAKEY))
+		return -ENOTSUP;
+
 	fmt.type = V4L2_BUF_TYPE_VIDEO_OVERLAY;
 	ret = ioctl(vo->dev->fd, VIDIOC_G_FMT, &fmt);
 	if (ret < 0) {
 		perror("VIDIOC_G_FMT\n");
-		return 0;
+		return -errno;
 	}
 
 	fmt.fmt.win.chromakey = val;
 	ret = ioctl(vo->dev->fd, VIDIOC_S_FMT, &fmt);
 	if (ret < 0) {
 		perror("VIDIOC_G_FMT\n");
-		return 0;
+		return -errno;
+	}
+
+	framebuffer.flags |= V4L2_FBUF_FLAG_CHROMAKEY;
+	framebuffer.flags &= ~V4L2_FBUF_FLAG_LOCAL_ALPHA;
+	ret = ioctl(vo->dev->fd, VIDIOC_S_FBUF, &framebuffer);
+	if (ret < 0) {
+		perror("VIDIOC_S_FBUF");
+		return -errno;
 	}
 
+	return 0;
+}
+
+int vo_disable_colorkey(struct videoout *vo)
+{
+	struct v4l2_framebuffer framebuffer;
+	int ret;
+
 	ret = ioctl(vo->dev->fd, VIDIOC_G_FBUF, &framebuffer);
 	if (ret < 0) {
 		perror("VIDIOC_G_FBUF");
-		return 0;
+		return -errno;
 	}
 
-	if (framebuffer.capability & V4L2_FBUF_CAP_CHROMAKEY) {
-		framebuffer.flags |= V4L2_FBUF_FLAG_CHROMAKEY;
-		framebuffer.flags &= ~V4L2_FBUF_FLAG_LOCAL_ALPHA;
-		ret = ioctl(vo->dev->fd, VIDIOC_S_FBUF, &framebuffer);
-		if (ret < 0) {
-			perror("VIDIOC_S_FBUF");
-			return 0;
-		}
+	if (!(framebuffer.capability & V4L2_FBUF_CAP_CHROMAKEY))
+		return 0;
+
+	framebuffer.flags &= ~V4L2_FBUF_FLAG_CHROMAKEY;
+	ret = ioctl(vo->dev->fd, VIDIOC_S_FBUF, &framebuffer);
+	if (ret < 0) {
+		perror("VIDIOC_S_FBUF");
+		return -errno;
 	}
 
-	return 1;
+	return 0;
 }
 
 struct v4l2_buffers_pool *vo_get_pool(struct videoout *vo)