diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | Makefile | 20 | ||||
| -rw-r--r-- | _static/.keep | 0 | ||||
| -rw-r--r-- | _templates/.keep | 0 | ||||
| -rw-r--r-- | conf.py | 28 | ||||
| -rw-r--r-- | configuring-pipelines.rst | 759 | ||||
| -rw-r--r-- | imx8mp-isp-links-enabled.dot | 17 | ||||
| -rw-r--r-- | imx8mp-isp.dot | 17 | ||||
| -rw-r--r-- | index.rst | 14 | ||||
| -rw-r--r-- | introduction.rst | 117 | ||||
| -rw-r--r-- | mc-v4l2-api.rst | 289 | ||||
| -rw-r--r-- | omap3isp.dot | 36 | ||||
| -rw-r--r-- | scaler-fast.dot | 13 | ||||
| -rw-r--r-- | scaler-hq.dot | 13 | ||||
| -rw-r--r-- | scaler.dot | 28 | ||||
| -rw-r--r-- | subdev.svg | 456 |
16 files changed, 1808 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..69fa449 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +_build/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/_static/.keep b/_static/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/_static/.keep diff --git a/_templates/.keep b/_templates/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/_templates/.keep @@ -0,0 +1,28 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'Media Controller & V4L2 documentation' +copyright = '2022, Laurent Pinchart' +author = 'Laurent Pinchart <laurent.pinchart@ideasonboard.com>' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ['sphinx.ext.graphviz', 'sphinx.ext.imgconverter'] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +graphviz_output_format = 'svg' +pygments_style = 'default' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'bizstyle' +html_static_path = ['_static'] diff --git a/configuring-pipelines.rst b/configuring-pipelines.rst new file mode 100644 index 0000000..440be2a --- /dev/null +++ b/configuring-pipelines.rst @@ -0,0 +1,759 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Configuring Pipelines +===================== + +Allowing fine-grained control of formats comes at the cost of increasing +complexity for applications. Where an application used to only have to set the +format on a video device, it must now set formats on all pads in the pipeline, +and ensure that the result forms a coherent configuration. + +As previously stated, the examples in this section will be based on the NXP +i.MX8MP ISP device: + +.. graphviz:: imx8mp-isp.dot + :caption: Media graph of the NXP i.MX8MP with default configuration + + +Anatomy of a Pipeline +--------------------- + +.. _media-ctl: https://git.linuxtv.org/v4l-utils.git/tree/utils/media-ctl +.. _v4l2-ctl: https://git.linuxtv.org/v4l-utils.git/tree/utils/v4l2-ctl + +The `media-ctl`_ utility can be used to display the formats on all pads in a +graph. Let's start by disecting the output of ``media-ctl -d /dev/media0 -p``: + +.. code-block:: none + :linenos: + :lineno-start: 1 + + $ media-ctl -d /dev/media0 -p + Media controller API version 6.0.0 + + Media device information + ------------------------ + driver rkisp1 + model rkisp1 + serial + bus info platform:rkisp1 + hw revision 0xe + driver version 6.0.0 + +media-ctl first provides summary information, to quickly identify the hardware +device corresponding to the media controller device (``/dev/media0`` here): + +- the driver name +- the device model name +- the device serial number (when available) +- which bus the device sits on +- the device hardware revision (in a device-specific format) +- the driver version (identical to the kernel version) + +The ISP integrated in the NXP i.MX8MP is also found in Rockchip SoCs. They are +both handled by the ``rkisp1`` kernel driver, as shown by the driver and device +model names. + +media-ctl then prints detailed information for all entities in the graph. +Entities are sorted by their numerical ID, which doesn't necessarily follow any +logical order. To improve readability, the entities below have been reordered +in source to sink order, as shown by the line numbers. This corresponds to the +top to bottom order in the graphical representation of the media graph. + +.. code-block:: none + :linenos: + :lineno-start: 73 + + - entity 34: imx290 2-001a (1 pad, 1 link) + type V4L2 subdev subtype Sensor flags 0 + device node name /dev/v4l-subdev3 + pad0: Source + [fmt:SRGGB10_1X10/1920x1080 field:none + crop.bounds:(0,0)/1945x1097 + crop:(12,8)/1920x1080] + -> "csis-32e40000.csi":0 [] + +All entities have a numerical ID and a name. The first entity in pipeline order +has ID ``34`` and is named ``imx290 2-001a``. Names are chosen by drivers but +should follow a set of naming rules. For devices controlled through an I2C bus, +as is the case here, the entity name is made of the device model (``imx290``) +and bus information (``2-001a``) separated by a space. The I2C bus information +is itself made of the bus number and the device address (in hexadecimal), +separated by a dash. + +This particular entity has a total of 1 pad and 1 link (line 73). It is a V4L2 +subdevice whose subtype is a camera sensor (line 74). The subdevice is exposed +to userspace through the ``/dev/v4l-subdev3`` device node (line 75). + +For each pad of the entity media-ctl then displays the pad index and type, the +available links, as well as pad information specific to the entity type. Pad +indices start at 0. The IMX290 camera sensor entity has a single pad, thus at +index 0 and named ``pad0``, whose type is a source pad (line 76). + +Links are displayed on lines starting with a ``->`` (for outbound links, on +source pads) or ``<-`` (for inbound links, on sink pads) sign, followed by the +link target (``"csis-32e40000.csi":0``) and link flags (``[]``). The link +target is a pad of another entity, expressed as the target entity name +surrounded by double quotes, followed by a colon and the target pad index. The +supported link flags (with their numerical values) are: + +ENABLED (0x1) + The link is enabled. In the absence of this flag, the link is disabled. + +IMMUTABLE (0x2) + The link is immutable, its ``ENABLED`` flag cannot be modified. Immutable + links in practice always have the ``ENABLED`` flag set. + +As the list of link flags is empty, this specific link is thus disabled, but +can be enabled as it isn't immutable. The graphical representation displays +this link as a dotted line. + +We will ignore for now the pad information specific to the entity type +displayed on lines 77 to 79, this will be explained further down. + +The next entity, in graph order, is the MIPI CSI-2 receiver that the camera +sensor is attached to: + +.. code-block:: none + :linenos: + :lineno-start: 64 + + - entity 29: csis-32e40000.csi (2 pads, 2 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev2 + pad0: Sink + [fmt:UYVY8_1X16/640x480 field:none colorspace:smpte170m xfer:709 ycbcr:601 quantization:lim-range] + <- "imx290 2-001a":0 [] + pad1: Source + [fmt:UYVY8_1X16/640x480 field:none colorspace:smpte170m xfer:709 ycbcr:601 quantization:lim-range] + -> "rkisp1_isp":0 [ENABLED] + +The CSI-2 receiver is an IP core internal to the SoC, modelled in the kernel as +a platform device. Entity names from platform devices are more free-formed. In +this case, the driver follows the usual recommendation, which uses the platform +device model name (``csis``) and the platform device name (``32e40000.csi``), +separated by a dash. On machines using Device Tree to describe the system (as +is the case on ARM and ARM64 machines), the platform device name is typically +make of the device's bus address (``32e40000``) and device node name (``csi``). + +This entity has two pads, a sink pad and a source pad, with one link each. The +sink pad, ``pad0`` (line 67), is the target of the link originating from the +camera sensor, and is thus without surprise displayed as connected to the +source pad of the camera sensor (line 69). The link flags being a property of +the link, not the pad, they match the link flags displayed in the camera sensor +entity. + +The source pad, ``pad1`` (line 70), is connected to pad 0 of the ``rkisp1_isp`` +entity (line 72). This link is enabled but not immutable, thus displayed as a +thin plain line in the graphical representation. + +The next entity represents the core of the ISP itself: + +.. code-block:: none + :linenos: + :lineno-start: 14 + + - entity 1: rkisp1_isp (4 pads, 4 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev0 + pad0: Sink + [fmt:SRGGB10_1X10/800x600 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range + crop.bounds:(0,0)/800x600 + crop:(0,0)/800x600] + <- "csis-32e40000.csi":1 [ENABLED] + pad1: Sink + [fmt:unknown/0x0 field:none] + <- "rkisp1_params":0 [ENABLED,IMMUTABLE] + pad2: Source + [fmt:YUYV8_2X8/800x600 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/800x600 + crop:(0,0)/800x600] + -> "rkisp1_resizer_mainpath":0 [ENABLED] + pad3: Source + [fmt:unknown/0x0 field:none] + -> "rkisp1_stats":0 [ENABLED,IMMUTABLE] + +Its name doesn't follow the previously described recommendation for entities +corresponding to platform devices, which shows that applications can't rely on +a particular format for entity names in general. This may be considered as a +kernel bug, but is unlikely to be changed as entity name changes may break +existing applications. + +This entity is also a V4L2 subdevice, with four pads, two sink pads (``pad0`` +and ``pad1`` on lines 17 and 22 respectively) and two source pads (``pad2`` and +``pad3`` on lines 25 and 30). Pads 1 and 3 are connected to entities +representing the video device nodes through which ISP parameters are supplied +and ISP statistics are captured. We will ignore them in this example as they +are not strictly required to operate the device. It is however worth noting +that the corresponding links are enabled and immutable, and thus displayed in +the graphical representation as thich plain lines. + +Pad 2 is a source pad that connects the ISP core to the resizer, which is the +next entity in the pipeline. + +.. code-block:: none + :linenos: + :lineno-start: 34 + + - entity 6: rkisp1_resizer_mainpath (2 pads, 2 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev1 + pad0: Sink + [fmt:YUYV8_2X8/800x600 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/800x600 + crop:(0,0)/800x600] + <- "rkisp1_isp":2 [ENABLED] + pad1: Source + [fmt:YUYV8_2X8/800x600 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range] + -> "rkisp1_mainpath":0 [ENABLED,IMMUTABLE] + +The resizer is also a V4L2 subdevice, and has nothing noteworthy at this point +compared to the entities previously described. Its source pad (line 42) is +connected to the last entity in the pipeline: + +.. code-block:: none + :linenos: + :lineno-start: 46 + + - entity 9: rkisp1_mainpath (1 pad, 1 link) + type Node subtype V4L flags 0 + device node name /dev/video0 + pad0: Sink + <- "rkisp1_resizer_mainpath":1 [ENABLED,IMMUTABLE] + +The ``rkisp1_mainpath`` entity, unlike all previous entities, is not a V4L2 +subdevice but a V4L2 video node (line 47). Its device node name is +``/dev/video0`` (line 48), corresponding to the output of the pipeline. This is +the device node from which frames originating from the sensor, processed by the +ISP, and scaled by the resizer are captured. + +Let's try to capture frames in NV16 format with a 800x600 resolution: + +.. code-block:: none + :emphasize-lines: 2 + + $ v4l2-ctl -d /dev/video0 -v pixelformat=NV16,width=800,height=600 --stream-count 10 --stream-mmap + VIDIOC_STREAMON returned -1 (Broken pipe) + +The kernel driver returns the `Broken pipe` error code, which indicates that +the pipeline isn't properly configured. The first possibly culprit to +investigate is the links. + + +Link Setup +---------- + +.. _EBNF metasyntax: https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form + +We have seen that, unless immutable, links can be enabled or disabled. Enabling +or disabling links allows configuring data routing through the graph. For +instance, a device that has multiple camera sensors connected to an ISP, but +can only capture from one of the sensors at a time, could be controlled by +enabling one of the links from the camera sensors to select the source. + +In the default configuration of the device, the link from the camera sensor to +the CSI-2 receiver is disabled. The pipeline is thus missing a source, which +causes the capture to fail with the `Broken pipe` error. To fix this, links +must be setup before starting capture. This is done with the media-ctl utility: + +.. code-block:: none + + $ media-ctl -d /dev/media0 --link '"imx290 2-001a":0 -> "csis-32e40000.csi":0 [1]' + +The ``-link`` option (or its shorter form ``-l``) takes a list of links, +expressed as follows (using a syntax close to the `EBNF metasyntax`_, found in +the output of ``media-ctl --help``): + +.. code-block:: ebnf + + links = link { ',' link } ; + link = pad '->' pad '[' flags ']' ; + pad = entity ':' pad-number ; + entity = entity-number | ( '"' entity-name '"' ) ; + +In the above example, we enable (``[1]``) the link from the source pad of the +camera sensor entity (``'"imx290 2-001a":0``) to the sink pad of the CSI-2 +receiver entity (``"csis-32e40000.csi":0``). Links can also be disabled by +clearing the ENABLED flag (using ``[0]``), and multiple links can be setup with +a single command, although this is discouraged in scripts as the resulting +syntax would be hard to read. + +After running the above command, ``media-ctl -p`` reports the link between the +camera sensor and the CSI-2 receiver as enabled: + +.. code-block:: none + :linenos: + :lineno-start: 73 + :emphasize-lines: 8 + + - entity 34: imx290 2-001a (1 pad, 1 link) + type V4L2 subdev subtype Sensor flags 0 + device node name /dev/v4l-subdev3 + pad0: Source + [fmt:SRGGB10_1X10/1920x1080 field:none + crop.bounds:(0,0)/1945x1097 + crop:(12,8)/1920x1080] + -> "csis-32e40000.csi":0 [ENABLED] + +This is also displayed in the graphical representation, with the corresponding +link now being a thin plain line instead of a dotted line: + +.. graphviz:: imx8mp-isp-links-enabled.dot + :caption: Media graph of the NXP i.MX8MP after enabling links + +.. note:: + + It is recommended, before setting up links in a pipeline, to reset all + non-immutable links to their disabled state with ``media-ctl --reset``. This + helps starting from a known state and, and avoids depending on a previous + configuration of the media graph. In this specific example, the links + between the CSI-2 receiver and the ISP, and between the ISP and the resizer, + would get disabled, and would need to be enabled manually to create a + complete pipeline. + +Let's try to capture frames in NV16 format with a 800x600 resolution again: + +.. code-block:: none + :emphasize-lines: 2 + + $ v4l2-ctl -d /dev/video0 -v pixelformat=NV16,width=800,height=600 --stream-count 10 --stream-mmap + VIDIOC_STREAMON returned -1 (Broken pipe) + +The kernel still returns the `Broken pipe` error, for an entirely different +reason: we haven't taken care of formats in the pipeline. + + +Formats in the Pipeline +----------------------- + +.. _Media Bus Format: https://linuxtv.org/downloads/v4l-dvb-apis/userspace-api/v4l/subdev-formats.html> + +It is time to stop ignoring the pad information specific to the entity type +that is printed by media-ctl. Let's have a look at the camera sensor entity +again: + +.. code-block:: none + :linenos: + :lineno-start: 73 + :emphasize-lines: 5-7 + + - entity 34: imx290 2-001a (1 pad, 1 link) + type V4L2 subdev subtype Sensor flags 0 + device node name /dev/v4l-subdev3 + pad0: Source + [fmt:SRGGB10_1X10/1920x1080 field:none + crop.bounds:(0,0)/1945x1097 + crop:(12,8)/1920x1080] + -> "csis-32e40000.csi":0 [ENABLED] + +For entities corresponding to V4L2 subdevices, media-ctl prints the active +configuration for each pad (lines 68 to 78). The configuration is made of +multiple properties: + +- The media bus format, expressed as + + .. code-block:: ebnf + + media bus format = "fmt:", fcc, "/", size ; + size = width, "x", height ; + + where ``fcc`` is the `media bus format`_ code. Format codes are listed in the + ``linux/media-bus-format.h`` header file of the Linux kernel, and expressed + here as a string without the ``MEDIA_BUS_FMT_`` prefix. + +- The field, for interlaced video: + + .. code-block:: ebnf + + field = "field:", v4l2-field ; + v4l2-field = "any" | "none" | "top" | "bottom" | "interlaced" + | "seq-tb" | "seq-bt" | "alternate" | "interlaced-tb" + | "interlaced-bt" ; + +- The color space, expressed as four properties, reporting the V4L2 color + space, transfer function, YCbCr encoding and quantization range: + + .. code-block:: ebnf + + color space = "colorspace:", v4l2-colorspace, " xfer:", v4l2-xfer-func, + " ycbcr:", v4l2-ycbcr-enc, " quantization:", v4l2-quant ; + v4l2-colorspace = "default" | "smpte170m" | "smpte240m" | "rec709" + | "470m" | "470bg" | "jpeg" | "srgb" | "oprgb" | "bt2020" + | "raw" | "dcip3" ; + v4l2-xfer-func = "default" | "709" | "srgb" | "oprgb" | "smpte240m" + | "smpte2084" | "dcip3" | "none" ; + v4l2-ycbcr-enc = "default" | "601" | "709" | "xv601" | "xv709" | "bt2020" + | "bt2020c" | "smpte240m" ; + v4l2-quant = "default" | "full-range" | "lim-range" ; + +- The compose and crop rectangles: + + .. code-block:: ebnf + + rectangle = "(", left, ",", top, ")", "/", size ; + size = width, "x", height ; + + +In order for a pipeline to be valid, the configuration of pads on both sides of +every link must match. Unless drivers implement custom validation rules, the +default constraint is that the media bus format on both sides must be +identical. For interlaced video, the fields must also be identical, or the +field on the sink side must be equal to ``none`` (in order to support +interlaced sources connected to sinks that only support progressive formats). +The crop and compose rectangles are not taken into account when validating +pipelines, as they are internal to subdevices. + +The default validation is performed by the ``v4l2_subdev_link_validate_default()`` +function in the kernel. If dynamic debugging is enabled (through the +``CONFIG_DYNAMIC_DEBUG`` kernel configuration option), extra messages can be +enabled through debugfs to diagnose the problem more precisely: + +.. code-block:: none + + $ mount -t debugfs none /sys/kernel/debug/ + $ echo 'func v4l2_subdev_link_validate_default +p' > /sys/kernel/debug/dynamic_debug/control + +After trying to capture frames with v4l2-ctl and obtaining the `Broken pipe` +error, the kernel log (obtained with ``dmesg``) provides useful information: + +.. code-block:: none + + rkisp1 32e10000.isp: Validating pad 'rkisp1_isp':0 + rkisp1 32e10000.isp: v4l2_subdev_link_validate_default: width does not match (source 640, sink 800) + rkisp1 32e10000.isp: v4l2_subdev_link_validate_default: height does not match (source 480, sink 600) + rkisp1 32e10000.isp: v4l2_subdev_link_validate_default: media bus code does not match (source 0x0000200f, sink 0x0000300f) + rkisp1 32e10000.isp: v4l2_subdev_link_validate_default: link was "csis-32e40000.csi":1 -> "rkisp1_isp":0 + rkisp1 32e10000.isp: Link 'csis-32e40000.csi':1 -> 'rkisp1_isp':0 failed validation: -32 + +This indicates a mismatch between the source pad of the CSI-2 receiver and the +sink pad of the ISP, both in the resolution and in the format code. The CSI-2 +receiver is configured to output UYVY8_1X16 in 640x480, and the ISP to receive +SRGGB10_1X10 in 800x600, leading to a broken pipeline. + +To fix this, formats must be configuration through the pipeline with a process +referred to as `format propagation`. + + +Format Propagation +------------------ + +Format propagation is a pipeline configuration mechanism where the format is +first configured on the source of the pipeline (the camera sensor in this +case), and then propagated downstream through all subdevices until the video +node at the end of the pipeline. It involves two different types of operations: + +- To ensure that formats on both ends of a link match, when configuring the + format on a source pad of a subdevice, the same format must be set on the + connected sink pad at the other end of the link. This is known as `format + propagation across links`. Subdevices do not automatically propagate formats + across links within the kernel in order to avoid hardcoding use cases in + drivers and to simplify driver implementation. This format propagation is the + responsibility of userspace. + +- Within a subdevice, the kernel ensures that pads configuration stays valid at + all times. When an application configures a format on a subdevice pad, the + driver adjusts parameters that are not valid to the closest supported values. + For instance, resolutions are clampedto the supported range, rounded if the + device has alignment constraints, and unsupported format codes are replaced + with supported values. This is known as `format negotiation` and is similar + to how the kernel handles format setting on V4L2 video devices. + + In addition to the format negotiation at the pad level, the kernel also + ensures that the configurations of all pads within a subdevice remain + coherent with each other. When an application configures the format on a sink + pad, the subdevice driver also adjusts the configuration of the source pads + if the new sink pad configuration would otherwise result in an invalid + subdevice configuration. For instance, the i.MX8MP resizer can scale images + but can't convert between bus format codes, so if an application sets a + format code on the resizer sink pad, the driver will automatically set the + same format code on the resizer source pad. This is known as `format + propagation inside subdevices`, and is the responsibility of the subdevice + drivers. + + Unlike across links where the formats on the source and sink sides of a link + must match, formats on the sink and source pads of a subdevice may be + different. For instance, the i.MX8MP ISP interpolates the raw Bayer data + provided by the camera sensor, and converts it to YUV. The format code on the + source pad of the ISP will thus differ from the sink pad. When userspace + configures the format on the ISP sink pad, the driver will propagate the sink + resolution to the source pad, but will adjust the sink Bayer format code to a + YUV format code on the source pad. + + Format propagation within subdevices ensures that their configuration remains + coherent, but doesn't save applications from having to configure formats on + source pads. Using the resizer example again, the driver will ensure that the + format on the source pad gets adjusted to a value compatible with the format + set on the sink pad, but applications will likely want to then configure the + format on the source pad explicitly in order to change the resolution and + select the scaling ratio. Format propagation within subdevices always operate + `from sink to source`. When userspace sets the format on a source pad, the + driver will adjust it to values compatible with the current configuration of + the sink pads if needed, but will never modify the sink pads configuration by + back-propagating the format from source pad to sink pad. + +Subdevice pad formats can be set with the media-ctl utility, using the +``--set-v4l2`` option (shortened to ``-V``). The syntax is identical to the +format printed by media-ctl with the ``-p`` option. The first step in our +example is to configure the format on the sensor source pad. We use the default +format reported by the media graph (10-bit raw Bayer in 1920x1080), which +corresponds to the sensor native resolution: + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"imx290 2-001a":0 [fmt:SRGGB10_1X10/1920x1080 colorspace:raw xfer:none ycbcr:601 quantization:full-range]' + +Running ``media-ctl -p`` again, the format on the sensor source pad has been +updated with the newly selected color space: + +.. code-block:: none + :linenos: + :lineno-start: 73 + :emphasize-lines: 5 + + - entity 34: imx290 2-001a (1 pad, 1 link) + type V4L2 subdev subtype Sensor flags 0 + device node name /dev/v4l-subdev3 + pad0: Source + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range + crop.bounds:(0,0)/1945x1097 + crop:(12,8)/1920x1080] + -> "csis-32e40000.csi":0 [ENABLED] + +The format is otherwise the same, as we have selected a format code and +resolution identical to the previously configured value. However, this is not +the only change in the media graph. The configuration of the CSI-2 receiver is +now different: + +.. code-block:: none + :linenos: + :lineno-start: 64 + :emphasize-lines: 5,8 + + - entity 29: csis-32e40000.csi (2 pads, 2 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev2 + pad0: Sink + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range] + <- "imx290 2-001a":0 [ENABLED] + pad1: Source + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range] + -> "rkisp1_isp":0 [ENABLED] + +The change in the sink pad format seems to contradict what we have seen before, +as the kernel isn't supposed to propagate formats across links. The reason for +this lies in the media-ctl utility. When setting a format on a source pad, +media-ctl automatically reads back the format from the subdevice (to take into +account possible adjustments applied by the driver) and sets that format on all +sink pads connected through enabled links. + +The source pad format is also modified, due to automatic format propagation by +drivers within subdevices. As the CSI-2 receiver can't transcode or scale, the +source pad format has thus been set to match the sink pad format. + +These two effects simplify the format propagation procedure, as formats on sink +pads don't need to be set manually. Instead, we can read back the format on the +source pad of the next subdevice (the CSI-2 receiver in this case) using +``media-ctl -p``, and set it on the same pad to let media-ctl propagate it to +the next subdevice: + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"csis-32e40000.csi":1 [fmt:SRGGB10_1X10/1920x1080 colorspace:raw xfer:none ycbcr:601 quantization:full-range]' + +We expect the next subdevice, ´´rkisp1_isp``, to output an image in a processed +YUV format with a 1920x1080 resolution. This is not what happens: + +.. code-block:: none + :linenos: + :lineno-start: 14 + :emphasize-lines: 6-7,13 + + - entity 1: rkisp1_isp (4 pads, 4 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev0 + pad0: Sink + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/800x600] + <- "csis-32e40000.csi":1 [ENABLED] + pad1: Sink + [fmt:unknown/0x0 field:none] + <- "rkisp1_params":0 [ENABLED,IMMUTABLE] + pad2: Source + [fmt:YUYV8_2X8/800x600 field:none colorspace:raw xfer:none ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/800x600 + crop:(0,0)/800x600] + -> "rkisp1_resizer_mainpath":0 [ENABLED] + pad3: Source + [fmt:unknown/0x0 field:none] + -> "rkisp1_stats":0 [ENABLED,IMMUTABLE] + +The output format code (``YUYV8_2X8``) is fine, but the output resolution has +been set to 800x600. This is due to the crop capability of the sink pad. While +the crop bounds have been adjusted (line 19), the crop rectangle itself (line +20) hasn't been adjusted as its previous size (``(0,0)/800x600``) is compatible +with the new sink format. We must thus set the crop rectangle manually on the +sink pad: + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"rkisp1_isp":0 [crop:(0,0)/1920x1080]' + +This improves the situation: + +.. code-block:: none + :linenos: + :lineno-start: 14 + :emphasize-lines: 7,13-15 + + - entity 1: rkisp1_isp (4 pads, 4 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev0 + pad0: Sink + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/1920x1080] + <- "csis-32e40000.csi":1 [ENABLED] + pad1: Sink + [fmt:unknown/0x0 field:none] + <- "rkisp1_params":0 [ENABLED,IMMUTABLE] + pad2: Source + [fmt:YUYV8_2X8/800x600 field:none colorspace:raw xfer:none ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/800x600] + -> "rkisp1_resizer_mainpath":0 [ENABLED] + pad3: Source + [fmt:unknown/0x0 field:none] + -> "rkisp1_stats":0 [ENABLED,IMMUTABLE] + +The crop bounds on the source pad (line 27) now cover the full resolution, but +the source crop rectangle (line 28) hasn't been adjusted, for the same reason +as previously. Fixing this is simple: + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"rkisp1_isp":2 [crop:(0,0)/1920x1080]' + +This gives us the configuration we want for the source resolution: + +.. code-block:: none + :linenos: + :lineno-start: 14 + :emphasize-lines: 13,15 + + - entity 1: rkisp1_isp (4 pads, 4 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev0 + pad0: Sink + [fmt:SRGGB10_1X10/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:full-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/1920x1080] + <- "csis-32e40000.csi":1 [ENABLED] + pad1: Sink + [fmt:unknown/0x0 field:none] + <- "rkisp1_params":0 [ENABLED,IMMUTABLE] + pad2: Source + [fmt:YUYV8_2X8/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/1920x1080] + -> "rkisp1_resizer_mainpath":0 [ENABLED] + pad3: Source + [fmt:unknown/0x0 field:none] + -> "rkisp1_stats":0 [ENABLED,IMMUTABLE] + +Finally, we set the format on the source pad to let media-ctl propagate it to +the next subdevice (``rkisp1_resizer_mainpath``): + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"rkisp1_isp":2 [fmt:YUYV8_2X8/1920x1080]' + +The resizer sink pad format is configured correctly: + +.. code-block:: none + :linenos: + :lineno-start: 34 + :emphasize-lines: 5,7,10 + + - entity 6: rkisp1_resizer_mainpath (2 pads, 2 links) + type V4L2 subdev subtype Unknown flags 0 + device node name /dev/v4l-subdev1 + pad0: Sink + [fmt:YUYV8_2X8/1920x1080 field:none colorspace:raw xfer:none ycbcr:601 quantization:lim-range + crop.bounds:(0,0)/1920x1080 + crop:(0,0)/800x600] + <- "rkisp1_isp":2 [ENABLED] + pad1: Source + [fmt:YUYV8_2X8/800x600 field:none colorspace:raw xfer:none ycbcr:601 quantization:lim-range] + -> "rkisp1_mainpath":0 [ENABLED,IMMUTABLE] + +The resizer source pad (line 43) matches the resolution we want to capture +(``800x600``). However, this is not achieved by scaling, but by cropping the +input frame (line 40). This isn't the desired behaviour, so, even if frame +capture would work with this configuration, we must adjust the sink pad crop +rectangle: + +.. code-block:: none + + $ media-ctl -d /dev/media0 -V '"rkisp1_resizer_mainpath":0 [crop:(0,0)/1920x1080]' + +The source pad format is unchanged and still set to ``YUYV8_2X8/800x600``. If +we wanted to scale to a different resolution, we would need to set the format +on the resizer source pad at this point. + +The next entity in the pipeline (``rkisp1_mainpath``) is not a subdevice but a +video device. There is no need to configure pad formats, as formats on video +devices are set through the classic V4L2 API. Care must be taken to configure +the video device with a pixel format compatible with the source pad of the last +subdevice in the pipeline, and an identical resolution. Which pixel formats +qualify as compatible is device-dependent, as explained in the +:ref:`v4l2-subdevice-formats` section. For the i.MX8MP, the DMA engine at the +output of the pipeline can write the YUV 4:2:2 data it receives from the +resizer in the semi-planar NV16 format: + +.. code-block:: none + + $ v4l2-ctl -d /dev/video0 -v pixelformat=NV16,width=800,height=600 --stream-count 10 --stream-mmap + <<<<<<<<<< + +The pipeline is valid, and 10 frames have been captured successfully. + + +Summary +------- + +Pipelines can be inspected with ``media-ctl -p`` and must configured by +userspace before being used. + +The first configuration step covers link setup. Links should be reset with +``media-ctl -r`` to avoid any influence from a previous configuration, and +individual links should then be enabled with ``media-ctl -l``. + +The second configuration step covers format configuration. Formats are set with +``media-ctl -V`` and must be propagated through the pipeline from source to +sink. The combination of automatic format propagation across links in media-ctl +with the automatic format propagation inside subdevices in the kernel +simplifies format propagation. The generic procedure is as follows: + +#. Set the format on the source pad of the first subdevice in the pipeline. +#. The kernel adjusts the format, and media-ctl automatically propagates it + over the link to the connected sink pad of the next subdevice. +#. If the next subdevice has crop or compose rectangleon its sink or source + pads, sets them as necessary, in the following order: + + - Sink crop + - Sink compose + - Source crop + +#. The kernel automatically propagates the format to the source pad of the + subdevice, adjusting it if needed. +#. Retrieve the propagated format on the source pad of the subdevice, modify it + if required (for instance to configure scaling), and set of on the same + source pad to let media-ctl propagate it across the next link. +#. Repeat this procedure from subdevice to subdevice, until the last subdevice + in the pipeline is reached. + +After configuring the pipeline, frames can be captured from the video device at +the pipeline output with the classic V4L2 API, in a pixel format and resolution +that matches the pipeline configuration. + diff --git a/imx8mp-isp-links-enabled.dot b/imx8mp-isp-links-enabled.dot new file mode 100644 index 0000000..46b0afe --- /dev/null +++ b/imx8mp-isp-links-enabled.dot @@ -0,0 +1,17 @@ +digraph board { + size="5" + rankdir=TB + n00000001 [label="{{<port0> 0 | <port1> 1} | rkisp1_isp\n/dev/v4l-subdev0 | {<port2> 2 | <port3> 3}}", shape=Mrecord, style=filled, fillcolor=green] + n00000001:port2 -> n00000006:port0 + n00000001:port3 -> n0000000d [style=bold] + n00000006 [label="{{<port0> 0} | rkisp1_resizer_mainpath\n/dev/v4l-subdev1 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000006:port1 -> n00000009 [style=bold] + n00000009 [label="rkisp1_mainpath\n/dev/video0", shape=box, style=filled, fillcolor=yellow] + n0000000d [label="rkisp1_stats\n/dev/video1", shape=box, style=filled, fillcolor=yellow] + n00000011 [label="rkisp1_params\n/dev/video2", shape=box, style=filled, fillcolor=yellow] + n00000011 -> n00000001:port1 [style=bold] + n0000001d [label="{{<port0> 0} | csis-32e40000.csi\n/dev/v4l-subdev2 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n0000001d:port1 -> n00000001:port0 + n00000022 [label="{{} | imx290 2-001a\n/dev/v4l-subdev3 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green] + n00000022:port0 -> n0000001d:port0 +} diff --git a/imx8mp-isp.dot b/imx8mp-isp.dot new file mode 100644 index 0000000..9fcb3be --- /dev/null +++ b/imx8mp-isp.dot @@ -0,0 +1,17 @@ +digraph board { + size="5" + rankdir=TB + n00000001 [label="{{<port0> 0 | <port1> 1} | rkisp1_isp\n/dev/v4l-subdev0 | {<port2> 2 | <port3> 3}}", shape=Mrecord, style=filled, fillcolor=green] + n00000001:port2 -> n00000006:port0 + n00000001:port3 -> n0000000d [style=bold] + n00000006 [label="{{<port0> 0} | rkisp1_resizer_mainpath\n/dev/v4l-subdev1 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000006:port1 -> n00000009 [style=bold] + n00000009 [label="rkisp1_mainpath\n/dev/video0", shape=box, style=filled, fillcolor=yellow] + n0000000d [label="rkisp1_stats\n/dev/video1", shape=box, style=filled, fillcolor=yellow] + n00000011 [label="rkisp1_params\n/dev/video2", shape=box, style=filled, fillcolor=yellow] + n00000011 -> n00000001:port1 [style=bold] + n0000001d [label="{{<port0> 0} | csis-32e40000.csi\n/dev/v4l-subdev2 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n0000001d:port1 -> n00000001:port0 + n00000022 [label="{{} | imx290 2-001a\n/dev/v4l-subdev3 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green] + n00000022:port0 -> n0000001d:port0 [style=dashed] +} diff --git a/index.rst b/index.rst new file mode 100644 index 0000000..3f12a84 --- /dev/null +++ b/index.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +============================================= +Application usage of media controller devices +============================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + introduction + mc-v4l2-api + configuring-pipelines + diff --git a/introduction.rst b/introduction.rst new file mode 100644 index 0000000..f595070 --- /dev/null +++ b/introduction.rst @@ -0,0 +1,117 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Introduction +============ + +.. note:: + + This document assumes familiarity of the reader with the concepts of the + traditional `V4L2 API`_, excluding the Media Controller extensions. + +.. _V4L2 API: https://linuxtv.org/downloads/v4l-dvb-apis/userspace-api/v4l/v4l2.html + + +The V4L2 history +---------------- + +When the original Video4Linux (V4L) API was created in 1999, the video capture +devices available for Linux were mostly analog TV capture cards and early +webcams (the first widespread USB webcam hit the market about a year later). +From the point of view of the operating system, devices provided streams of +frames ready to be consumed by applications, with a small set of high-level +parameters to control the frame size or modify the image brightness and +contrast. + +Those devices have shaped the API design. As they are fairly monolithic in the +sense that they appear to the operating system as a black box with relative +high-level controls, the V4L API exposed a device to userspace as one video +device node in ``/dev`` with a set of ioctls to handle buffer management, +format selection, stream control and access to parameters. Many mistakes in the +original design were fixed in Video4Linux2 (V4L2), released in 2002. The +original V4L API got deprecated in 2006 and removed from the Linux kernel in +2010. + +.. note:: + + While the V4L2 API supports both video capture and video output, this + document mostly focusses on the former. + +V4L2 covers a wide range of features for both analog and digital video devices, +including tuner and audio control, and has grown over time to accommodate more +features as video capture devices became more complex. It can enumerate the +device capabilities and parameters (supported video and audio inputs, formats, +frame sizes and frame rates, cropping and composing, analog video standards and +digital video timings, and control parameters), expose them to applications +(with get, try and set access, and a negotiation mechanism), manage buffers +(allocate, queue, dequeue and free them, with the ability to share buffers with +other devices for zero copy operation through dmabuf), start and stop video +streams, and report various conditions to applications through an event +mechanism. The V4L2 API has proven its ability to be extended (from the 51 +ioctls present in 2005 in version 2.6.12 of the kernel, 2 were removed and 33 +added as of version 6.0 in 2022), but has and still retains the same monolithic +device model as its predecessor. + + +Modularity with V4L2 subdevices +------------------------------- + +As Linux moved towards the embedded space, the video capture devices started +exposing the multiple hardware components they contained (such as camera +sensors, TV tuners, video encoders and decoders, image signal processors, ...) +to the operating system instead of hiding them in a black box. The same camera +sensor or TV tuner could be used on different systems with different SoCs, +calling for a different architecture inside the kernel that would enable code +reuse. + +In 2008, the Linux media subsystem gained support for a modular model of video +capture drivers. A new V4L2 subdevice object (``struct v4l2_subdev``) was +created to model external hardware components and expose them to the rest of +the kernel through an abstract API (``struct v4l2_subdev_ops``). The main +driver, also called the bridge driver as it controls the components the bridge +external devices with system memory, still creates the video devices (``struct +video_device``) that are exposed to userspace, but translates and delegates the +API calls from applications into calls to the appropriate subdevices. For +instance, when an application sets a V4L2 control on the video device, the +bridge driver will locate the subdevice that implements that control and +forward it the set control call. + +The bridge driver also creates a top-level V4L2 device (``struct v4l2_device``) +and registers it with the V4L2 framework core, to bind together the subdevices +and video devices inside the kernel. This new model provided code reuse and +modularity inside the kernel. + +.. figure:: subdev.svg + + Modularity with V4L2 subdevices + +The new model only addressed in-kernel issues and kept the monolithic V4L2 +userspace API untouched. The relief it brought was short-lived, as development +of the first Linux kernel driver for an image signal processor (the TI OMAP3 +ISP) showed a need for lower-level control of device internals from +applications. + +An ISP is a complex piece of hardware made of multiple processing blocks. Those +blocks are assembled in image processing pipelines, and in many devices data +routing within pipelines is configurable. Inline pipelines connect a video +source (usually a raw Bayer camera sensor) to the ISP and process frames on the +fly, writing fully processed images to memory. Offline pipelines first capture +raw images to memory and process them in memory-to-memory mode. Hybrid +architectures are also possible, and the same device may be configurable in +different modes depending on the use case. With different devices having +different processing blocks and different routing options, applications need to +control data routing within the device. + +Furthermore, similar operations can often be performed in different places in +the pipeline. For instance, both camera sensors and ISPs are able to scale down +images, with the former usually offering lower-quality scaling than the latter, +but with the ability to achieve higher frame rates. Digital gains and colour +gains are also often found in both camera sensors and ISPs. As where to apply a +given image processing operation is dependent on the use case, a bridge driver +can't correctly decide how to delegate V4L2 API calls from applications to the +appropriate V4L2 subdevice without hardcoding and restricting possible use +cases. + +The OMAP3 ISP driver reached the limits of the monolithic V4L2 API. Two years +of development were needed to fix this problem and finally merge, at the +beginning of 2011, the Media Controller API in the kernel. + diff --git a/mc-v4l2-api.rst b/mc-v4l2-api.rst new file mode 100644 index 0000000..5ac0c01 --- /dev/null +++ b/mc-v4l2-api.rst @@ -0,0 +1,289 @@ +.. SPDX-License-Identifier: CC-BY-SA-4.0 + +Media Controller and V4L2 Subdevice APIs +======================================== + +The term `Media Controller` usually covers two distinct APIs: + +- The Media Controller (MC) API itself, whose task it is to expose the internal + topology of the device to applications. + +- The V4L2 subdevice userspace API, which exposes low-level control of + individual subdevices to applications. + +Collectively, and in collaboration with the V4L2 API, these offer the features +needed by applications to control complex video capture devices. + + +The Media Controller API +------------------------ + +.. _media-ctl: https://git.linuxtv.org/v4l-utils.git/tree/utils/media-ctl + +The Media Controller kernel framework and userspace API model devices as a +directed acyclic graph of `entities`. Each entity represents a hardware block, +which can be an external on-board component, an IP core in the SoC, or a piece +of either of those. The API doesn't precisely define how a device should be +split in entities. Individual drivers decide on the exact model they want to +expose, to allow fine-grained control of the hardware blocks while minimizing +the number of entities to avoid unnecessary complexity. + +Entities include `pads`, which model input and output ports through which +entities receive or produce data. Data inputs are called `sinks`, and data +outputs `sources`. The data flow through the graph is modelled by `links` that +connect sources to sinks. Each link connects one source pad of an entity to a +sink pad of another entity. Cycles in the graph are not allowed. + +.. note:: + + The Media Controller API is not limited to video capture devices and has + been designed to model any type of data flow in a media device. This + includes, for instance, audio and display devices. However, as of version + 6.0 of the kernel, the API is only used in the Linux media subsystem, by + V4L2 and DVB drivers, and hasn't made its way to the ALSA and DRM/KMS + subsystems. + +When used in a V4L2 driver, an entity models either a video device (``struct +video_device``) or a subdevice (``struct v4l2_subdevice``). For video capture +devices, subdevices represent video sources (camera sensors, input connectors, +...) or processing elements, and video devices represent the connection to +system memory at the end of a pipeline (typically a DMA engine, but it can also +be a USB connection for USB webcams). The entity type is exposed to +applications as an entity `function`, for instance + +- ``MEDIA_ENT_F_CAM_SENSOR`` for a camera sensor +- ``MEDIA_ENT_F_PROC_VIDEO_SCALER`` for a video scaler +- ``MEDIA_ENT_F_IO_V4L`` for a connection to system memory through a V4L2 video + device + +The kernel media controller device (``struct media_device``) is exposed to +userspace through a media device node, typically named ``/dev/media[0-9]+``. +The `media-ctl`_ tool can query the topology of a Media Controller device and +display it in either plain text (``--print-topology`` or ``-p``) or `DOT format +<https://graphviz.org/doc/info/lang.html>`_ (``--print-dot``). + +.. code-block:: sh + + $ media-ctl -d /dev/media0 --print-dot | dot -Tsvg > omap3isp.svg + +The following graph represents the TI OMAP3 ISP, with entities corresponding to +subdevices in green and entities corresponding to video devices in yellow. + +.. graphviz:: omap3isp.dot + :caption: Media graph of the TI OMAP3 ISP + +The ``mt9p031 2-0048`` on the top row is a camera sensor, all other entities +are internal to the OMAP3 SoC and part of the ISP. + +Entities, their pads, and the links are intrinsic properties of the device. +They are created by the driver at initialization time to model the hardware +topology. Unless parts of the device is hot-pluggable, no entities or links are +created or removed after initialization. Only their properties can be modified +by applications. + +Data flow routing is controlled by enabling or disabling links, using the +``MEDIA_LNK_FL_ENABLED`` link flag. Links that model immutable connections at +the hardware level are displayed as a thick plain line in the media graph. They +have the ``MEDIA_LNK_FL_IMMUTABLE`` and ``MEDIA_LNK_FL_ENABLED`` flags set and +can't be modified. Links that model configurable routing options can be +controlled, and are displayed as a dotted line if they are disabled or as thin +plain line if they are enabled. + +As the Media Controller API can model any type of data flow, it doesn't expose +any property specific to a particular device type, such as, for instance, pixel +formats or frame rates. This is left to other, device-specific APIs + + +The V4L2 Subdevice Userspace API +-------------------------------- + +.. _V4L2 Subdevice Userspace API: https://linuxtv.org/downloads/v4l-dvb-apis/userspace-api/v4l/dev-subdev.html +.. _V4L2 controls ioctls: https://linuxtv.org/downloads/v4l-dvb-apis/userspace-api/v4l/vidioc-g-ext-ctrls.html +.. _v4l2-ctl: https://git.linuxtv.org/v4l-utils.git/tree/utils/v4l2-ctl + +The `V4L2 Subdevice Userspace API`_ (often shortened to just V4L2 Subdevice API +when this doesn't cause any ambiguity with the in-kernel V4L2 subdevice +operations) has been developed along the Media Controller API to expose to +applications the properties of entities corresponding to V4L2 subdevices. It +allows accessing V4L2 controls directly on a subdevice, as well as formats and +selection rectangles on the subdevice pads. + +Subdevices are exposed to userspace through V4L2 subdevice nodes, typically +named ``/dev/v4l-subdev[0-9]+``. They are controlled using ioctls in a similar +fashion as the V4L2 video devices. The `v4l2-ctl`_ tool supports a wide range +of subdevice-specific options to access subdevices from the command line (see +``v4l2-ctl --help-subdev`` for a detailed list). + +The rest of this document will use the NXP i.MX8MP ISP as an example. Its media +graph is as follows: + +.. graphviz:: imx8mp-isp.dot + :caption: Media graph of the NXP i.MX8MP + +It contains the following V4L2 subdevices: + +- A raw camera sensor (``imx290 2-001a``), with a single source pad connected + to the SoC through a MIPI CSI-2 link. +- A MIPI CSI-2 receiver (``csis-32e40000.csi``), internal to the SoC, that + receives data from the sensor on its sink pad and provides it to the ISP on + its source pad. +- An ISP (``rkisp1_isp``), with two sink pads that receive image data and + processing parameters (0 and 1 respectively) and two source pads that output + image data and statistcis (2 and 3 respectively). +- A scaler (``rkisp1_resizer_mainpath``) that can scale the frames up or down. + +It also contains the following video devices: + +- A capture device that writes video frames to memory (``rkisp1_mainpath``). +- A capture device that writes statistics to memory (``rkisp1_stats``). +- An output device that reads ISP parameters from memory (``rkisp1_params``). + + +V4L2 Subdevice Controls +~~~~~~~~~~~~~~~~~~~~~~~ + +Subdevice controls are accessed using the `V4L2 controls ioctls`_ in exactly +the same way as for video device, except that the ioctls should be issued on +the subdevice node. Tools that access controls on video devices can usually be +used unmodified on subdevices. For instance, to list the controls supported by +the IMX290 camera sensor subdevice, + +.. code-block:: none + + $ v4l2-ctl -d /dev/v4l-subdev3 -l + + User Controls + + exposure 0x00980911 (int) : min=1 max=1123 step=1 default=1123 value=1123 + + Camera Controls + + camera_orientation 0x009a0922 (menu) : min=0 max=2 default=0 value=0 (Front) flags=read-only + camera_sensor_rotation 0x009a0923 (int) : min=0 max=0 step=1 default=0 value=0 flags=read-only + + Image Source Controls + + vertical_blanking 0x009e0901 (int) : min=45 max=45 step=1 default=45 value=45 flags=read-only + horizontal_blanking 0x009e0902 (int) : min=280 max=280 step=1 default=280 value=280 flags=read-only + analogue_gain 0x009e0903 (int) : min=0 max=240 step=1 default=0 value=0 + + Image Processing Controls + + link_frequency 0x009f0901 (intmenu): min=0 max=1 default=0 value=0 (222750000 0xd46e530) flags=read-only + pixel_rate 0x009f0902 (int64) : min=1 max=2147483647 step=1 default=178200000 value=178200000 flags=read-only + test_pattern 0x009f0903 (menu) : min=0 max=7 default=0 value=0 (Disabled) + +By accessing controls on subdevices, applications can control the behaviour of +each subdevice independently. If multiple subdevices in the graph implement the +same control (such as a digital gain), those controls can be set individually. +This wouldn't be possible using with the traditional V4L2 API on video devices, +as the identical controls from two different subdevices would conflict. + + +.. _v4l2-subdevice-formats: + +V4L2 Subdevice Formats +~~~~~~~~~~~~~~~~~~~~~~ + +Where video devices expose only the format of the frames being captured to +memory, subdevices allow fine-grained configuration of formats on every pad in +the pipeline. This enables setting up pipelines with different internal +configurations to match precise use cases. To understand why this is needed, +let's consider the following simplified example, where a 12MP camera sensor +(IMX477) is connected to an SoC that includes an ISP and a scaler. + +.. graphviz:: scaler.dot + :caption: Scaling pipeline + +All three components can affect the image size: + +- The camera sensor can subsample the image through mechanisms such as binning + and skipping. +- The ISP can subsample the image horizontally through averaging. +- The scaler uses a polyphase filter for high quality scaling. + +All these components can further crop the image if desired. + +Different use cases will call for cropping and resizing the image in different +ways through the pipeline. Let's assume that, in all cases, we want to capture +1.5MP images from the 12MP native sensor resolution, When frame rate is more +important than quality, the sensor will typically subsample the image to comply +with the bandwidth limitations of the ISP. As the subsampling factor is +restricted to powers of two, the scaler is further used to achieve the exact +desired size. + +.. graphviz:: scaler-fast.dot + :caption: Fast scaling + +On the other hand, when capturing still images, the full image should be +processed through the pipeline and resized at the very end using the higher +quality scaler. + +.. graphviz:: scaler-hq.dot + :caption: High quality scaling + +Using the traditional V4L2 API on video nodes, the bridge driver configures the +internal pipeline based on the desired capture format. As the use cases above +produce the same format at the output of the pipeline, the bridge driver won't +be able to differentiate between them and configure the pipeline appropriately +for each use case. To solve this problem, the V4L2 subdevice userspace API let +applications access formats on pads directly. + +Formats on subdevice pads are called `media bus formats`. They are described by +the ``v4l2_mbus_framefmt`` structure: + +.. code-block:: c + + struct v4l2_mbus_framefmt { + __u32 width; + __u32 height; + __u32 code; + __u32 field; + __u32 colorspace; + union { + __u16 ycbcr_enc; + __u16 hsv_enc; + }; + __u16 quantization; + __u16 xfer_func; + __u16 flags; + __u16 reserved[10]; + }; + +Unlike the pixel formats used on video devices, which describe how image data +is stored in memory (using the ``v4l2_pix_format`` and +``v4l2_pix_format_mplane`` structures), media bus formats describe how image +data is transmitted on buses between subdevices. The ``bytesperline`` and +``sizeimage`` fields of the pixel format are thus not found in the media bus +formats, as they refer to memory sizes. + +This difference between the two concepts causes a second difference between the +media bus and pixel format structures. The FourCC values used to described +pixel formats are not applicable to bus formats, as they also describe data +organization in memory. Media bus formats instead use `format codes` that +describe how individual bits are organized and transferred on a bus. The format +codes are 32-bit numerical values defined by the ``MEDIA_BUS_FMT_*`` macros and +are documented in the `Media Bus Formats`_ section of the V4L2 API +documentation. + +.. _Media Bus Formats: https://linuxtv.org/downloads/v4l-dvb-apis/userspace-api/v4l/subdev-formats.html> + +.. note:: + + In the remaining of this document, the terms `media bus format`, `bus + format` or `format`, when applying to subdevice pads, refers to the + combination of all fields of the ``v4l2_mbus_framefmt`` structure. To refer + to the media bus format code specifically, the terms `media bus code`, + `format code` or `code` will be used. + +In general, there is no 1:1 universal mapping between pixel formats and media +bus formats. To understand this, let's consider the +``MEDIA_BUS_FMT_UYVY8_1X16`` media bus code that describes on common way to +transmit YUV 4:2:2 data on a 16-bit parallel bus. When the image data reaches +the DMA engine at the end of the pipeline and is written to memory, it can be +rearranged in different ways, producing for instance the ``V4L2_PIX_FMT_UYVY`` +packed pixel format that seem to be a direct match, but also the semi-planar +``V4L2_PIX_FMT_NV16`` format by writing the luma and chroma data to separate +memory planes. How a media bus code is translated to pixel formats depends on +the capabilities of the DMA engine, and is thus device-specific. + diff --git a/omap3isp.dot b/omap3isp.dot new file mode 100644 index 0000000..daa3b41 --- /dev/null +++ b/omap3isp.dot @@ -0,0 +1,36 @@ +digraph board { + rankdir=TB + n00000001 [label="{{<port0> 0} | OMAP3 ISP CCP2\n/dev/v4l-subdev0 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000001:port1 -> n00000005:port0 [style=dashed] + n00000002 [label="OMAP3 ISP CCP2 input\n/dev/video2", shape=box, style=filled, fillcolor=yellow] + n00000002 -> n00000001:port0 [style=dashed] + n00000003 [label="{{<port0> 0} | OMAP3 ISP CSI2a\n/dev/v4l-subdev1 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000003:port1 -> n00000004 [style=dashed] + n00000003:port1 -> n00000005:port0 [style=dashed] + n00000004 [label="OMAP3 ISP CSI2a output\n/dev/video3", shape=box, style=filled, fillcolor=yellow] + n00000005 [label="{{<port0> 0} | OMAP3 ISP CCDC\n/dev/v4l-subdev2 | {<port1> 1 | <port2> 2}}", shape=Mrecord, style=filled, fillcolor=green] + n00000005:port1 -> n00000006 + n00000005:port2 -> n00000007:port0 [style=dashed] + n00000005:port1 -> n0000000a:port0 [style=dashed] + n00000005:port2 -> n0000000d:port0 [style=bold] + n00000005:port2 -> n0000000e:port0 [style=bold] + n00000005:port2 -> n0000000f:port0 [style=bold] + n00000006 [label="OMAP3 ISP CCDC output\n/dev/video4", shape=box, style=filled, fillcolor=yellow] + n00000007 [label="{{<port0> 0} | OMAP3 ISP preview\n/dev/v4l-subdev3 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n00000007:port1 -> n00000009 [style=dashed] + n00000007:port1 -> n0000000a:port0 + n00000008 [label="OMAP3 ISP preview input\n/dev/video5", shape=box, style=filled, fillcolor=yellow] + n00000008 -> n00000007:port0 + n00000009 [label="OMAP3 ISP preview output\n/dev/video6", shape=box, style=filled, fillcolor=yellow] + n0000000a [label="{{<port0> 0} | OMAP3 ISP resizer\n/dev/v4l-subdev4 | {<port1> 1}}", shape=Mrecord, style=filled, fillcolor=green] + n0000000a:port1 -> n0000000c + n0000000b [label="OMAP3 ISP resizer input\n/dev/video7", shape=box, style=filled, fillcolor=yellow] + n0000000b -> n0000000a:port0 [style=dashed] + n0000000c [label="OMAP3 ISP resizer output\n/dev/video8", shape=box, style=filled, fillcolor=yellow] + n0000000d [label="{{<port0> 0} | OMAP3 ISP AEWB\n/dev/v4l-subdev5 | {}}", shape=Mrecord, style=filled, fillcolor=green] + n0000000e [label="{{<port0> 0} | OMAP3 ISP AF\n/dev/v4l-subdev6 | {}}", shape=Mrecord, style=filled, fillcolor=green] + n0000000f [label="{{<port0> 0} | OMAP3 ISP histogram\n/dev/v4l-subdev7 | {}}", shape=Mrecord, style=filled, fillcolor=green] + n00000010 [label="{{} | mt9p031 2-0048\n/dev/v4l-subdev8 | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green] + n00000010:port0 -> n00000005:port0 +} + diff --git a/scaler-fast.dot b/scaler-fast.dot new file mode 100644 index 0000000..4e5b642 --- /dev/null +++ b/scaler-fast.dot @@ -0,0 +1,13 @@ +digraph board { + size="6" + rankdir=LR + + sensor [label="{ | IMX477\nsensor | <port0> 0}", shape=Mrecord, style=filled, fillcolor=green] + isp [label="{<port0> 0 | ISP | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + scaler [label="{<port0> 0 | scaler | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + capture [label="DMA", shape=box, style=filled, fillcolor=yellow] + + sensor:port0 -> isp:port0 [labelangle=70, labeldistance=4, headlabel="4056x3040", style=bold] + isp:port1 -> scaler:port0 [labelangle=70, labeldistance=4, headlabel="2028x1520", style=bold] + scaler:port1 -> capture [labelangle=70, labeldistance=4, headlabel="1440x1080", style=bold] +} diff --git a/scaler-hq.dot b/scaler-hq.dot new file mode 100644 index 0000000..cfc0dd3 --- /dev/null +++ b/scaler-hq.dot @@ -0,0 +1,13 @@ +digraph board { + size="6" + rankdir=LR + + sensor [label="{ | IMX477\nsensor | <port0> 0}", shape=Mrecord, style=filled, fillcolor=green] + isp [label="{<port0> 0 | ISP | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + scaler [label="{<port0> 0 | scaler | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + capture [label="DMA", shape=box, style=filled, fillcolor=yellow] + + sensor:port0 -> isp:port0 [labelangle=70, labeldistance=4, headlabel="4056x3040", style=bold] + isp:port1 -> scaler:port0 [labelangle=70, labeldistance=4, headlabel="4056x3040", style=bold] + scaler:port1 -> capture [labelangle=70, labeldistance=4, headlabel="1440x1080", style=bold] +} diff --git a/scaler.dot b/scaler.dot new file mode 100644 index 0000000..3cb2743 --- /dev/null +++ b/scaler.dot @@ -0,0 +1,28 @@ +digraph board { + size="6" + rankdir=LR + + { + rank=same + sensor [label="{ | IMX477\nsensor | <port0> 0}", shape=Mrecord, style=filled, fillcolor=green] + sensor_label [label="Binning &\nskipping", shape=plaintext] + } + + { + rank=same + isp [label="{<port0> 0 | ISP | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + isp_label [label="Horizontal\naveraging", shape=plaintext] + } + + { + rank=same + scaler [label="{<port0> 0 | scaler | <port1> 1}", shape=Mrecord, style=filled, fillcolor=green] + scaler_label [label="Polyphase\nfilter", shape=plaintext] + } + + capture [label="DMA", shape=box, style=filled, fillcolor=yellow] + + sensor:port0 -> isp:port0 [style=bold] + isp:port1 -> scaler:port0 [style=bold] + scaler:port1 -> capture [style=bold] +} diff --git a/subdev.svg b/subdev.svg new file mode 100644 index 0000000..82a25e8 --- /dev/null +++ b/subdev.svg @@ -0,0 +1,456 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + width="72mm" + height="72mm" + viewBox="0 0 72 72" + version="1.1" + id="svg5" + inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)" + sodipodi:docname="subdev.svg" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns="http://www.w3.org/2000/svg" + xmlns:svg="http://www.w3.org/2000/svg"> + <sodipodi:namedview + id="namedview7" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageshadow="2" + inkscape:pageopacity="0.0" + inkscape:pagecheckerboard="0" + inkscape:document-units="mm" + showgrid="true" + inkscape:zoom="5.6568543" + inkscape:cx="295.12869" + inkscape:cy="111.28093" + inkscape:window-width="3840" + inkscape:window-height="2101" + inkscape:window-x="0" + inkscape:window-y="0" + inkscape:window-maximized="1" + inkscape:current-layer="layer1" + width="296mm"> + <inkscape:grid + type="xygrid" + id="grid58406" /> + </sodipodi:namedview> + <defs + id="defs2"> + <marker + style="overflow:visible" + id="Arrow2Lend" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow2Lend" + inkscape:isstock="true"> + <path + transform="matrix(-1.1,0,0,-1.1,-1.1,0)" + d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z" + style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round" + id="path67267" /> + </marker> + <marker + style="overflow:visible" + id="Arrow1Lend" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Lend" + inkscape:isstock="true"> + <path + transform="matrix(-0.8,0,0,-0.8,-10,0)" + style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path67249" /> + </marker> + <marker + style="overflow:visible" + id="Arrow1Lstart" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow1Lstart" + inkscape:isstock="true"> + <path + transform="matrix(0.8,0,0,0.8,10,0)" + style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt" + d="M 0,0 5,-5 -12.5,0 5,5 Z" + id="path67246" /> + </marker> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-8" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-8-7" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-0" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-0-9" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-0-9-7" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-7" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-7-6" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-0-9-7-3" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <inkscape:path-effect + effect="fillet_chamfer" + id="path-effect1277-0-9-7-5" + is_visible="true" + lpeversion="1" + satellites_param="F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1 @ F,0,0,1,0,1.3229167,0,1" + unit="px" + method="auto" + mode="F" + radius="5" + chamfer_steps="1" + flexible="false" + use_knot_distance="true" + apply_no_radius="true" + apply_with_radius="true" + only_selected="false" + hide_knots="false" /> + <marker + style="overflow:visible" + id="Arrow2Lend-4" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow2Lend" + inkscape:isstock="true"> + <path + transform="matrix(-1.1,0,0,-1.1,-1.1,0)" + d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z" + style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round" + id="path67267-8" /> + </marker> + <marker + style="overflow:visible" + id="Arrow2Lend-4-2" + refX="0" + refY="0" + orient="auto" + inkscape:stockid="Arrow2Lend" + inkscape:isstock="true"> + <path + transform="matrix(-1.1,0,0,-1.1,-1.1,0)" + d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z" + style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:0.625;stroke-linejoin:round" + id="path67267-8-9" /> + </marker> + </defs> + <g + inkscape:label="Layer 1" + inkscape:groupmode="layer" + id="layer1"> + <path + style="fill:none;stroke:#008080;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171" + width="26.458332" + height="18.520832" + x="1.3229166" + y="1.3229166" + inkscape:path-effect="#path-effect1277" + d="M 2.6458333,1.3229166 H 26.458332 a 1.3229167,1.3229167 45 0 1 1.322917,1.3229167 V 18.520832 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322917 H 2.6458333 A 1.3229167,1.3229167 45 0 1 1.3229166,18.520832 l 0,-15.8749987 A 1.3229167,1.3229167 135 0 1 2.6458333,1.3229166 Z" + sodipodi:type="rect" /> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="14.455621" + y="9.196681" + id="text18204"><tspan + sodipodi:role="line" + id="tspan18202" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.455621" + y="9.196681">MT9M001</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.45562" + y="15.370294" + id="tspan18206">Sensor</tspan></text> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="56.788952" + y="9.196681" + id="text18204-15"><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="9.196681" + id="tspan18206-4">PXA270</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="15.370294" + id="tspan61445">SoC</tspan></text> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="56.788952" + y="34.3321" + id="text18204-15-9"><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="34.3321" + id="tspan61445-7">EM28xx</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="40.505711" + id="tspan64644">USB</tspan></text> + <path + style="fill:none;stroke:#800040;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171-6-0-1" + width="26.458332" + height="18.520836" + x="43.65625" + y="51.593746" + inkscape:path-effect="#path-effect1277-0-9-7" + sodipodi:type="rect" + d="m 44.979167,51.593746 h 23.812498 a 1.3229167,1.3229167 45 0 1 1.322917,1.322917 v 15.875002 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322917 H 44.979167 A 1.3229167,1.3229167 45 0 1 43.65625,68.791665 V 52.916663 a 1.3229167,1.3229167 135 0 1 1.322917,-1.322917 z" /> + <path + style="fill:none;stroke:#800040;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171-6-0-1-6" + width="26.458332" + height="18.520836" + x="43.65625" + y="26.45833" + inkscape:path-effect="#path-effect1277-0-9-7-3" + sodipodi:type="rect" + d="m 44.979167,26.45833 h 23.812498 a 1.3229167,1.3229167 45 0 1 1.322917,1.322917 v 15.875002 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322917 H 44.979167 A 1.3229167,1.3229167 45 0 1 43.65625,43.656249 V 27.781247 a 1.3229167,1.3229167 135 0 1 1.322917,-1.322917 z" /> + <path + style="fill:none;stroke:#800040;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171-6-0-1-63" + width="26.458332" + height="18.520836" + x="43.65625" + y="1.3229147" + inkscape:path-effect="#path-effect1277-0-9-7-5" + sodipodi:type="rect" + d="m 44.979167,1.3229147 23.812498,0 a 1.3229167,1.3229167 45 0 1 1.322917,1.3229167 V 18.520834 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322917 H 44.979167 A 1.3229167,1.3229167 45 0 1 43.65625,18.520834 V 2.6458314 a 1.3229167,1.3229167 135 0 1 1.322917,-1.3229167 z" /> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="56.788952" + y="59.46751" + id="text18204-15-9-1"><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="59.46751" + id="tspan61445-7-5">EM28xx</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#800040;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="56.788952" + y="65.641121" + id="tspan64644-9">USB</tspan></text> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="14.455623" + y="34.3321" + id="text18204-1"><tspan + sodipodi:role="line" + id="tspan18202-7" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.455623" + y="34.3321">MT9M001</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.455622" + y="40.505711" + id="tspan18206-2">Sensor</tspan></text> + <text + xml:space="preserve" + style="font-style:normal;font-weight:normal;font-size:4.93889px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none" + x="14.455623" + y="59.46751" + id="text18204-1-2"><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.455623" + y="59.46751" + id="tspan18206-2-1">SAA7114</tspan><tspan + sodipodi:role="line" + style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:4.93889px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#008080;fill-opacity:1;stroke:none;stroke-width:0.265;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + x="14.455623" + y="65.641121" + id="tspan58688">Decoder</tspan></text> + <path + style="fill:none;stroke:#008080;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171-7" + width="26.458332" + height="18.520832" + x="1.3229166" + y="26.45833" + inkscape:path-effect="#path-effect1277-7" + sodipodi:type="rect" + d="M 2.6458333,26.45833 H 26.458332 a 1.3229167,1.3229167 45 0 1 1.322917,1.322917 v 15.874999 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322916 l -23.8124987,0 A 1.3229167,1.3229167 45 0 1 1.3229166,43.656246 l 0,-15.874999 A 1.3229167,1.3229167 135 0 1 2.6458333,26.45833 Z" /> + <path + style="fill:none;stroke:#008080;stroke-width:0.665;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1" + id="rect1171-7-7" + width="26.458332" + height="18.520832" + x="1.3229166" + y="51.593746" + inkscape:path-effect="#path-effect1277-7-6" + sodipodi:type="rect" + d="M 2.6458333,51.593746 H 26.458332 a 1.3229167,1.3229167 45 0 1 1.322917,1.322917 v 15.874999 a 1.3229167,1.3229167 135 0 1 -1.322917,1.322916 H 2.6458333 A 1.3229167,1.3229167 45 0 1 1.3229166,68.791662 l 0,-15.874999 a 1.3229167,1.3229167 135 0 1 1.3229167,-1.322917 z" /> + <path + style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend)" + d="M 29.104166,10.583331 H 42.333332" + id="path67244" /> + <path + style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend-4)" + d="M 29.104167,35.71875 H 42.333333" + id="path67244-1" /> + <path + style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Arrow2Lend-4-2)" + d="M 29.104167,60.854167 H 42.333333" + id="path67244-1-3" /> + </g> +</svg> |
