Commit d0841b87 authored by Jonathan Corbet's avatar Jonathan Corbet
Browse files

Merge branch 'media-uapi' into docs-mw 

Mauro says:

In the past, media used Docbook to generate documentation, together
with some logic to ensure that cross-references would match the
actual defined uAPI.

The rationale is that we wanted to automatically check for uAPI
documentation gaps.

The same logic was migrated to Sphinx. Back then, broken links
were reported. However, recent versions of it and/or changes at
conf.py disabled such checks.

The result is that several symbols are now not cross-referenced,
and we don't get warnings anymore when something breaks.

This series consist on 2 parts:

Part 1: extra patches to parse_data_structs.py and kernel_include.py;
Part 2: media documentation fixes.
parents 44abc8fc be63b06b

Documentation/sphinx/kernel_include.py

+93 −15
Original line number Diff line number Diff line
@@ -87,6 +87,8 @@ import os.path 
import re

import sys



from difflib import get_close_matches



from docutils import io, nodes, statemachine

from docutils.statemachine import ViewList

from docutils.parsers.rst import Directive, directives
@@ -104,6 +106,8 @@ logger = logging.getLogger(__name__) 


RE_DOMAIN_REF = re.compile(r'\\ :(ref|c:type|c:func):`([^<`]+)(?:<([^>]+)>)?`\\')

RE_SIMPLE_REF = re.compile(r'`([^`]+)`')

RE_LINENO_REF = re.compile(r'^\s*-\s+LINENO_(\d+):\s+(.*)')

RE_SPLIT_DOMAIN = re.compile(r"(.*)\.(.*)")



def ErrorString(exc):  # Shamelessly stolen from docutils

    return f'{exc.__class__.__name}: {exc}'
@@ -212,14 +216,16 @@ class KernelInclude(Directive): 
        - a TOC table containing cross references.

        """

        parser = ParseDataStructs()

        parser.parse_file(path)



        if 'exception-file' in self.options:

            source_dir = os.path.dirname(os.path.abspath(

                self.state_machine.input_lines.source(

                    self.lineno - self.state_machine.input_offset - 1)))

            exceptions_file = os.path.join(source_dir, self.options['exception-file'])

            parser.process_exceptions(exceptions_file)

        else:

            exceptions_file = None



        parser.parse_file(path, exceptions_file)



        # Store references on a symbol dict to be used at check time

        if 'warn-broken' in self.options:
@@ -242,23 +248,32 @@ class KernelInclude(Directive): 
        # TOC output is a ReST file, not a literal. So, we can add line

        # numbers



        rawtext = parser.gen_toc()

        startline = self.options.get('start-line', None)

        endline = self.options.get('end-line', None)



        relpath = os.path.relpath(path, srctree)



        include_lines = statemachine.string2lines(rawtext, tab_width,

                                                  convert_whitespace=True)

        result = ViewList()

        for line in parser.gen_toc().split("\n"):

            match = RE_LINENO_REF.match(line)

            if not match:

                result.append(line, path)

                continue



        # Append line numbers data

            ln, ref = match.groups()

            ln = int(ln)



        startline = self.options.get('start-line', None)

            # Filter line range if needed

            if startline and (ln < startline):

                continue



        result = ViewList()

        if startline and startline > 0:

            offset = startline - 1

        else:

            offset = 0

            if endline and (ln > endline):

                continue



        for ln, line in enumerate(include_lines, start=offset):

            result.append(line, path, ln)

            # Sphinx numerates starting with zero, but text editors

            # and other tools start from one

            realln = ln + 1

            result.append(f"- {ref}: {relpath}#{realln}", path, ln)



        self.state_machine.insert_input(result, path)
@@ -388,6 +403,63 @@ class KernelInclude(Directive): 
# ==============================================================================



reported = set()

DOMAIN_INFO = {}

all_refs = {}



def fill_domain_info(env):

    """

    Get supported reference types for each Sphinx domain and C namespaces

    """

    if DOMAIN_INFO:

        return



    for domain_name, domain_instance in env.domains.items():

        try:

            object_types = list(domain_instance.object_types.keys())

            DOMAIN_INFO[domain_name] = object_types

        except AttributeError:

            # Ignore domains that we can't retrieve object types, if any

            pass



    for domain in DOMAIN_INFO.keys():

        domain_obj = env.get_domain(domain)

        for name, dispname, objtype, docname, anchor, priority in domain_obj.get_objects():

            ref_name = name.lower()



            if domain == "c":

                if '.' in ref_name:

                    ref_name = ref_name.split(".")[-1]



            if not ref_name in all_refs:

                all_refs[ref_name] = []



            all_refs[ref_name].append(f"\t{domain}:{objtype}:`{name}` (from {docname})")



def get_suggestions(app, env, node,

                    original_target, original_domain, original_reftype):

    """Check if target exists in the other domain or with different reftypes."""

    original_target = original_target.lower()



    # Remove namespace if present

    if original_domain == "c":

        if '.' in original_target:

            original_target = original_target.split(".")[-1]



    suggestions = []



    # If name exists, propose exact name match on different domains

    if original_target in all_refs:

        return all_refs[original_target]



    # If not found, get a close match, using difflib.

    # Such method is based on Ratcliff-Obershelp Algorithm, which seeks

    # for a close match within a certain distance. We're using the defaults

    # here, e.g. cutoff=0.6, proposing 3 alternatives

    matches = get_close_matches(original_target, all_refs.keys())

    for match in matches:

        suggestions += all_refs[match]



    return suggestions



def check_missing_refs(app, env, node, contnode):

    """Check broken refs for the files it creates xrefs"""
@@ -404,11 +476,13 @@ def check_missing_refs(app, env, node, contnode): 
    if node.source not in xref_files:

        return None



    fill_domain_info(env)



    target = node.get('reftarget', '')

    domain = node.get('refdomain', 'std')

    reftype = node.get('reftype', '')



    msg = f"can't link to: {domain}:{reftype}:: {target}"

    msg = f"Invalid xref: {domain}:{reftype}:`{target}`"



    # Don't duplicate warnings

    data = (node.source, msg)
@@ -416,6 +490,10 @@ def check_missing_refs(app, env, node, contnode): 
        return None

    reported.add(data)



    suggestions = get_suggestions(app, env, node, target, domain, reftype)

    if suggestions:

        msg += ". Possible alternatives:\n" + '\n'.join(suggestions)



    logger.warning(msg, location=node, type='ref', subtype='missing')



    return None

Documentation/userspace-api/media/cec/cec-header.rst

+5 −3
Original line number Diff line number Diff line
@@ -2,10 +2,12 @@ 


.. _cec_header:



***************

CEC Header File

***************

****************

CEC uAPI Symbols

****************



.. kernel-include:: include/uapi/linux/cec.h

    :generate-cross-refs:

    :exception-file: cec.h.rst.exceptions

    :toc:

    :warn-broken:

Documentation/userspace-api/media/cec/cec.h.rst.exceptions

+3 −0
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0



# All symbols belong to CEC namespace

namespace CEC



# Ignore header name

ignore define _CEC_UAPI_H

Documentation/userspace-api/media/dvb/dmx.h.rst.exceptions

+41 −45
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0



# All symbols belone to this namespace

namespace DTV.dmx



# Ignore header name

ignore define _UAPI_DVBDMX_H_



# Ignore limit constants

ignore define DMX_FILTER_SIZE



# dmx_pes_type_t enum symbols

replace enum dmx_ts_pes :c:type:`dmx_pes_type`

replace symbol DMX_PES_AUDIO0 :c:type:`dmx_pes_type`

replace symbol DMX_PES_VIDEO0 :c:type:`dmx_pes_type`

replace symbol DMX_PES_TELETEXT0 :c:type:`dmx_pes_type`

replace symbol DMX_PES_SUBTITLE0 :c:type:`dmx_pes_type`

replace symbol DMX_PES_PCR0 :c:type:`dmx_pes_type`

replace symbol DMX_PES_AUDIO1 :c:type:`dmx_pes_type`

replace symbol DMX_PES_VIDEO1 :c:type:`dmx_pes_type`

replace symbol DMX_PES_TELETEXT1 :c:type:`dmx_pes_type`

replace symbol DMX_PES_SUBTITLE1 :c:type:`dmx_pes_type`

replace symbol DMX_PES_PCR1 :c:type:`dmx_pes_type`

replace symbol DMX_PES_AUDIO2 :c:type:`dmx_pes_type`

replace symbol DMX_PES_VIDEO2 :c:type:`dmx_pes_type`

replace symbol DMX_PES_TELETEXT2 :c:type:`dmx_pes_type`

replace symbol DMX_PES_SUBTITLE2 :c:type:`dmx_pes_type`

replace symbol DMX_PES_PCR2 :c:type:`dmx_pes_type`

replace symbol DMX_PES_AUDIO3 :c:type:`dmx_pes_type`

replace symbol DMX_PES_VIDEO3 :c:type:`dmx_pes_type`

replace symbol DMX_PES_TELETEXT3 :c:type:`dmx_pes_type`

replace symbol DMX_PES_SUBTITLE3 :c:type:`dmx_pes_type`

replace symbol DMX_PES_PCR3 :c:type:`dmx_pes_type`

replace symbol DMX_PES_OTHER :c:type:`dmx_pes_type`

# dmx_ts_pes_type_t enum symbols

replace symbol DMX_PES_AUDIO0 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_VIDEO0 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_TELETEXT0 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_SUBTITLE0 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_PCR0 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_AUDIO1 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_VIDEO1 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_TELETEXT1 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_SUBTITLE1 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_PCR1 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_AUDIO2 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_VIDEO2 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_TELETEXT2 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_SUBTITLE2 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_PCR2 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_AUDIO3 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_VIDEO3 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_TELETEXT3 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_SUBTITLE3 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_PCR3 :c:type:`DTV.dmx.dmx_ts_pes`

replace symbol DMX_PES_OTHER :c:type:`DTV.dmx.dmx_ts_pes`



# Ignore obsolete symbols

ignore define DMX_PES_AUDIO
@@ -38,29 +40,23 @@ ignore define DMX_PES_SUBTITLE 
ignore define DMX_PES_PCR



# dmx_input_t symbols

replace enum dmx_input :c:type:`dmx_input`

replace symbol DMX_IN_FRONTEND :c:type:`dmx_input`

replace symbol DMX_IN_DVR :c:type:`dmx_input`

replace symbol DMX_IN_FRONTEND :c:enum:`DTV.dmx.dmx_input`

replace symbol DMX_IN_DVR :c:enum:`DTV.dmx.dmx_input`



# Flags for struct dmx_sct_filter_params

replace define DMX_CHECK_CRC :c:type:`dmx_sct_filter_params`

replace define DMX_ONESHOT :c:type:`dmx_sct_filter_params`

replace define DMX_IMMEDIATE_START :c:type:`dmx_sct_filter_params`



# some typedefs should point to struct/enums

replace typedef dmx_filter_t :c:type:`dmx_filter`

replace typedef dmx_pes_type_t :c:type:`dmx_pes_type`

replace typedef dmx_input_t :c:type:`dmx_input`



replace symbol DMX_BUFFER_FLAG_HAD_CRC32_DISCARD :c:type:`dmx_buffer_flags`

replace	symbol DMX_BUFFER_FLAG_TEI :c:type:`dmx_buffer_flags`

replace	symbol DMX_BUFFER_PKT_COUNTER_MISMATCH :c:type:`dmx_buffer_flags`

replace	symbol DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED :c:type:`dmx_buffer_flags`

replace	symbol DMX_BUFFER_FLAG_DISCONTINUITY_INDICATOR :c:type:`dmx_buffer_flags`



replace symbol DMX_OUT_DECODER :c:type:`dmx_output`

replace symbol DMX_OUT_TAP :c:type:`dmx_output`

replace symbol DMX_OUT_TS_TAP :c:type:`dmx_output`

replace symbol DMX_OUT_TSDEMUX_TAP :c:type:`dmx_output`

replace define DMX_CHECK_CRC :c:type:`DTV.dmx.dmx_sct_filter_params`

replace define DMX_ONESHOT :c:type:`DTV.dmx.dmx_sct_filter_params`

replace define DMX_IMMEDIATE_START :c:type:`DTV.dmx.dmx_sct_filter_params`



replace symbol DMX_BUFFER_FLAG_HAD_CRC32_DISCARD :c:type:`DTV.dmx.dmx_buffer_flags`

replace symbol DMX_BUFFER_FLAG_TEI :c:type:`DTV.dmx.dmx_buffer_flags`

replace symbol DMX_BUFFER_PKT_COUNTER_MISMATCH :c:type:`DTV.dmx.dmx_buffer_flags`

replace symbol DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED :c:type:`DTV.dmx.dmx_buffer_flags`

replace symbol DMX_BUFFER_FLAG_DISCONTINUITY_INDICATOR :c:type:`DTV.dmx.dmx_buffer_flags`



replace symbol DMX_OUT_DECODER :c:type:`DTV.dmx.dmx_output`

replace symbol DMX_OUT_TAP :c:type:`DTV.dmx.dmx_output`

replace symbol DMX_OUT_TS_TAP :c:type:`DTV.dmx.dmx_output`

replace symbol DMX_OUT_TSDEMUX_TAP :c:type:`DTV.dmx.dmx_output`



replace ioctl DMX_DQBUF dmx_qbuf

Documentation/userspace-api/media/dvb/dmx_types.rst

+1 −0
Original line number Diff line number Diff line 
.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later

.. c:namespace:: DTV.dmx



.. _dmx_types: