tools: ynl: use the common YAML loading and validation code (30a5c6c8) · Commits · git / linux-net

tools/net/ynl/lib/ynl.py

+25 −93

Original line number	Diff line number	Diff line
		# SPDX-License-Identifier: BSD-3-Clause

		import functools
		import jsonschema
		import os
		import random
		import socket
		import struct
		import yaml

		from .nlspec import SpecFamily

		#
		# Generic Netlink code which should really be in some library, but I can't quickly find one.
		#
		@@ -158,8 +159,8 @@ class NlMsg:
		# We don't have the ability to parse nests yet, so only do global
		if 'miss-type' in self.extack and 'miss-nest' not in self.extack:
		miss_type = self.extack['miss-type']
		if len(attr_space.attr_list) > miss_type:
		spec = attr_space.attr_list[miss_type]
		if miss_type in attr_space.attrs_by_val:
		spec = attr_space.attrs_by_val[miss_type]
		desc = spec['name']
		if 'doc' in spec:
		desc += f" ({spec['doc']})"
		@@ -289,100 +290,31 @@ class GenlFamily:
		#


		class YnlAttrSpace:
		def __init__(self, family, yaml):
		self.yaml = yaml

		self.attrs = dict()
		self.name = self.yaml['name']
		self.subspace_of = self.yaml['subset-of'] if 'subspace-of' in self.yaml else None

		val = 0
		max_val = 0
		for elem in self.yaml['attributes']:
		if 'value' in elem:
		val = elem['value']
		else:
		elem['value'] = val
		if val > max_val:
		max_val = val
		val += 1

		self.attrs[elem['name']] = elem

		self.attr_list = [None] * (max_val + 1)
		for elem in self.yaml['attributes']:
		self.attr_list[elem['value']] = elem

		def __getitem__(self, key):
		return self.attrs[key]

		def __contains__(self, key):
		return key in self.yaml

		def __iter__(self):
		yield from self.attrs

		def items(self):
		return self.attrs.items()


		class YnlFamily:
		class YnlFamily(SpecFamily):
		def __init__(self, def_path, schema=None):
		self.include_raw = False
		super().__init__(def_path, schema)

		with open(def_path, "r") as stream:
		self.yaml = yaml.safe_load(stream)

		if schema:
		with open(schema, "r") as stream:
		schema = yaml.safe_load(stream)

		jsonschema.validate(self.yaml, schema)
		self.include_raw = False

		self.sock = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, Netlink.NETLINK_GENERIC)
		self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_CAP_ACK, 1)
		self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_EXT_ACK, 1)

		self._ops = dict()
		self._spaces = dict()
		self._types = dict()

		for elem in self.yaml['attribute-sets']:
		self._spaces[elem['name']] = YnlAttrSpace(self, elem)

		for elem in self.yaml['definitions']:
		self._types[elem['name']] = elem

		async_separation = 'async-prefix' in self.yaml['operations']
		self.async_msg_ids = set()
		self.async_msg_queue = []
		val = 0
		max_val = 0
		for elem in self.yaml['operations']['list']:
		if not (async_separation and ('notify' in elem or 'event' in elem)):
		if 'value' in elem:
		val = elem['value']
		else:
		elem['value'] = val
		val += 1
		max_val = max(val, max_val)

		if 'notify' in elem or 'event' in elem:
		self.async_msg_ids.add(elem['value'])

		self._ops[elem['name']] = elem

		op_name = elem['name'].replace('-', '_')

		bound_f = functools.partial(self._op, elem['name'])
		setattr(self, op_name, bound_f)
		for msg in self.msgs.values():
		if msg.is_async:
		self.async_msg_ids.add(msg.value)

		self._op_array = [None] * max_val
		for _, op in self._ops.items():
		self._op_array[op['value']] = op
		if 'notify' in op:
		op['attribute-set'] = self._ops[op['notify']]['attribute-set']
		for op_name, op in self.ops.items():
		bound_f = functools.partial(self._op, op_name)
		setattr(self, op.ident_name, bound_f)

		self.family = GenlFamily(self.yaml['name'])

		@@ -395,8 +327,8 @@ class YnlFamily:
		self.family.genl_family['mcast'][mcast_name])

		def _add_attr(self, space, name, value):
		attr = self._spaces[space][name]
		nl_type = attr['value']
		attr = self.attr_sets[space][name]
		nl_type = attr.value
		if attr["type"] == 'nest':
		nl_type \|= Netlink.NLA_F_NESTED
		attr_payload = b''
		@@ -430,10 +362,10 @@ class YnlFamily:
		rsp[attr_spec['name']] = value

		def _decode(self, attrs, space):
		attr_space = self._spaces[space]
		attr_space = self.attr_sets[space]
		rsp = dict()
		for attr in attrs:
		attr_spec = attr_space.attr_list[attr.type]
		attr_spec = attr_space.attrs_by_val[attr.type]
		if attr_spec["type"] == 'nest':
		subdict = self._decode(NlAttrs(attr.raw), attr_spec['nested-attributes'])
		rsp[attr_spec['name']] = subdict
		@@ -457,9 +389,9 @@ class YnlFamily:
		if self.include_raw:
		msg['nlmsg'] = nl_msg
		msg['genlmsg'] = genl_msg
		op = self._op_array[genl_msg.genl_cmd]
		op = self.msgs_by_value[genl_msg.genl_cmd]
		msg['name'] = op['name']
		msg['msg'] = self._decode(genl_msg.raw_attrs, op['attribute-set'])
		msg['msg'] = self._decode(genl_msg.raw_attrs, op.attr_set.name)
		self.async_msg_queue.append(msg)

		def check_ntf(self):
		@@ -487,16 +419,16 @@ class YnlFamily:
		self.handle_ntf(nl_msg, gm)

		def _op(self, method, vals, dump=False):
		op = self._ops[method]
		op = self.ops[method]

		nl_flags = Netlink.NLM_F_REQUEST \| Netlink.NLM_F_ACK
		if dump:
		nl_flags \|= Netlink.NLM_F_DUMP

		req_seq = random.randint(1024, 65535)
		msg = _genl_msg(self.family.family_id, nl_flags, op['value'], 1, req_seq)
		msg = _genl_msg(self.family.family_id, nl_flags, op.value, 1, req_seq)
		for name, value in vals.items():
		msg += self._add_attr(op['attribute-set'], name, value)
		msg += self._add_attr(op.attr_set.name, name, value)
		msg = _genl_msg_finalize(msg)

		self.sock.send(msg, 0)
		@@ -505,7 +437,7 @@ class YnlFamily:
		rsp = []
		while not done:
		reply = self.sock.recv(128 * 1024)
		nms = NlMsgs(reply, attr_space=self._spaces[op['attribute-set']])
		nms = NlMsgs(reply, attr_space=op.attr_set)
		for nl_msg in nms:
		if nl_msg.error:
		print("Netlink error:", os.strerror(-nl_msg.error))
		@@ -517,7 +449,7 @@ class YnlFamily:

		gm = GenlMsg(nl_msg)
		# Check if this is a reply to our request
		if nl_msg.nl_seq != req_seq or gm.genl_cmd != op['value']:
		if nl_msg.nl_seq != req_seq or gm.genl_cmd != op.value:
		if gm.genl_cmd in self.async_msg_ids:
		self.handle_ntf(nl_msg, gm)
		continue
		@@ -525,7 +457,7 @@ class YnlFamily:
		print('Unexpected message: ' + repr(gm))
		continue

		rsp.append(self._decode(gm.raw_attrs, op['attribute-set']))
		rsp.append(self._decode(gm.raw_attrs, op.attr_set.name))

		if not rsp:
		return None

tools/net/ynl/ynl-gen-c.py

+117 −139

Original line number	Diff line number	Diff line
		@@ -2,10 +2,11 @@

		import argparse
		import collections
		import jsonschema
		import os
		import yaml

		from lib import SpecFamily, SpecAttrSet, SpecAttr, SpecOperation


		def c_upper(name):
		return name.upper().replace('-', '_')
		@@ -28,12 +29,12 @@ class BaseNlLib:
		"ynl_cb_array, NLMSG_MIN_TYPE)"


		class Type:
		def __init__(self, family, attr_set, attr):
		self.family = family
		class Type(SpecAttr):
		def __init__(self, family, attr_set, attr, value):
		super().__init__(family, attr_set, attr, value)

		self.attr = attr
		self.value = attr['value']
		self.name = c_lower(attr['name'])
		self.attr_set = attr_set
		self.type = attr['type']
		self.checks = attr.get('checks', {})

		@@ -46,17 +47,17 @@ class Type:
		else:
		self.nested_render_name = f"{family.name}_{c_lower(self.nested_attrs)}"

		self.enum_name = f"{attr_set.name_prefix}{self.name}"
		self.enum_name = c_upper(self.enum_name)
		self.c_name = c_lower(self.name)
		if self.c_name in _C_KW:
		self.c_name += '_'

		def __getitem__(self, key):
		return self.attr[key]
		# Added by resolve():
		self.enum_name = None
		delattr(self, "enum_name")

		def __contains__(self, key):
		return key in self.attr
		def resolve(self):
		self.enum_name = f"{self.attr_set.name_prefix}{self.name}"
		self.enum_name = c_upper(self.enum_name)

		def is_multi_val(self):
		return None
		@@ -214,24 +215,34 @@ class TypePad(Type):


		class TypeScalar(Type):
		def __init__(self, family, attr_set, attr):
		super().__init__(family, attr_set, attr)
		def __init__(self, family, attr_set, attr, value):
		super().__init__(family, attr_set, attr, value)

		self.byte_order_comment = ''
		if 'byte-order' in attr:
		self.byte_order_comment = f" /* {attr['byte-order']} */"

		# Added by resolve():
		self.is_bitfield = None
		delattr(self, "is_bitfield")
		self.type_name = None
		delattr(self, "type_name")

		def resolve(self):
		self.resolve_up(super())

		self.is_bitfield = False
		if 'enum' in self.attr:
		self.is_bitfield = family.consts[self.attr['enum']]['type'] == 'flags'
		if 'enum-as-flags' in self.attr and self.attr['enum-as-flags']:
		self.is_bitfield = True
		elif 'enum' in self.attr:
		self.is_bitfield = self.family.consts[self.attr['enum']]['type'] == 'flags'
		else:
		self.is_bitfield = False

		if 'enum' in self.attr and not self.is_bitfield:
		self.type_name = f"enum {family.name}_{c_lower(self.attr['enum'])}"
		self.type_name = f"enum {self.family.name}_{c_lower(self.attr['enum'])}"
		else:
		self.type_name = '__' + self.type

		self.byte_order_comment = ''
		if 'byte-order' in attr:
		self.byte_order_comment = f" /* {attr['byte-order']} */"

		def _mnl_type(self):
		t = self.type
		# mnl does not have a helper for signed types
		@@ -648,14 +659,11 @@ class EnumSet:
		return mask


		class AttrSet:
		class AttrSet(SpecAttrSet):
		def __init__(self, family, yaml):
		self.yaml = yaml
		super().__init__(family, yaml)

		self.attrs = dict()
		self.name = self.yaml['name']
		if 'subset-of' not in yaml:
		self.subset_of = None
		if self.subset_of is None:
		if 'name-prefix' in yaml:
		pfx = yaml['name-prefix']
		elif self.name == family.name:
		@@ -665,83 +673,68 @@ class AttrSet:
		self.name_prefix = c_upper(pfx)
		self.max_name = c_upper(self.yaml.get('attr-max-name', f"{self.name_prefix}max"))
		else:
		self.subset_of = self.yaml['subset-of']
		self.name_prefix = family.attr_sets[self.subset_of].name_prefix
		self.max_name = family.attr_sets[self.subset_of].max_name

		# Added by resolve:
		self.c_name = None
		delattr(self, "c_name")

		def resolve(self):
		self.c_name = c_lower(self.name)
		if self.c_name in _C_KW:
		self.c_name += '_'
		if self.c_name == family.c_name:
		if self.c_name == self.family.c_name:
		self.c_name = ''

		val = 0
		for elem in self.yaml['attributes']:
		if 'value' in elem:
		val = elem['value']
		else:
		elem['value'] = val
		val += 1

		def new_attr(self, elem, value):
		if 'multi-attr' in elem and elem['multi-attr']:
		attr = TypeMultiAttr(family, self, elem)
		return TypeMultiAttr(self.family, self, elem, value)
		elif elem['type'] in scalars:
		attr = TypeScalar(family, self, elem)
		return TypeScalar(self.family, self, elem, value)
		elif elem['type'] == 'unused':
		attr = TypeUnused(family, self, elem)
		return TypeUnused(self.family, self, elem, value)
		elif elem['type'] == 'pad':
		attr = TypePad(family, self, elem)
		return TypePad(self.family, self, elem, value)
		elif elem['type'] == 'flag':
		attr = TypeFlag(family, self, elem)
		return TypeFlag(self.family, self, elem, value)
		elif elem['type'] == 'string':
		attr = TypeString(family, self, elem)
		return TypeString(self.family, self, elem, value)
		elif elem['type'] == 'binary':
		attr = TypeBinary(family, self, elem)
		return TypeBinary(self.family, self, elem, value)
		elif elem['type'] == 'nest':
		attr = TypeNest(family, self, elem)
		return TypeNest(self.family, self, elem, value)
		elif elem['type'] == 'array-nest':
		attr = TypeArrayNest(family, self, elem)
		return TypeArrayNest(self.family, self, elem, value)
		elif elem['type'] == 'nest-type-value':
		attr = TypeNestTypeValue(family, self, elem)
		return TypeNestTypeValue(self.family, self, elem, value)
		else:
		raise Exception(f"No typed class for type {elem['type']}")

		self.attrs[elem['name']] = attr

		def __getitem__(self, key):
		return self.attrs[key]
		class Operation(SpecOperation):
		def __init__(self, family, yaml, req_value, rsp_value):
		super().__init__(family, yaml, req_value, rsp_value)

		def __contains__(self, key):
		return key in self.yaml

		def __iter__(self):
		yield from self.attrs
		if req_value != rsp_value:
		raise Exception("Directional messages not supported by codegen")

		def items(self):
		return self.attrs.items()


		class Operation:
		def __init__(self, family, yaml, value):
		self.yaml = yaml
		self.value = value

		self.name = self.yaml['name']
		self.render_name = family.name + '_' + c_lower(self.name)
		self.is_async = 'notify' in yaml or 'event' in yaml
		if not self.is_async:
		self.enum_name = family.op_prefix + c_upper(self.name)
		else:
		self.enum_name = family.async_op_prefix + c_upper(self.name)

		self.dual_policy = ('do' in yaml and 'request' in yaml['do']) and \
		('dump' in yaml and 'request' in yaml['dump'])

		def __getitem__(self, key):
		return self.yaml[key]
		# Added by resolve:
		self.enum_name = None
		delattr(self, "enum_name")

		def __contains__(self, key):
		return key in self.yaml
		def resolve(self):
		self.resolve_up(super())

		if not self.is_async:
		self.enum_name = self.family.op_prefix + c_upper(self.name)
		else:
		self.enum_name = self.family.async_op_prefix + c_upper(self.name)

		def add_notification(self, op):
		if 'notify' not in self.yaml:
		@@ -751,21 +744,23 @@ class Operation:
		self.yaml['notify']['cmds'].append(op)


		class Family:
		class Family(SpecFamily):
		def __init__(self, file_name):
		with open(file_name, "r") as stream:
		self.yaml = yaml.safe_load(stream)

		self.proto = self.yaml.get('protocol', 'genetlink')

		with open(os.path.dirname(os.path.dirname(file_name)) +
		f'/{self.proto}.yaml', "r") as stream:
		schema = yaml.safe_load(stream)

		jsonschema.validate(self.yaml, schema)

		if self.yaml.get('protocol', 'genetlink') not in {'genetlink', 'genetlink-c', 'genetlink-legacy'}:
		raise Exception("Codegen only supported for genetlink")
		# Added by resolve:
		self.c_name = None
		delattr(self, "c_name")
		self.op_prefix = None
		delattr(self, "op_prefix")
		self.async_op_prefix = None
		delattr(self, "async_op_prefix")
		self.mcgrps = None
		delattr(self, "mcgrps")
		self.consts = None
		delattr(self, "consts")
		self.hooks = None
		delattr(self, "hooks")

		super().__init__(file_name)

		self.fam_key = c_upper(self.yaml.get('c-family-name', self.yaml["name"] + '_FAMILY_NAME'))
		self.ver_key = c_upper(self.yaml.get('c-version-name', self.yaml["name"] + '_FAMILY_VERSION'))
		@@ -773,12 +768,18 @@ class Family:
		if 'definitions' not in self.yaml:
		self.yaml['definitions'] = []

		self.name = self.yaml['name']
		self.c_name = c_lower(self.name)
		if 'uapi-header' in self.yaml:
		self.uapi_header = self.yaml['uapi-header']
		else:
		self.uapi_header = f"linux/{self.name}.h"

		def resolve(self):
		self.resolve_up(super())

		if self.yaml.get('protocol', 'genetlink') not in {'genetlink', 'genetlink-c', 'genetlink-legacy'}:
		raise Exception("Codegen only supported for genetlink")

		self.c_name = c_lower(self.name)
		if 'name-prefix' in self.yaml['operations']:
		self.op_prefix = c_upper(self.yaml['operations']['name-prefix'])
		else:
		@@ -791,12 +792,6 @@ class Family:
		self.mcgrps = self.yaml.get('mcast-groups', {'list': []})

		self.consts = dict()
		# list of all operations
		self.msg_list = []
		# dict of operations which have their own message type (have attributes)
		self.ops = collections.OrderedDict()
		self.attr_sets = dict()
		self.attr_sets_list = []

		self.hooks = dict()
		for when in ['pre', 'post']:
		@@ -824,11 +819,11 @@ class Family:
		if self.kernel_policy == 'global':
		self._load_global_policy()

		def __getitem__(self, key):
		return self.yaml[key]
		def new_attr_set(self, elem):
		return AttrSet(self, elem)

		def get(self, key, default=None):
		return self.yaml.get(key, default)
		def new_operation(self, elem, req_value, rsp_value):
		return Operation(self, elem, req_value, rsp_value)

		# Fake a 'do' equivalent of all events, so that we can render their response parsing
		def _mock_up_events(self):
		@@ -847,27 +842,10 @@ class Family:
		else:
		self.consts[elem['name']] = elem

		for elem in self.yaml['attribute-sets']:
		attr_set = AttrSet(self, elem)
		self.attr_sets[elem['name']] = attr_set
		self.attr_sets_list.append((elem['name'], attr_set), )

		ntf = []
		val = 0
		for elem in self.yaml['operations']['list']:
		if 'value' in elem:
		val = elem['value']

		op = Operation(self, elem, val)
		val += 1

		self.msg_list.append(op)
		if 'notify' in elem:
		ntf.append(op)
		continue
		if 'attribute-set' not in elem:
		continue
		self.ops[elem['name']] = op
		for msg in self.msgs.values():
		if 'notify' in msg:
		ntf.append(msg)
		for n in ntf:
		self.ops[n['notify']].add_notification(n)

		@@ -2033,7 +2011,7 @@ def render_uapi(family, cw):

		max_by_define = family.get('max-by-define', False)

		for _, attr_set in family.attr_sets_list:
		for _, attr_set in family.attr_sets.items():
		if attr_set.subset_of:
		continue

		@@ -2044,9 +2022,9 @@ def render_uapi(family, cw):
		uapi_enum_start(family, cw, attr_set.yaml, 'enum-name')
		for _, attr in attr_set.items():
		suffix = ','
		if attr['value'] != val:
		suffix = f" = {attr['value']},"
		val = attr['value']
		if attr.value != val:
		suffix = f" = {attr.value},"
		val = attr.value
		val += 1
		cw.p(attr.enum_name + suffix)
		cw.nl()
		@@ -2066,7 +2044,7 @@ def render_uapi(family, cw):
		max_value = f"({cnt_name} - 1)"

		uapi_enum_start(family, cw, family['operations'], 'enum-name')
		for op in family.msg_list:
		for op in family.msgs.values():
		if separate_ntf and ('notify' in op or 'event' in op):
		continue

		@@ -2085,7 +2063,7 @@ def render_uapi(family, cw):

		if separate_ntf:
		uapi_enum_start(family, cw, family['operations'], enum_name='async-enum')
		for op in family.msg_list:
		for op in family.msgs.values():
		if separate_ntf and not ('notify' in op or 'event' in op):
		continue