Merge branch 'mlx5-support-setting-a-parent-for-a-devlink-rate-node'

Tariq Toukan says:

====================
mlx5: Support setting a parent for a devlink rate node

This series by Carolina adds mlx5 support for the setting of a parent to
devlink rate nodes.

By introducing a hierarchical level to scheduling nodes, these changes
allow for more granular control over bandwidth allocation and isolation
of Virtual Functions.

Function renaming for parent setting on leafs:
- net/mlx5: Rename devlink rate parent set function for leaf nodes

Add support for hierarchy level tracking:
- net/mlx5: Introduce hierarchy level tracking on scheduling nodes
- net/mlx5: Preserve rate settings when creating a rate node

Support setting parent for rate nodes:
- net/mlx5: Add support for setting parent of nodes
====================

Link: https://patch.msgid.link/1741642016-44918-1-git-send-email-tariqt@nvidia.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

drivers/net/ethernet/mellanox/mlx5/core/devlink.c

@@ -327,7 +327,8 @@ static const struct devlink_ops mlx5_devlink_ops = {
 	.rate_node_tx_max_set = mlx5_esw_devlink_rate_node_tx_max_set,
 	.rate_node_new = mlx5_esw_devlink_rate_node_new,
 	.rate_node_del = mlx5_esw_devlink_rate_node_del,
-	.rate_leaf_parent_set = mlx5_esw_devlink_rate_parent_set,
+	.rate_leaf_parent_set = mlx5_esw_devlink_rate_leaf_parent_set,
+	.rate_node_parent_set = mlx5_esw_devlink_rate_node_parent_set,
 #endif
 #ifdef CONFIG_MLX5_SF_MANAGER
 	.port_new = mlx5_devlink_sf_port_new,

drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c

@@ -90,15 +90,30 @@ struct mlx5_esw_sched_node {
 	struct list_head children;
 	/* Valid only if this node is associated with a vport. */
 	struct mlx5_vport *vport;
+	/* Level in the hierarchy. The root node level is 1. */
+	u8 level;
 };
 
+static void esw_qos_node_attach_to_parent(struct mlx5_esw_sched_node *node)
+{
+	if (!node->parent) {
+		/* Root children are assigned a depth level of 2. */
+		node->level = 2;
+		list_add_tail(&node->entry, &node->esw->qos.domain->nodes);
+	} else {
+		node->level = node->parent->level + 1;
+		list_add_tail(&node->entry, &node->parent->children);
+	}
+}
+
 static void
 esw_qos_node_set_parent(struct mlx5_esw_sched_node *node, struct mlx5_esw_sched_node *parent)
 {
 	list_del_init(&node->entry);
 	node->parent = parent;
-	list_add_tail(&node->entry, &parent->children);
-	node->esw = parent->esw;
+	if (parent)
+		node->esw = parent->esw;
+	esw_qos_node_attach_to_parent(node);
 }
 
 void mlx5_esw_qos_vport_qos_free(struct mlx5_vport *vport)
@@ -305,8 +320,9 @@ static int esw_qos_set_node_min_rate(struct mlx5_esw_sched_node *node,
 	return 0;
 }
 
-static int esw_qos_create_node_sched_elem(struct mlx5_core_dev *dev, u32 parent_element_id,
-					  u32 *tsar_ix)
+static int
+esw_qos_create_node_sched_elem(struct mlx5_core_dev *dev, u32 parent_element_id,
+			       u32 max_rate, u32 bw_share, u32 *tsar_ix)
 {
 	u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {};
 	void *attr;
@@ -323,6 +339,8 @@ static int esw_qos_create_node_sched_elem(struct mlx5_core_dev *dev, u32 parent_
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR);
 	MLX5_SET(scheduling_context, tsar_ctx, parent_element_id,
 		 parent_element_id);
+	MLX5_SET(scheduling_context, tsar_ctx, max_average_bw, max_rate);
+	MLX5_SET(scheduling_context, tsar_ctx, bw_share, bw_share);
 	attr = MLX5_ADDR_OF(scheduling_context, tsar_ctx, element_attributes);
 	MLX5_SET(tsar_element, attr, tsar_type, TSAR_ELEMENT_TSAR_TYPE_DWRR);
 
@@ -358,7 +376,6 @@ static struct mlx5_esw_sched_node *
 __esw_qos_alloc_node(struct mlx5_eswitch *esw, u32 tsar_ix, enum sched_node_type type,
 		     struct mlx5_esw_sched_node *parent)
 {
-	struct list_head *parent_children;
 	struct mlx5_esw_sched_node *node;
 
 	node = kzalloc(sizeof(*node), GFP_KERNEL);
@@ -370,8 +387,7 @@ __esw_qos_alloc_node(struct mlx5_eswitch *esw, u32 tsar_ix, enum sched_node_type
 	node->type = type;
 	node->parent = parent;
 	INIT_LIST_HEAD(&node->children);
-	parent_children = parent ? &parent->children : &esw->qos.domain->nodes;
-	list_add_tail(&node->entry, parent_children);
+	esw_qos_node_attach_to_parent(node);
 
 	return node;
 }
@@ -396,7 +412,8 @@ __esw_qos_create_vports_sched_node(struct mlx5_eswitch *esw, struct mlx5_esw_sch
 	u32 tsar_ix;
 	int err;
 
-	err = esw_qos_create_node_sched_elem(esw->dev, esw->qos.root_tsar_ix, &tsar_ix);
+	err = esw_qos_create_node_sched_elem(esw->dev, esw->qos.root_tsar_ix, 0,
+					     0, &tsar_ix);
 	if (err) {
 		NL_SET_ERR_MSG_MOD(extack, "E-Switch create TSAR for node failed");
 		return ERR_PTR(err);
@@ -463,7 +480,8 @@ static int esw_qos_create(struct mlx5_eswitch *esw, struct netlink_ext_ack *exta
 	if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, esw_scheduling))
 		return -EOPNOTSUPP;
 
-	err = esw_qos_create_node_sched_elem(esw->dev, 0, &esw->qos.root_tsar_ix);
+	err = esw_qos_create_node_sched_elem(esw->dev, 0, 0, 0,
+					     &esw->qos.root_tsar_ix);
 	if (err) {
 		esw_warn(dev, "E-Switch create root TSAR failed (%d)\n", err);
 		return err;
@@ -986,10 +1004,10 @@ int mlx5_esw_qos_vport_update_parent(struct mlx5_vport *vport, struct mlx5_esw_s
 	return err;
 }
 
-int mlx5_esw_devlink_rate_parent_set(struct devlink_rate *devlink_rate,
-				     struct devlink_rate *parent,
-				     void *priv, void *parent_priv,
-				     struct netlink_ext_ack *extack)
+int mlx5_esw_devlink_rate_leaf_parent_set(struct devlink_rate *devlink_rate,
+					  struct devlink_rate *parent,
+					  void *priv, void *parent_priv,
+					  struct netlink_ext_ack *extack)
 {
 	struct mlx5_esw_sched_node *node;
 	struct mlx5_vport *vport = priv;
@@ -1000,3 +1018,105 @@ int mlx5_esw_devlink_rate_parent_set(struct devlink_rate *devlink_rate,
 	node = parent_priv;
 	return mlx5_esw_qos_vport_update_parent(vport, node, extack);
 }
+
+static int
+mlx5_esw_qos_node_validate_set_parent(struct mlx5_esw_sched_node *node,
+				      struct mlx5_esw_sched_node *parent,
+				      struct netlink_ext_ack *extack)
+{
+	u8 new_level, max_level;
+
+	if (parent && parent->esw != node->esw) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot assign node to another E-Switch");
+		return -EOPNOTSUPP;
+	}
+
+	if (!list_empty(&node->children)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot reassign a node that contains rate objects");
+		return -EOPNOTSUPP;
+	}
+
+	new_level = parent ? parent->level + 1 : 2;
+	max_level = 1 << MLX5_CAP_QOS(node->esw->dev, log_esw_max_sched_depth);
+	if (new_level > max_level) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Node hierarchy depth exceeds the maximum supported level");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int esw_qos_vports_node_update_parent(struct mlx5_esw_sched_node *node,
+					     struct mlx5_esw_sched_node *parent,
+					     struct netlink_ext_ack *extack)
+{
+	struct mlx5_esw_sched_node *curr_parent = node->parent;
+	struct mlx5_eswitch *esw = node->esw;
+	u32 parent_ix;
+	int err;
+
+	parent_ix = parent ? parent->ix : node->esw->qos.root_tsar_ix;
+	mlx5_destroy_scheduling_element_cmd(esw->dev,
+					    SCHEDULING_HIERARCHY_E_SWITCH,
+					    node->ix);
+	err = esw_qos_create_node_sched_elem(esw->dev, parent_ix,
+					     node->max_rate, 0, &node->ix);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Failed to create a node under the new hierarchy.");
+		if (esw_qos_create_node_sched_elem(esw->dev, curr_parent->ix,
+						   node->max_rate,
+						   node->bw_share,
+						   &node->ix))
+			esw_warn(esw->dev, "Node restore QoS failed\n");
+
+		return err;
+	}
+	esw_qos_node_set_parent(node, parent);
+
+	return 0;
+}
+
+static int mlx5_esw_qos_node_update_parent(struct mlx5_esw_sched_node *node,
+					   struct mlx5_esw_sched_node *parent,
+					   struct netlink_ext_ack *extack)
+{
+	struct mlx5_esw_sched_node *curr_parent;
+	struct mlx5_eswitch *esw = node->esw;
+	int err;
+
+	err = mlx5_esw_qos_node_validate_set_parent(node, parent, extack);
+	if (err)
+		return err;
+
+	esw_qos_lock(esw);
+	curr_parent = node->parent;
+	err = esw_qos_vports_node_update_parent(node, parent, extack);
+	if (err)
+		goto out;
+
+	esw_qos_normalize_min_rate(esw, curr_parent, extack);
+	esw_qos_normalize_min_rate(esw, parent, extack);
+
+out:
+	esw_qos_unlock(esw);
+
+	return err;
+}
+
+int mlx5_esw_devlink_rate_node_parent_set(struct devlink_rate *devlink_rate,
+					  struct devlink_rate *parent,
+					  void *priv, void *parent_priv,
+					  struct netlink_ext_ack *extack)
+{
+	struct mlx5_esw_sched_node *node = priv, *parent_node;
+
+	if (!parent)
+		return mlx5_esw_qos_node_update_parent(node, NULL, extack);
+
+	parent_node = parent_priv;
+	return mlx5_esw_qos_node_update_parent(node, parent_node, extack);
+}

drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h

@@ -29,10 +29,14 @@ int mlx5_esw_devlink_rate_node_new(struct devlink_rate *rate_node, void **priv,
 				   struct netlink_ext_ack *extack);
 int mlx5_esw_devlink_rate_node_del(struct devlink_rate *rate_node, void *priv,
 				   struct netlink_ext_ack *extack);
-int mlx5_esw_devlink_rate_parent_set(struct devlink_rate *devlink_rate,
-				     struct devlink_rate *parent,
-				     void *priv, void *parent_priv,
-				     struct netlink_ext_ack *extack);
+int mlx5_esw_devlink_rate_leaf_parent_set(struct devlink_rate *devlink_rate,
+					  struct devlink_rate *parent,
+					  void *priv, void *parent_priv,
+					  struct netlink_ext_ack *extack);
+int mlx5_esw_devlink_rate_node_parent_set(struct devlink_rate *devlink_rate,
+					  struct devlink_rate *parent,
+					  void *priv, void *parent_priv,
+					  struct netlink_ext_ack *extack);
 #endif
 
 #endif