diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 703f0e3af67d..c3408b3f7010 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -57,7 +57,7 @@ struct mlx5e_l2_table { bool promisc_enabled; }; -#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_TT - 1) +#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_INDIR_TIRS) #define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\ MLX5_HASH_FIELD_SEL_DST_IP) @@ -132,7 +132,8 @@ struct mlx5e_ptp_fs; void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, struct mlx5e_rx_res *rx_res, - struct ttc_params *ttc_params, bool tunnel); + struct ttc_params *ttc_params, bool tunnel, + bool ipsec_rss); void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs); int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 65dc3529283b..ef2878f0c20e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -61,6 +61,7 @@ struct mlx5e_ipsec_rx { struct mlx5_flow_table *pol_miss_ft; struct mlx5_flow_handle *pol_miss_rule; u8 allow_tunnel_mode : 1; + u8 ttc_rules_added : 1; }; /* IPsec RX flow steering */ @@ -585,6 +586,20 @@ static int ipsec_miss_create(struct mlx5_core_dev *mdev, return err; } +static struct mlx5_flow_destination +ipsec_rx_decrypted_pkt_def_dest(struct mlx5_ttc_table *ttc, u32 family) +{ + struct mlx5_flow_destination dest; + + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return mlx5_ttc_get_default_dest(ttc, family2tt(family)); + + dest.ft = mlx5_get_ttc_flow_table(ttc); + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + + return dest; +} + static void ipsec_rx_update_default_dest(struct mlx5e_ipsec_rx *rx, struct mlx5_flow_destination *old_dest, struct mlx5_flow_destination *new_dest) @@ -598,10 +613,10 @@ static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) { struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination old_dest, new_dest; - old_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), - family2tt(family)); + old_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, ns, &old_dest, family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, MLX5E_NIC_PRIO); @@ -614,12 +629,12 @@ static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) static void handle_ipsec_rx_cleanup(struct mlx5e_ipsec *ipsec, u32 family) { struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination old_dest, new_dest; old_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); old_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - new_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), - family2tt(family)); + new_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family); ipsec_rx_update_default_dest(rx, &old_dest, &new_dest); mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, ipsec->mdev); @@ -669,10 +684,13 @@ static void ipsec_mpv_work_handler(struct work_struct *_work) complete(&work->master_priv->ipsec->comp); } -static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family) +static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + if (rx->ttc_rules_added) + mlx5_ttc_destroy_ipsec_rules(ttc); mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); } @@ -707,7 +725,7 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, { /* disconnect */ if (rx != ipsec->rx_esw) - ipsec_rx_ft_disconnect(ipsec, family); + ipsec_rx_ft_disconnect(ipsec, rx, family); mlx5_del_flow_rules(rx->sa.rule); mlx5_destroy_flow_group(rx->sa.group); @@ -764,7 +782,7 @@ static int ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, if (rx == ipsec->rx_esw) return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest); - *dest = mlx5_ttc_get_default_dest(attr->ttc, family2tt(attr->family)); + *dest = ipsec_rx_decrypted_pkt_def_dest(attr->ttc, attr->family); err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest, attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, attr->prio); @@ -807,10 +825,16 @@ static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx_create_attr *attr) { struct mlx5_flow_destination dest = {}; + struct mlx5_ttc_table *ttc, *inner_ttc; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = rx->ft.sa; - mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest); + if (mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest)) + return; + + ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + inner_ttc = mlx5e_fs_get_ttc(ipsec->fs, true); + rx->ttc_rules_added = !mlx5_ttc_create_ipsec_rules(ttc, inner_ttc); } static int ipsec_rx_chains_create_miss(struct mlx5e_ipsec *ipsec, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 265c4ca85f7d..8928d2dcd43f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -905,6 +905,9 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs, ft_attr->prio = MLX5E_NIC_PRIO; for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? @@ -914,9 +917,17 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs, } } +static bool mlx5e_ipsec_rss_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, ipsec_next_header) && + MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, outer_l4_type_ext) && + MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, inner_l4_type_ext); +} + void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, struct mlx5e_rx_res *rx_res, - struct ttc_params *ttc_params, bool tunnel) + struct ttc_params *ttc_params, bool tunnel, + bool ipsec_rss) { struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr; @@ -927,7 +938,13 @@ void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, ft_attr->level = MLX5E_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; + ttc_params->ipsec_rss = ipsec_rss && + mlx5e_ipsec_rss_supported(fs->mdev); + for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? @@ -1293,7 +1310,7 @@ int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, { struct ttc_params ttc_params = {}; - mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true); + mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true, true); fs->ttc = mlx5_create_ttc_table(fs->mdev, &ttc_params); return PTR_ERR_OR_ZERO(fs->ttc); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 645221e3f21f..2f3454374c86 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -974,7 +974,7 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) MLX5_FLOW_NAMESPACE_KERNEL), false); /* The inner_ttc in the ttc params is intentionally not set */ - mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false); + mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false, false); if (rep->vport != MLX5_VPORT_UPLINK) /* To give uplik rep TTC a lower level for chaining from root ft */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 09c3eecb836d..b6d6584fc6fe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -838,6 +838,9 @@ static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp, ttc_params->ns_type = MLX5_FLOW_NAMESPACE_KERNEL; for (tt = 0; tt < MLX5_NUM_TT; tt++) { + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c index ca9ecec358b2..7adad784ad46 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c @@ -9,7 +9,7 @@ #include "mlx5_core.h" #include "lib/fs_ttc.h" -#define MLX5_TTC_MAX_NUM_GROUPS 4 +#define MLX5_TTC_MAX_NUM_GROUPS 7 #define MLX5_TTC_GROUP_TCPUDP_SIZE (MLX5_TT_IPV6_UDP + 1) struct mlx5_fs_ttc_groups { @@ -31,10 +31,14 @@ static int mlx5_fs_ttc_table_size(const struct mlx5_fs_ttc_groups *groups) /* L3/L4 traffic type classifier */ struct mlx5_ttc_table { int num_groups; + const struct mlx5_fs_ttc_groups *groups; + struct mlx5_core_dev *mdev; struct mlx5_flow_table *t; struct mlx5_flow_group **g; struct mlx5_ttc_rule rules[MLX5_NUM_TT]; struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT]; + u32 refcnt; + struct mutex mutex; /* Protect adding rules for ipsec crypto offload */ }; struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc) @@ -163,6 +167,8 @@ static struct mlx5_etype_proto ttc_tunnel_rules[] = { enum TTC_GROUP_TYPE { TTC_GROUPS_DEFAULT = 0, TTC_GROUPS_USE_L4_TYPE = 1, + TTC_GROUPS_DEFAULT_ESP = 2, + TTC_GROUPS_USE_L4_TYPE_ESP = 3, }; static const struct mlx5_fs_ttc_groups ttc_groups[] = { @@ -184,6 +190,31 @@ static const struct mlx5_fs_ttc_groups ttc_groups[] = { BIT(0), }, }, + [TTC_GROUPS_DEFAULT_ESP] = { + .num_groups = 6, + .group_size = { + MLX5_TTC_GROUP_TCPUDP_SIZE + BIT(1) + + MLX5_NUM_TUNNEL_TT, + BIT(2), /* decrypted outer L4 */ + BIT(2), /* decrypted inner L4 */ + BIT(1), /* ESP */ + BIT(1), + BIT(0), + }, + }, + [TTC_GROUPS_USE_L4_TYPE_ESP] = { + .use_l4_type = true, + .num_groups = 7, + .group_size = { + MLX5_TTC_GROUP_TCPUDP_SIZE, + BIT(1) + MLX5_NUM_TUNNEL_TT, + BIT(2), /* decrypted outer L4 */ + BIT(2), /* decrypted inner L4 */ + BIT(1), /* ESP */ + BIT(1), + BIT(0), + }, + }, }; static const struct mlx5_fs_ttc_groups inner_ttc_groups[] = { @@ -207,6 +238,23 @@ static const struct mlx5_fs_ttc_groups inner_ttc_groups[] = { }, }; +static const struct mlx5_fs_ttc_groups * +mlx5_ttc_get_fs_groups(bool use_l4_type, bool ipsec_rss) +{ + if (!ipsec_rss) + return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] : + &ttc_groups[TTC_GROUPS_DEFAULT]; + + return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP] : + &ttc_groups[TTC_GROUPS_DEFAULT_ESP]; +} + +bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc) +{ + return ttc->groups == &ttc_groups[TTC_GROUPS_DEFAULT_ESP] || + ttc->groups == &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP]; +} + u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt) { return ttc_tunnel_rules[tt].proto; @@ -257,6 +305,31 @@ static u8 mlx5_etype_to_ipv(u16 ethertype) return 0; } +static void mlx5_fs_ttc_set_match_ipv_outer(struct mlx5_core_dev *mdev, + struct mlx5_flow_spec *spec, + u16 etype) +{ + int match_ipv_outer = + MLX5_CAP_FLOWTABLE_NIC_RX(mdev, + ft_field_support.outer_ip_version); + u8 ipv; + + ipv = mlx5_etype_to_ipv(etype); + if (match_ipv_outer && ipv) { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.ip_version, ipv); + } else { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.ethertype); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.ethertype, etype); + } + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; +} + static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v, u8 proto, bool use_l4_type) { @@ -279,16 +352,12 @@ static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v, static struct mlx5_flow_handle * mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, struct mlx5_flow_destination *dest, u16 etype, u8 proto, - bool use_l4_type) + bool use_l4_type, bool ipsec_rss) { - int match_ipv_outer = - MLX5_CAP_FLOWTABLE_NIC_RX(dev, - ft_field_support.outer_ip_version); MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; int err = 0; - u8 ipv; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) @@ -305,15 +374,15 @@ mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft, proto, use_l4_type); } - ipv = mlx5_etype_to_ipv(etype); - if (match_ipv_outer && ipv) { - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv); - } else if (etype) { - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype); + if (etype) + mlx5_fs_ttc_set_match_ipv_outer(dev, spec, etype); + + if (ipsec_rss && proto == IPPROTO_ESP) { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_2.ipsec_next_header); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.ipsec_next_header, 0); + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; } rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); @@ -342,12 +411,16 @@ static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev, for (tt = 0; tt < MLX5_NUM_TT; tt++) { struct mlx5_ttc_rule *rule = &rules[tt]; + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + if (test_bit(tt, params->ignore_dests)) continue; rule->rule = mlx5_generate_ttc_rule(dev, ft, ¶ms->dests[tt], ttc_rules[tt].etype, ttc_rules[tt].proto, - use_l4_type); + use_l4_type, + params->ipsec_rss); if (IS_ERR(rule->rule)) { err = PTR_ERR(rule->rule); rule->rule = NULL; @@ -370,7 +443,7 @@ static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev, ¶ms->tunnel_dests[tt], ttc_tunnel_rules[tt].etype, ttc_tunnel_rules[tt].proto, - use_l4_type); + use_l4_type, false); if (IS_ERR(trules[tt])) { err = PTR_ERR(trules[tt]); trules[tt] = NULL; @@ -385,10 +458,78 @@ static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev, return err; } -static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc, - bool use_ipv, - const struct mlx5_fs_ttc_groups *groups) +static int mlx5_create_ttc_table_ipsec_groups(struct mlx5_ttc_table *ttc, + bool use_ipv, + u32 *in, int *next_ix) { + u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + const struct mlx5_fs_ttc_groups *groups = ttc->groups; + int ix = *next_ix; + + MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0); + + /* decrypted ESP outer group */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.l4_type_ext); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + MLX5_SET(fte_match_param, mc, outer_headers.l4_type_ext, 0); + + /* decrypted ESP inner group */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS); + if (use_ipv) + MLX5_SET(fte_match_param, mc, outer_headers.ip_version, 0); + else + MLX5_SET(fte_match_param, mc, outer_headers.ethertype, 0); + MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version); + MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.l4_type_ext); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + MLX5_SET(fte_match_param, mc, inner_headers.ip_version, 0); + MLX5_SET(fte_match_param, mc, inner_headers.l4_type_ext, 0); + + /* undecrypted ESP group */ + MLX5_SET_CFG(in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2); + if (use_ipv) + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version); + else + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol); + MLX5_SET_TO_ONES(fte_match_param, mc, + misc_parameters_2.ipsec_next_header); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += groups->group_size[ttc->num_groups]; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in); + if (IS_ERR(ttc->g[ttc->num_groups])) + goto err; + ttc->num_groups++; + + *next_ix = ix; + + return 0; + +err: + return PTR_ERR(ttc->g[ttc->num_groups]); +} + +static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc, + bool use_ipv) +{ + const struct mlx5_fs_ttc_groups *groups = ttc->groups; int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int ix = 0; u32 *in; @@ -436,8 +577,18 @@ static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc, goto err; ttc->num_groups++; + if (mlx5_ttc_has_esp_flow_group(ttc)) { + err = mlx5_create_ttc_table_ipsec_groups(ttc, use_ipv, in, &ix); + if (err) + goto err; + + MLX5_SET(fte_match_param, mc, + misc_parameters_2.ipsec_next_header, 0); + } + /* L3 Group */ MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0); + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); MLX5_SET_CFG(in, start_flow_index, ix); ix += groups->group_size[ttc->num_groups]; MLX5_SET_CFG(in, end_flow_index, ix - 1); @@ -527,6 +678,9 @@ static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev, for (tt = 0; tt < MLX5_NUM_TT; tt++) { struct mlx5_ttc_rule *rule = &rules[tt]; + if (mlx5_ttc_is_decrypted_esp_tt(tt)) + continue; + if (test_bit(tt, params->ignore_dests)) continue; rule->rule = mlx5_generate_inner_ttc_rule(dev, ft, @@ -700,6 +854,7 @@ void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc) kfree(ttc->g); mlx5_destroy_flow_table(ttc->t); + mutex_destroy(&ttc->mutex); kvfree(ttc); } @@ -709,7 +864,6 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(dev, ft_field_support.outer_ip_version); - const struct mlx5_fs_ttc_groups *groups; struct mlx5_flow_namespace *ns; struct mlx5_ttc_table *ttc; bool use_l4_type; @@ -738,11 +892,10 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, return ERR_PTR(-EOPNOTSUPP); } - groups = use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] : - &ttc_groups[TTC_GROUPS_DEFAULT]; + ttc->groups = mlx5_ttc_get_fs_groups(use_l4_type, params->ipsec_rss); WARN_ON_ONCE(params->ft_attr.max_fte); - params->ft_attr.max_fte = mlx5_fs_ttc_table_size(groups); + params->ft_attr.max_fte = mlx5_fs_ttc_table_size(ttc->groups); ttc->t = mlx5_create_flow_table(ns, ¶ms->ft_attr); if (IS_ERR(ttc->t)) { err = PTR_ERR(ttc->t); @@ -750,7 +903,7 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, return ERR_PTR(err); } - err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer, groups); + err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer); if (err) goto destroy_ft; @@ -758,6 +911,9 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev, if (err) goto destroy_ft; + ttc->mdev = dev; + mutex_init(&ttc->mutex); + return ttc; destroy_ft: @@ -791,3 +947,194 @@ int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc, return mlx5_ttc_fwd_dest(ttc, type, &dest); } + +static void _mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc) +{ + enum mlx5_traffic_types i; + + for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) { + if (!ttc->rules[i].rule) + continue; + + mlx5_del_flow_rules(ttc->rules[i].rule); + ttc->rules[i].rule = NULL; + } +} + +void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc) +{ + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return; + + mutex_lock(&ttc->mutex); + if (--ttc->refcnt) + goto unlock; + + _mlx5_ttc_destroy_ipsec_rules(ttc); +unlock: + mutex_unlock(&ttc->mutex); +} + +static int mlx5_ttc_get_tt_attrs(enum mlx5_traffic_types type, + u16 *etype, int *l4_type_ext, + enum mlx5_traffic_types *tir_tt) +{ + switch (type) { + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP: + *etype = ETH_P_IP; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP; + *tir_tt = MLX5_TT_IPV4_TCP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP: + *etype = ETH_P_IPV6; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP; + *tir_tt = MLX5_TT_IPV6_TCP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP: + *etype = ETH_P_IP; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP; + *tir_tt = MLX5_TT_IPV4_UDP; + break; + case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP: + case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP: + *etype = ETH_P_IPV6; + *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP; + *tir_tt = MLX5_TT_IPV6_UDP; + break; + default: + return -EINVAL; + } + + return 0; +} + +static struct mlx5_flow_handle * +mlx5_ttc_create_ipsec_outer_rule(struct mlx5_ttc_table *ttc, + enum mlx5_traffic_types type) +{ + struct mlx5_flow_destination dest; + MLX5_DECLARE_FLOW_ACT(flow_act); + enum mlx5_traffic_types tir_tt; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int l4_type_ext; + u16 etype; + int err; + + err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt); + if (err) + return ERR_PTR(err); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + mlx5_fs_ttc_set_match_ipv_outer(ttc->mdev, spec, etype); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.l4_type_ext); + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.l4_type_ext, l4_type_ext); + + dest = mlx5_ttc_get_default_dest(ttc, tir_tt); + + rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__); + } + + kvfree(spec); + return err ? ERR_PTR(err) : rule; +} + +static struct mlx5_flow_handle * +mlx5_ttc_create_ipsec_inner_rule(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc, + enum mlx5_traffic_types type) +{ + struct mlx5_flow_destination dest; + MLX5_DECLARE_FLOW_ACT(flow_act); + enum mlx5_traffic_types tir_tt; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int l4_type_ext; + u16 etype; + int err; + + err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt); + if (err) + return ERR_PTR(err); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + inner_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, + inner_headers.ip_version, mlx5_etype_to_ipv(etype)); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + inner_headers.l4_type_ext); + MLX5_SET(fte_match_param, spec->match_value, + inner_headers.l4_type_ext, l4_type_ext); + + dest = mlx5_ttc_get_default_dest(inner_ttc, tir_tt); + + spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS; + + rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__); + } + + kvfree(spec); + return err ? ERR_PTR(err) : rule; +} + +int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc) +{ + struct mlx5_flow_handle *rule; + enum mlx5_traffic_types i; + + if (!mlx5_ttc_has_esp_flow_group(ttc)) + return 0; + + mutex_lock(&ttc->mutex); + if (ttc->refcnt) + goto skip; + + for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP; i++) { + rule = mlx5_ttc_create_ipsec_outer_rule(ttc, i); + if (IS_ERR(rule)) + goto err_out; + + ttc->rules[i].rule = rule; + } + + for (i = MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP; + i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) { + rule = mlx5_ttc_create_ipsec_inner_rule(ttc, inner_ttc, i); + if (IS_ERR(rule)) + goto err_out; + + ttc->rules[i].rule = rule; + } + +skip: + ttc->refcnt++; + mutex_unlock(&ttc->mutex); + return 0; + +err_out: + _mlx5_ttc_destroy_ipsec_rules(ttc); + mutex_unlock(&ttc->mutex); + return PTR_ERR(rule); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h index ab9434fe3ae6..95f6e56724a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h @@ -18,6 +18,14 @@ enum mlx5_traffic_types { MLX5_TT_IPV4, MLX5_TT_IPV6, MLX5_TT_ANY, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP, + MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP, + MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP, MLX5_NUM_TT, MLX5_NUM_INDIR_TIRS = MLX5_TT_ANY, }; @@ -47,6 +55,7 @@ struct ttc_params { bool inner_ttc; DECLARE_BITMAP(ignore_tunnel_dests, MLX5_NUM_TUNNEL_TT); struct mlx5_flow_destination tunnel_dests[MLX5_NUM_TUNNEL_TT]; + bool ipsec_rss; }; const char *mlx5_ttc_get_name(enum mlx5_traffic_types tt); @@ -70,4 +79,14 @@ int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc, bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev); u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt); +bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc); +int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc, + struct mlx5_ttc_table *inner_ttc); +void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc); +static inline bool mlx5_ttc_is_decrypted_esp_tt(enum mlx5_traffic_types tt) +{ + return tt >= MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP && + tt <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; +} + #endif /* __MLX5_FS_TTC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c index b7d4b1a2baf2..d524f0220513 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c @@ -164,6 +164,8 @@ ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, roce->rule = rule; memset(spec, 0, sizeof(*spec)); + if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -178,6 +180,8 @@ ipsec_fs_roce_rx_rule_setup(struct mlx5_core_dev *mdev, goto out; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE) + flow_act.flags &= ~FLOW_ACT_IGNORE_FLOW_LEVEL; dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; dst.ft = roce->ft_rdma; rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst,