summaryrefslogtreecommitdiff
path: root/net/smc
diff options
context:
space:
mode:
Diffstat (limited to 'net/smc')
-rw-r--r--net/smc/Kconfig22
-rw-r--r--net/smc/Makefile2
-rw-r--r--net/smc/af_smc.c67
-rw-r--r--net/smc/smc.h4
-rw-r--r--net/smc/smc_clc.c76
-rw-r--r--net/smc/smc_core.c71
-rw-r--r--net/smc/smc_core.h13
-rw-r--r--net/smc/smc_diag.c2
-rw-r--r--net/smc/smc_hs_bpf.c140
-rw-r--r--net/smc/smc_hs_bpf.h31
-rw-r--r--net/smc/smc_ib.c31
-rw-r--r--net/smc/smc_inet.c13
-rw-r--r--net/smc/smc_ism.c233
-rw-r--r--net/smc/smc_ism.h36
-rw-r--r--net/smc/smc_llc.c2
-rw-r--r--net/smc/smc_loopback.c421
-rw-r--r--net/smc/smc_loopback.h60
-rw-r--r--net/smc/smc_pnet.c70
-rw-r--r--net/smc/smc_sysctl.c113
-rw-r--r--net/smc/smc_sysctl.h2
-rw-r--r--net/smc/smc_tx.c3
-rw-r--r--net/smc/smc_wr.c31
-rw-r--r--net/smc/smc_wr.h2
23 files changed, 659 insertions, 786 deletions
diff --git a/net/smc/Kconfig b/net/smc/Kconfig
index ba5e6a2dd2fd..325addf83cc6 100644
--- a/net/smc/Kconfig
+++ b/net/smc/Kconfig
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: GPL-2.0-only
config SMC
tristate "SMC socket protocol family"
- depends on INET && INFINIBAND
- depends on m || ISM != m
+ depends on INET && INFINIBAND && DIBS
help
SMC-R provides a "sockets over RDMA" solution making use of
RDMA over Converged Ethernet (RoCE) technology to upgrade
@@ -21,15 +20,12 @@ config SMC_DIAG
if unsure, say Y.
-config SMC_LO
- bool "SMC intra-OS shortcut with loopback-ism"
- depends on SMC
- default n
+config SMC_HS_CTRL_BPF
+ bool "Generic eBPF hook for SMC handshake flow"
+ depends on SMC && BPF_SYSCALL
+ default y
help
- SMC_LO enables the creation of an Emulated-ISM device named
- loopback-ism in SMC and makes use of it for transferring data
- when communication occurs within the same OS. This helps in
- convenient testing of SMC-D since loopback-ism is independent
- of architecture or hardware.
-
- if unsure, say N.
+ SMC_HS_CTRL_BPF enables support to register generic eBPF hook for SMC
+ handshake flow, which offer much greater flexibility in modifying the behavior
+ of the SMC protocol stack compared to a complete kernel-based approach. Select
+ this option if you want filtring the handshake process via eBPF programs. \ No newline at end of file
diff --git a/net/smc/Makefile b/net/smc/Makefile
index 60f1c87d5212..5368634c5dd6 100644
--- a/net/smc/Makefile
+++ b/net/smc/Makefile
@@ -6,4 +6,4 @@ smc-y := af_smc.o smc_pnet.o smc_ib.o smc_clc.o smc_core.o smc_wr.o smc_llc.o
smc-y += smc_cdc.o smc_tx.o smc_rx.o smc_close.o smc_ism.o smc_netlink.o smc_stats.o
smc-y += smc_tracepoint.o smc_inet.o
smc-$(CONFIG_SYSCTL) += smc_sysctl.o
-smc-$(CONFIG_SMC_LO) += smc_loopback.o
+smc-$(CONFIG_SMC_HS_CTRL_BPF) += smc_hs_bpf.o
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 9311c38f7abe..f97f77b041d9 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -16,8 +16,7 @@
* based on prototype from Frank Blaschka
*/
-#define KMSG_COMPONENT "smc"
-#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
+#define pr_fmt(fmt) "smc: " fmt
#include <linux/module.h>
#include <linux/socket.h>
@@ -57,8 +56,8 @@
#include "smc_stats.h"
#include "smc_tracepoint.h"
#include "smc_sysctl.h"
-#include "smc_loopback.h"
#include "smc_inet.h"
+#include "smc_hs_bpf.h"
static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
* creation on server
@@ -422,7 +421,7 @@ static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
return sk;
}
-int smc_bind(struct socket *sock, struct sockaddr *uaddr,
+int smc_bind(struct socket *sock, struct sockaddr_unsized *uaddr,
int addr_len)
{
struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
@@ -1097,8 +1096,7 @@ static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
}
/* Check for VLAN ID and register it on ISM device just for CLC handshake */
-static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
- struct smc_init_info *ini)
+static int smc_connect_ism_vlan_setup(struct smc_init_info *ini)
{
if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
return SMC_CLC_DECL_ISMVLANERR;
@@ -1113,7 +1111,7 @@ static int smc_find_proposal_devices(struct smc_sock *smc,
/* check if there is an ism device available */
if (!(ini->smcd_version & SMC_V1) ||
smc_find_ism_device(smc, ini) ||
- smc_connect_ism_vlan_setup(smc, ini))
+ smc_connect_ism_vlan_setup(ini))
ini->smcd_version &= ~SMC_V1;
/* else ISM V1 is supported for this connection */
@@ -1158,8 +1156,7 @@ static int smc_find_proposal_devices(struct smc_sock *smc,
/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
* used, the VLAN ID will be registered again during the connection setup.
*/
-static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
- struct smc_init_info *ini)
+static int smc_connect_ism_vlan_cleanup(struct smc_init_info *ini)
{
if (!smcd_indicated(ini->smc_type_v1))
return 0;
@@ -1582,13 +1579,13 @@ static int __smc_connect(struct smc_sock *smc)
goto vlan_cleanup;
SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
- smc_connect_ism_vlan_cleanup(smc, ini);
+ smc_connect_ism_vlan_cleanup(ini);
kfree(buf);
kfree(ini);
return 0;
vlan_cleanup:
- smc_connect_ism_vlan_cleanup(smc, ini);
+ smc_connect_ism_vlan_cleanup(ini);
kfree(buf);
fallback:
kfree(ini);
@@ -1645,7 +1642,7 @@ out:
release_sock(&smc->sk);
}
-int smc_connect(struct socket *sock, struct sockaddr *addr,
+int smc_connect(struct socket *sock, struct sockaddr_unsized *addr,
int alen, int flags)
{
struct sock *sk = sock->sk;
@@ -1697,7 +1694,7 @@ int smc_connect(struct socket *sock, struct sockaddr *addr,
rc = -EALREADY;
goto out;
}
- rc = kernel_connect(smc->clcsock, addr, alen, flags);
+ rc = kernel_connect(smc->clcsock, (struct sockaddr_unsized *)addr, alen, flags);
if (rc && rc != -EINPROGRESS)
goto out;
@@ -2143,7 +2140,7 @@ static void smc_check_ism_v2_match(struct smc_init_info *ini,
}
}
-static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
+static void smc_init_info_store_rc(u32 rc, struct smc_init_info *ini)
{
if (!ini->rc)
ini->rc = rc;
@@ -2206,7 +2203,7 @@ static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
mutex_unlock(&smcd_dev_list.mutex);
if (!ini->ism_dev[0]) {
- smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
+ smc_init_info_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
goto not_found;
}
@@ -2223,7 +2220,7 @@ static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
ini->ism_selected = i;
rc = smc_listen_ism_init(new_smc, ini);
if (rc) {
- smc_find_ism_store_rc(rc, ini);
+ smc_init_info_store_rc(rc, ini);
/* try next active ISM device */
continue;
}
@@ -2263,7 +2260,7 @@ static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
return; /* V1 ISM device found */
not_found:
- smc_find_ism_store_rc(rc, ini);
+ smc_init_info_store_rc(rc, ini);
ini->smcd_version &= ~SMC_V1;
ini->ism_dev[0] = NULL;
ini->is_smcd = false;
@@ -2314,7 +2311,7 @@ static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
rc = smc_find_rdma_device(new_smc, ini);
if (rc) {
- smc_find_ism_store_rc(rc, ini);
+ smc_init_info_store_rc(rc, ini);
goto not_found;
}
if (!ini->smcrv2.uses_gateway)
@@ -2331,7 +2328,7 @@ static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
if (!rc)
return;
ini->smcr_version = smcr_version;
- smc_find_ism_store_rc(rc, ini);
+ smc_init_info_store_rc(rc, ini);
not_found:
ini->smcr_version &= ~SMC_V2;
@@ -2378,7 +2375,7 @@ static int smc_listen_find_device(struct smc_sock *new_smc,
/* check for matching IP prefix and subnet length (V1) */
prfx_rc = smc_listen_prfx_check(new_smc, pclc);
if (prfx_rc)
- smc_find_ism_store_rc(prfx_rc, ini);
+ smc_init_info_store_rc(prfx_rc, ini);
/* get vlan id from IP device */
if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
@@ -2405,7 +2402,7 @@ static int smc_listen_find_device(struct smc_sock *new_smc,
int rc;
rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
- smc_find_ism_store_rc(rc, ini);
+ smc_init_info_store_rc(rc, ini);
return (!rc) ? 0 : ini->rc;
}
return prfx_rc;
@@ -2568,8 +2565,9 @@ static void smc_listen_work(struct work_struct *work)
goto out_decl;
}
- smc_listen_out_connected(new_smc);
SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
+ /* smc_listen_out() will release smcsk */
+ smc_listen_out_connected(new_smc);
goto out_free;
out_unlock:
@@ -3536,15 +3534,15 @@ static int __init smc_init(void)
rc = -ENOMEM;
- smc_tcp_ls_wq = alloc_workqueue("smc_tcp_ls_wq", 0, 0);
+ smc_tcp_ls_wq = alloc_workqueue("smc_tcp_ls_wq", WQ_PERCPU, 0);
if (!smc_tcp_ls_wq)
goto out_pnet;
- smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
+ smc_hs_wq = alloc_workqueue("smc_hs_wq", WQ_PERCPU, 0);
if (!smc_hs_wq)
goto out_alloc_tcp_ls_wq;
- smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
+ smc_close_wq = alloc_workqueue("smc_close_wq", WQ_PERCPU, 0);
if (!smc_close_wq)
goto out_alloc_hs_wq;
@@ -3592,28 +3590,28 @@ static int __init smc_init(void)
goto out_sock;
}
- rc = smc_loopback_init();
- if (rc) {
- pr_err("%s: smc_loopback_init fails with %d\n", __func__, rc);
- goto out_ib;
- }
-
rc = tcp_register_ulp(&smc_ulp_ops);
if (rc) {
pr_err("%s: tcp_ulp_register fails with %d\n", __func__, rc);
- goto out_lo;
+ goto out_ib;
}
rc = smc_inet_init();
if (rc) {
pr_err("%s: smc_inet_init fails with %d\n", __func__, rc);
goto out_ulp;
}
+ rc = bpf_smc_hs_ctrl_init();
+ if (rc) {
+ pr_err("%s: bpf_smc_hs_ctrl_init fails with %d\n", __func__,
+ rc);
+ goto out_inet;
+ }
static_branch_enable(&tcp_have_smc);
return 0;
+out_inet:
+ smc_inet_exit();
out_ulp:
tcp_unregister_ulp(&smc_ulp_ops);
-out_lo:
- smc_loopback_exit();
out_ib:
smc_ib_unregister_client();
out_sock:
@@ -3652,7 +3650,6 @@ static void __exit smc_exit(void)
tcp_unregister_ulp(&smc_ulp_ops);
sock_unregister(PF_SMC);
smc_core_exit();
- smc_loopback_exit();
smc_ib_unregister_client();
smc_ism_exit();
destroy_workqueue(smc_close_wq);
diff --git a/net/smc/smc.h b/net/smc/smc.h
index 2c9084963739..9e6af72784ba 100644
--- a/net/smc/smc.h
+++ b/net/smc/smc.h
@@ -42,9 +42,9 @@ void smc_unhash_sk(struct sock *sk);
void smc_release_cb(struct sock *sk);
int smc_release(struct socket *sock);
-int smc_bind(struct socket *sock, struct sockaddr *uaddr,
+int smc_bind(struct socket *sock, struct sockaddr_unsized *uaddr,
int addr_len);
-int smc_connect(struct socket *sock, struct sockaddr *addr,
+int smc_connect(struct socket *sock, struct sockaddr_unsized *addr,
int alen, int flags);
int smc_accept(struct socket *sock, struct socket *new_sock,
struct proto_accept_arg *arg);
diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c
index 5a4db151fe95..87c87edadde7 100644
--- a/net/smc/smc_clc.c
+++ b/net/smc/smc_clc.c
@@ -426,8 +426,6 @@ smc_clc_msg_decl_valid(struct smc_clc_msg_decline *dclc)
{
struct smc_clc_msg_hdr *hdr = &dclc->hdr;
- if (hdr->typev1 != SMC_TYPE_R && hdr->typev1 != SMC_TYPE_D)
- return false;
if (hdr->version == SMC_V1) {
if (ntohs(hdr->length) != sizeof(struct smc_clc_msg_decline))
return false;
@@ -511,10 +509,10 @@ static bool smc_clc_msg_hdr_valid(struct smc_clc_msg_hdr *clcm, bool check_trl)
}
/* find ipv4 addr on device and get the prefix len, fill CLC proposal msg */
-static int smc_clc_prfx_set4_rcu(struct dst_entry *dst, __be32 ipv4,
+static int smc_clc_prfx_set4_rcu(struct net_device *dev, __be32 ipv4,
struct smc_clc_msg_proposal_prefix *prop)
{
- struct in_device *in_dev = __in_dev_get_rcu(dst->dev);
+ struct in_device *in_dev = __in_dev_get_rcu(dev);
const struct in_ifaddr *ifa;
if (!in_dev)
@@ -532,12 +530,12 @@ static int smc_clc_prfx_set4_rcu(struct dst_entry *dst, __be32 ipv4,
}
/* fill CLC proposal msg with ipv6 prefixes from device */
-static int smc_clc_prfx_set6_rcu(struct dst_entry *dst,
+static int smc_clc_prfx_set6_rcu(struct net_device *dev,
struct smc_clc_msg_proposal_prefix *prop,
struct smc_clc_ipv6_prefix *ipv6_prfx)
{
#if IS_ENABLED(CONFIG_IPV6)
- struct inet6_dev *in6_dev = __in6_dev_get(dst->dev);
+ struct inet6_dev *in6_dev = __in6_dev_get(dev);
struct inet6_ifaddr *ifa;
int cnt = 0;
@@ -566,41 +564,44 @@ static int smc_clc_prfx_set(struct socket *clcsock,
struct smc_clc_msg_proposal_prefix *prop,
struct smc_clc_ipv6_prefix *ipv6_prfx)
{
- struct dst_entry *dst = sk_dst_get(clcsock->sk);
struct sockaddr_storage addrs;
struct sockaddr_in6 *addr6;
struct sockaddr_in *addr;
+ struct net_device *dev;
+ struct dst_entry *dst;
int rc = -ENOENT;
- if (!dst) {
- rc = -ENOTCONN;
- goto out;
- }
- if (!dst->dev) {
- rc = -ENODEV;
- goto out_rel;
- }
/* get address to which the internal TCP socket is bound */
if (kernel_getsockname(clcsock, (struct sockaddr *)&addrs) < 0)
- goto out_rel;
+ goto out;
+
/* analyze IP specific data of net_device belonging to TCP socket */
addr6 = (struct sockaddr_in6 *)&addrs;
+
rcu_read_lock();
+
+ dst = __sk_dst_get(clcsock->sk);
+ dev = dst ? dst_dev_rcu(dst) : NULL;
+ if (!dev) {
+ rc = -ENODEV;
+ goto out_unlock;
+ }
+
if (addrs.ss_family == PF_INET) {
/* IPv4 */
addr = (struct sockaddr_in *)&addrs;
- rc = smc_clc_prfx_set4_rcu(dst, addr->sin_addr.s_addr, prop);
+ rc = smc_clc_prfx_set4_rcu(dev, addr->sin_addr.s_addr, prop);
} else if (ipv6_addr_v4mapped(&addr6->sin6_addr)) {
/* mapped IPv4 address - peer is IPv4 only */
- rc = smc_clc_prfx_set4_rcu(dst, addr6->sin6_addr.s6_addr32[3],
+ rc = smc_clc_prfx_set4_rcu(dev, addr6->sin6_addr.s6_addr32[3],
prop);
} else {
/* IPv6 */
- rc = smc_clc_prfx_set6_rcu(dst, prop, ipv6_prfx);
+ rc = smc_clc_prfx_set6_rcu(dev, prop, ipv6_prfx);
}
+
+out_unlock:
rcu_read_unlock();
-out_rel:
- dst_release(dst);
out:
return rc;
}
@@ -656,26 +657,26 @@ static int smc_clc_prfx_match6_rcu(struct net_device *dev,
int smc_clc_prfx_match(struct socket *clcsock,
struct smc_clc_msg_proposal_prefix *prop)
{
- struct dst_entry *dst = sk_dst_get(clcsock->sk);
+ struct net_device *dev;
+ struct dst_entry *dst;
int rc;
- if (!dst) {
- rc = -ENOTCONN;
- goto out;
- }
- if (!dst->dev) {
+ rcu_read_lock();
+
+ dst = __sk_dst_get(clcsock->sk);
+ dev = dst ? dst_dev_rcu(dst) : NULL;
+ if (!dev) {
rc = -ENODEV;
- goto out_rel;
+ goto out;
}
- rcu_read_lock();
+
if (!prop->ipv6_prefixes_cnt)
- rc = smc_clc_prfx_match4_rcu(dst->dev, prop);
+ rc = smc_clc_prfx_match4_rcu(dev, prop);
else
- rc = smc_clc_prfx_match6_rcu(dst->dev, prop);
- rcu_read_unlock();
-out_rel:
- dst_release(dst);
+ rc = smc_clc_prfx_match6_rcu(dev, prop);
out:
+ rcu_read_unlock();
+
return rc;
}
@@ -889,6 +890,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
return SMC_CLC_DECL_CNFERR;
}
pclc_base->hdr.typev1 = SMC_TYPE_N;
+ ini->smc_type_v1 = SMC_TYPE_N;
} else {
pclc_base->iparea_offset = htons(sizeof(*pclc_smcd));
plen += sizeof(*pclc_prfx) +
@@ -915,7 +917,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
/* add SMC-D specifics */
if (ini->ism_dev[0]) {
smcd = ini->ism_dev[0];
- smcd->ops->get_local_gid(smcd, &smcd_gid);
+ copy_to_smcdgid(&smcd_gid, &smcd->dibs->gid);
pclc_smcd->ism.gid = htonll(smcd_gid.gid);
pclc_smcd->ism.chid =
htons(smc_ism_get_chid(ini->ism_dev[0]));
@@ -965,7 +967,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, struct smc_init_info *ini)
if (ini->ism_offered_cnt) {
for (i = 1; i <= ini->ism_offered_cnt; i++) {
smcd = ini->ism_dev[i];
- smcd->ops->get_local_gid(smcd, &smcd_gid);
+ copy_to_smcdgid(&smcd_gid, &smcd->dibs->gid);
gidchids[entry].chid =
htons(smc_ism_get_chid(ini->ism_dev[i]));
gidchids[entry].gid = htonll(smcd_gid.gid);
@@ -1058,7 +1060,7 @@ smcd_clc_prep_confirm_accept(struct smc_connection *conn,
/* SMC-D specific settings */
memcpy(clc->hdr.eyecatcher, SMCD_EYECATCHER,
sizeof(SMCD_EYECATCHER));
- smcd->ops->get_local_gid(smcd, &smcd_gid);
+ copy_to_smcdgid(&smcd_gid, &smcd->dibs->gid);
clc->hdr.typev1 = SMC_TYPE_D;
clc->d0.gid = htonll(smcd_gid.gid);
clc->d0.token = htonll(conn->rmb_desc->token);
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index 262746e304dd..e4eabc83719e 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -85,7 +85,7 @@ static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
* otherwise there is a risk of out-of-sync link groups.
*/
if (!lgr->freeing) {
- mod_delayed_work(system_wq, &lgr->free_work,
+ mod_delayed_work(system_percpu_wq, &lgr->free_work,
(!lgr->is_smcd && lgr->role == SMC_CLNT) ?
SMC_LGR_FREE_DELAY_CLNT :
SMC_LGR_FREE_DELAY_SERV);
@@ -555,7 +555,7 @@ static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr,
if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id)))
goto errattr;
- smcd->ops->get_local_gid(smcd, &smcd_gid);
+ copy_to_smcdgid(&smcd_gid, &smcd->dibs->gid);
if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_GID,
smcd_gid.gid, SMC_NLA_LGR_D_PAD))
goto errattr;
@@ -810,6 +810,8 @@ int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,
lnk->clearing = 0;
lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu;
lnk->link_id = smcr_next_link_id(lgr);
+ lnk->max_send_wr = lgr->max_send_wr;
+ lnk->max_recv_wr = lgr->max_recv_wr;
lnk->lgr = lgr;
smc_lgr_hold(lgr); /* lgr_put in smcr_link_clear() */
lnk->link_idx = link_idx;
@@ -836,27 +838,39 @@ int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,
rc = smc_llc_link_init(lnk);
if (rc)
goto out;
- rc = smc_wr_alloc_link_mem(lnk);
- if (rc)
- goto clear_llc_lnk;
rc = smc_ib_create_protection_domain(lnk);
if (rc)
- goto free_link_mem;
- rc = smc_ib_create_queue_pair(lnk);
- if (rc)
- goto dealloc_pd;
+ goto clear_llc_lnk;
+ do {
+ rc = smc_ib_create_queue_pair(lnk);
+ if (rc)
+ goto dealloc_pd;
+ rc = smc_wr_alloc_link_mem(lnk);
+ if (!rc)
+ break;
+ else if (rc != -ENOMEM) /* give up */
+ goto destroy_qp;
+ /* retry with smaller ... */
+ lnk->max_send_wr /= 2;
+ lnk->max_recv_wr /= 2;
+ /* ... unless droping below old SMC_WR_BUF_SIZE */
+ if (lnk->max_send_wr < 16 || lnk->max_recv_wr < 48)
+ goto destroy_qp;
+ smc_ib_destroy_queue_pair(lnk);
+ } while (1);
+
rc = smc_wr_create_link(lnk);
if (rc)
- goto destroy_qp;
+ goto free_link_mem;
lnk->state = SMC_LNK_ACTIVATING;
return 0;
+free_link_mem:
+ smc_wr_free_link_mem(lnk);
destroy_qp:
smc_ib_destroy_queue_pair(lnk);
dealloc_pd:
smc_ib_dealloc_protection_domain(lnk);
-free_link_mem:
- smc_wr_free_link_mem(lnk);
clear_llc_lnk:
smc_llc_link_clear(lnk, false);
out:
@@ -896,7 +910,7 @@ static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini)
rc = SMC_CLC_DECL_MEM;
goto ism_put_vlan;
}
- lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0,
+ lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", WQ_PERCPU, 0,
SMC_LGR_ID_SIZE, &lgr->id);
if (!lgr->tx_wq) {
rc = -ENOMEM;
@@ -924,7 +938,7 @@ static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini)
if (ini->is_smcd) {
/* SMC-D specific settings */
smcd = ini->ism_dev[ini->ism_selected];
- get_device(smcd->ops->get_dev(smcd));
+ get_device(&smcd->dibs->dev);
lgr->peer_gid.gid =
ini->ism_peer_gid[ini->ism_selected].gid;
lgr->peer_gid.gid_ext =
@@ -1474,7 +1488,7 @@ static void smc_lgr_free(struct smc_link_group *lgr)
destroy_workqueue(lgr->tx_wq);
if (lgr->is_smcd) {
smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);
- put_device(lgr->smcd->ops->get_dev(lgr->smcd));
+ put_device(&lgr->smcd->dibs->dev);
}
smc_lgr_put(lgr); /* theoretically last lgr_put */
}
@@ -1883,35 +1897,32 @@ static int smc_vlan_by_tcpsk_walk(struct net_device *lower_dev,
/* Determine vlan of internal TCP socket. */
int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini)
{
- struct dst_entry *dst = sk_dst_get(clcsock->sk);
struct netdev_nested_priv priv;
struct net_device *ndev;
+ struct dst_entry *dst;
int rc = 0;
ini->vlan_id = 0;
- if (!dst) {
- rc = -ENOTCONN;
- goto out;
- }
- if (!dst->dev) {
+
+ rcu_read_lock();
+
+ dst = __sk_dst_get(clcsock->sk);
+ ndev = dst ? dst_dev_rcu(dst) : NULL;
+ if (!ndev) {
rc = -ENODEV;
- goto out_rel;
+ goto out;
}
- ndev = dst->dev;
if (is_vlan_dev(ndev)) {
ini->vlan_id = vlan_dev_vlan_id(ndev);
- goto out_rel;
+ goto out;
}
priv.data = (void *)&ini->vlan_id;
- rtnl_lock();
- netdev_walk_all_lower_dev(ndev, smc_vlan_by_tcpsk_walk, &priv);
- rtnl_unlock();
-
-out_rel:
- dst_release(dst);
+ netdev_walk_all_lower_dev_rcu(ndev, smc_vlan_by_tcpsk_walk, &priv);
out:
+ rcu_read_unlock();
+
return rc;
}
diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h
index 48a1b1dcb576..5c18f08a4c8a 100644
--- a/net/smc/smc_core.h
+++ b/net/smc/smc_core.h
@@ -13,6 +13,7 @@
#define _SMC_CORE_H
#include <linux/atomic.h>
+#include <linux/types.h>
#include <linux/smc.h>
#include <linux/pci.h>
#include <rdma/ib_verbs.h>
@@ -33,6 +34,8 @@
* distributions may modify it to a value between
* 16-255 as needed.
*/
+#define SMCR_MAX_SEND_WR_DEF 16 /* Default number of work requests per send queue */
+#define SMCR_MAX_RECV_WR_DEF 48 /* Default number of work requests per recv queue */
struct smc_lgr_list { /* list of link group definition */
struct list_head list;
@@ -172,6 +175,8 @@ struct smc_link {
struct completion llc_testlink_resp; /* wait for rx of testlink */
int llc_testlink_time; /* testlink interval */
atomic_t conn_cnt; /* connections on this link */
+ u16 max_send_wr;
+ u16 max_recv_wr;
};
/* For now we just allow one parallel link per link group. The SMC protocol
@@ -221,6 +226,10 @@ struct smc_buf_desc {
/* virtually contiguous */
};
struct { /* SMC-D */
+ /* SMC-D tx buffer */
+ bool is_attached;
+ /* no need for explicit writes */
+ /* SMC-D rx buffer: */
unsigned short sba_idx;
/* SBA index number */
u64 token;
@@ -361,6 +370,10 @@ struct smc_link_group {
/* max conn can be assigned to lgr */
u8 max_links;
/* max links can be added in lgr */
+ u16 max_send_wr;
+ /* number of WR buffers on send */
+ u16 max_recv_wr;
+ /* number of WR buffers on recv */
};
struct { /* SMC-D */
struct smcd_gid peer_gid;
diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c
index 8ed2f6689b01..bf0beaa23bdb 100644
--- a/net/smc/smc_diag.c
+++ b/net/smc/smc_diag.c
@@ -175,7 +175,7 @@ static int __smc_diag_dump(struct sock *sk, struct sk_buff *skb,
dinfo.linkid = *((u32 *)conn->lgr->id);
dinfo.peer_gid = conn->lgr->peer_gid.gid;
dinfo.peer_gid_ext = conn->lgr->peer_gid.gid_ext;
- smcd->ops->get_local_gid(smcd, &smcd_gid);
+ copy_to_smcdgid(&smcd_gid, &smcd->dibs->gid);
dinfo.my_gid = smcd_gid.gid;
dinfo.my_gid_ext = smcd_gid.gid_ext;
dinfo.token = conn->rmb_desc->token;
diff --git a/net/smc/smc_hs_bpf.c b/net/smc/smc_hs_bpf.c
new file mode 100644
index 000000000000..063d23d85850
--- /dev/null
+++ b/net/smc/smc_hs_bpf.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Shared Memory Communications over RDMA (SMC-R) and RoCE
+ *
+ * Generic hook for SMC handshake flow.
+ *
+ * Copyright IBM Corp. 2016
+ * Copyright (c) 2025, Alibaba Inc.
+ *
+ * Author: D. Wythe <alibuda@linux.alibaba.com>
+ */
+
+#include <linux/bpf_verifier.h>
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/rculist.h>
+
+#include "smc_hs_bpf.h"
+
+static DEFINE_SPINLOCK(smc_hs_ctrl_list_lock);
+static LIST_HEAD(smc_hs_ctrl_list);
+
+static int smc_hs_ctrl_reg(struct smc_hs_ctrl *ctrl)
+{
+ int ret = 0;
+
+ spin_lock(&smc_hs_ctrl_list_lock);
+ /* already exist or duplicate name */
+ if (smc_hs_ctrl_find_by_name(ctrl->name))
+ ret = -EEXIST;
+ else
+ list_add_tail_rcu(&ctrl->list, &smc_hs_ctrl_list);
+ spin_unlock(&smc_hs_ctrl_list_lock);
+ return ret;
+}
+
+static void smc_hs_ctrl_unreg(struct smc_hs_ctrl *ctrl)
+{
+ spin_lock(&smc_hs_ctrl_list_lock);
+ list_del_rcu(&ctrl->list);
+ spin_unlock(&smc_hs_ctrl_list_lock);
+
+ /* Ensure that all readers to complete */
+ synchronize_rcu();
+}
+
+struct smc_hs_ctrl *smc_hs_ctrl_find_by_name(const char *name)
+{
+ struct smc_hs_ctrl *ctrl;
+
+ list_for_each_entry_rcu(ctrl, &smc_hs_ctrl_list, list) {
+ if (strcmp(ctrl->name, name) == 0)
+ return ctrl;
+ }
+ return NULL;
+}
+
+static int __smc_bpf_stub_set_tcp_option(struct tcp_sock *tp) { return 1; }
+static int __smc_bpf_stub_set_tcp_option_cond(const struct tcp_sock *tp,
+ struct inet_request_sock *ireq)
+{
+ return 1;
+}
+
+static struct smc_hs_ctrl __smc_bpf_hs_ctrl = {
+ .syn_option = __smc_bpf_stub_set_tcp_option,
+ .synack_option = __smc_bpf_stub_set_tcp_option_cond,
+};
+
+static int smc_bpf_hs_ctrl_init(struct btf *btf) { return 0; }
+
+static int smc_bpf_hs_ctrl_reg(void *kdata, struct bpf_link *link)
+{
+ if (link)
+ return -EOPNOTSUPP;
+
+ return smc_hs_ctrl_reg(kdata);
+}
+
+static void smc_bpf_hs_ctrl_unreg(void *kdata, struct bpf_link *link)
+{
+ smc_hs_ctrl_unreg(kdata);
+}
+
+static int smc_bpf_hs_ctrl_init_member(const struct btf_type *t,
+ const struct btf_member *member,
+ void *kdata, const void *udata)
+{
+ const struct smc_hs_ctrl *u_ctrl;
+ struct smc_hs_ctrl *k_ctrl;
+ u32 moff;
+
+ u_ctrl = (const struct smc_hs_ctrl *)udata;
+ k_ctrl = (struct smc_hs_ctrl *)kdata;
+
+ moff = __btf_member_bit_offset(t, member) / 8;
+ switch (moff) {
+ case offsetof(struct smc_hs_ctrl, name):
+ if (bpf_obj_name_cpy(k_ctrl->name, u_ctrl->name,
+ sizeof(u_ctrl->name)) <= 0)
+ return -EINVAL;
+ return 1;
+ case offsetof(struct smc_hs_ctrl, flags):
+ if (u_ctrl->flags & ~SMC_HS_CTRL_ALL_FLAGS)
+ return -EINVAL;
+ k_ctrl->flags = u_ctrl->flags;
+ return 1;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static const struct bpf_func_proto *
+bpf_smc_hs_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+ return bpf_base_func_proto(func_id, prog);
+}
+
+static const struct bpf_verifier_ops smc_bpf_verifier_ops = {
+ .get_func_proto = bpf_smc_hs_func_proto,
+ .is_valid_access = bpf_tracing_btf_ctx_access,
+};
+
+static struct bpf_struct_ops bpf_smc_hs_ctrl_ops = {
+ .name = "smc_hs_ctrl",
+ .init = smc_bpf_hs_ctrl_init,
+ .reg = smc_bpf_hs_ctrl_reg,
+ .unreg = smc_bpf_hs_ctrl_unreg,
+ .cfi_stubs = &__smc_bpf_hs_ctrl,
+ .verifier_ops = &smc_bpf_verifier_ops,
+ .init_member = smc_bpf_hs_ctrl_init_member,
+ .owner = THIS_MODULE,
+};
+
+int bpf_smc_hs_ctrl_init(void)
+{
+ return register_bpf_struct_ops(&bpf_smc_hs_ctrl_ops, smc_hs_ctrl);
+}
diff --git a/net/smc/smc_hs_bpf.h b/net/smc/smc_hs_bpf.h
new file mode 100644
index 000000000000..f5f1807c079e
--- /dev/null
+++ b/net/smc/smc_hs_bpf.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Shared Memory Communications over RDMA (SMC-R) and RoCE
+ *
+ * Generic hook for SMC handshake flow.
+ *
+ * Copyright IBM Corp. 2016
+ * Copyright (c) 2025, Alibaba Inc.
+ *
+ * Author: D. Wythe <alibuda@linux.alibaba.com>
+ */
+
+#ifndef __SMC_HS_CTRL
+#define __SMC_HS_CTRL
+
+#include <net/smc.h>
+
+/* Find hs_ctrl by the target name, which required to be a c-string.
+ * Return NULL if no such ctrl was found,otherwise, return a valid ctrl.
+ *
+ * Note: Caller MUST ensure it's was invoked under rcu_read_lock.
+ */
+struct smc_hs_ctrl *smc_hs_ctrl_find_by_name(const char *name);
+
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+int bpf_smc_hs_ctrl_init(void);
+#else
+static inline int bpf_smc_hs_ctrl_init(void) { return 0; }
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
+
+#endif /* __SMC_HS_CTRL */
diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
index 53828833a3f7..1154907c5c05 100644
--- a/net/smc/smc_ib.c
+++ b/net/smc/smc_ib.c
@@ -669,11 +669,6 @@ int smc_ib_create_queue_pair(struct smc_link *lnk)
.recv_cq = lnk->smcibdev->roce_cq_recv,
.srq = NULL,
.cap = {
- /* include unsolicited rdma_writes as well,
- * there are max. 2 RDMA_WRITE per 1 WR_SEND
- */
- .max_send_wr = SMC_WR_BUF_CNT * 3,
- .max_recv_wr = SMC_WR_BUF_CNT * 3,
.max_send_sge = SMC_IB_MAX_SEND_SGE,
.max_recv_sge = lnk->wr_rx_sge_cnt,
.max_inline_data = 0,
@@ -683,6 +678,11 @@ int smc_ib_create_queue_pair(struct smc_link *lnk)
};
int rc;
+ /* include unsolicited rdma_writes as well,
+ * there are max. 2 RDMA_WRITE per 1 WR_SEND
+ */
+ qp_attr.cap.max_send_wr = 3 * lnk->lgr->max_send_wr;
+ qp_attr.cap.max_recv_wr = lnk->lgr->max_recv_wr;
lnk->roce_qp = ib_create_qp(lnk->roce_pd, &qp_attr);
rc = PTR_ERR_OR_ZERO(lnk->roce_qp);
if (IS_ERR(lnk->roce_qp))
@@ -742,6 +742,9 @@ bool smc_ib_is_sg_need_sync(struct smc_link *lnk,
unsigned int i;
bool ret = false;
+ if (!lnk->smcibdev->ibdev->dma_device)
+ return ret;
+
/* for now there is just one DMA address */
for_each_sg(buf_slot->sgt[lnk->link_idx].sgl, sg,
buf_slot->sgt[lnk->link_idx].nents, i) {
@@ -971,13 +974,17 @@ static int smc_ib_add_dev(struct ib_device *ibdev)
smcibdev->pnetid[i]))
smc_pnetid_by_table_ib(smcibdev, i + 1);
smc_copy_netdev_ifindex(smcibdev, i);
- pr_warn_ratelimited("smc: ib device %s port %d has pnetid "
- "%.16s%s\n",
- smcibdev->ibdev->name, i + 1,
- smcibdev->pnetid[i],
- smcibdev->pnetid_by_user[i] ?
- " (user defined)" :
- "");
+ if (smc_pnet_is_pnetid_set(smcibdev->pnetid[i]))
+ pr_warn_ratelimited("smc: ib device %s port %d has pnetid %.16s%s\n",
+ smcibdev->ibdev->name, i + 1,
+ smcibdev->pnetid[i],
+ smcibdev->pnetid_by_user[i] ?
+ " (user defined)" :
+ "");
+ else
+ pr_warn_ratelimited("smc: ib device %s port %d has no pnetid\n",
+ smcibdev->ibdev->name, i + 1);
+
}
schedule_work(&smcibdev->port_event_work);
return 0;
diff --git a/net/smc/smc_inet.c b/net/smc/smc_inet.c
index a944e7dcb8b9..a94084b4a498 100644
--- a/net/smc/smc_inet.c
+++ b/net/smc/smc_inet.c
@@ -56,7 +56,6 @@ static struct inet_protosw smc_inet_protosw = {
.protocol = IPPROTO_SMC,
.prot = &smc_inet_prot,
.ops = &smc_inet_stream_ops,
- .flags = INET_PROTOSW_ICSK,
};
#if IS_ENABLED(CONFIG_IPV6)
@@ -104,27 +103,15 @@ static struct inet_protosw smc_inet6_protosw = {
.protocol = IPPROTO_SMC,
.prot = &smc_inet6_prot,
.ops = &smc_inet6_stream_ops,
- .flags = INET_PROTOSW_ICSK,
};
#endif /* CONFIG_IPV6 */
-static unsigned int smc_sync_mss(struct sock *sk, u32 pmtu)
-{
- /* No need pass it through to clcsock, mss can always be set by
- * sock_create_kern or smc_setsockopt.
- */
- return 0;
-}
-
static int smc_inet_init_sock(struct sock *sk)
{
struct net *net = sock_net(sk);
/* init common smc sock */
smc_sk_init(net, sk, IPPROTO_SMC);
-
- inet_csk(sk)->icsk_sync_mss = smc_sync_mss;
-
/* create clcsock */
return smc_create_clcsk(net, sk, sk->sk_family);
}
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index 84f98e18c7db..7b228ca2f96a 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -17,7 +17,7 @@
#include "smc_ism.h"
#include "smc_pnet.h"
#include "smc_netlink.h"
-#include "linux/ism.h"
+#include "linux/dibs.h"
struct smcd_dev_list smcd_dev_list = {
.list = LIST_HEAD_INIT(smcd_dev_list.list),
@@ -27,21 +27,24 @@ struct smcd_dev_list smcd_dev_list = {
static bool smc_ism_v2_capable;
static u8 smc_ism_v2_system_eid[SMC_MAX_EID_LEN];
-#if IS_ENABLED(CONFIG_ISM)
-static void smcd_register_dev(struct ism_dev *ism);
-static void smcd_unregister_dev(struct ism_dev *ism);
-static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event);
-static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
+static void smcd_register_dev(struct dibs_dev *dibs);
+static void smcd_unregister_dev(struct dibs_dev *dibs);
+static void smcd_handle_event(struct dibs_dev *dibs,
+ const struct dibs_event *event);
+static void smcd_handle_irq(struct dibs_dev *dibs, unsigned int dmbno,
u16 dmbemask);
-static struct ism_client smc_ism_client = {
- .name = "SMC-D",
- .add = smcd_register_dev,
- .remove = smcd_unregister_dev,
+static struct dibs_client_ops smc_client_ops = {
+ .add_dev = smcd_register_dev,
+ .del_dev = smcd_unregister_dev,
.handle_event = smcd_handle_event,
.handle_irq = smcd_handle_irq,
};
-#endif
+
+static struct dibs_client smc_dibs_client = {
+ .name = "SMC-D",
+ .ops = &smc_client_ops,
+};
static void smc_ism_create_system_eid(void)
{
@@ -68,8 +71,12 @@ static void smc_ism_create_system_eid(void)
int smc_ism_cantalk(struct smcd_gid *peer_gid, unsigned short vlan_id,
struct smcd_dev *smcd)
{
- return smcd->ops->query_remote_gid(smcd, peer_gid, vlan_id ? 1 : 0,
- vlan_id);
+ struct dibs_dev *dibs = smcd->dibs;
+ uuid_t ism_rgid;
+
+ copy_to_dibsgid(&ism_rgid, peer_gid);
+ return dibs->ops->query_remote_gid(dibs, &ism_rgid, vlan_id ? 1 : 0,
+ vlan_id);
}
void smc_ism_get_system_eid(u8 **eid)
@@ -82,7 +89,7 @@ void smc_ism_get_system_eid(u8 **eid)
u16 smc_ism_get_chid(struct smcd_dev *smcd)
{
- return smcd->ops->get_chid(smcd);
+ return smcd->dibs->ops->get_fabric_id(smcd->dibs);
}
/* HW supports ISM V2 and thus System EID is defined */
@@ -131,7 +138,7 @@ int smc_ism_get_vlan(struct smcd_dev *smcd, unsigned short vlanid)
if (!vlanid) /* No valid vlan id */
return -EINVAL;
- if (!smcd->ops->add_vlan_id)
+ if (!smcd->dibs->ops->add_vlan_id)
return -EOPNOTSUPP;
/* create new vlan entry, in case we need it */
@@ -154,7 +161,7 @@ int smc_ism_get_vlan(struct smcd_dev *smcd, unsigned short vlanid)
/* no existing entry found.
* add new entry to device; might fail, e.g., if HW limit reached
*/
- if (smcd->ops->add_vlan_id(smcd, vlanid)) {
+ if (smcd->dibs->ops->add_vlan_id(smcd->dibs, vlanid)) {
kfree(new_vlan);
rc = -EIO;
goto out;
@@ -178,7 +185,7 @@ int smc_ism_put_vlan(struct smcd_dev *smcd, unsigned short vlanid)
if (!vlanid) /* No valid vlan id */
return -EINVAL;
- if (!smcd->ops->del_vlan_id)
+ if (!smcd->dibs->ops->del_vlan_id)
return -EOPNOTSUPP;
spin_lock_irqsave(&smcd->lock, flags);
@@ -196,7 +203,7 @@ int smc_ism_put_vlan(struct smcd_dev *smcd, unsigned short vlanid)
}
/* Found and the last reference just gone */
- if (smcd->ops->del_vlan_id(smcd, vlanid))
+ if (smcd->dibs->ops->del_vlan_id(smcd->dibs, vlanid))
rc = -EIO;
list_del(&vlan->list);
kfree(vlan);
@@ -205,43 +212,43 @@ out:
return rc;
}
-int smc_ism_unregister_dmb(struct smcd_dev *smcd, struct smc_buf_desc *dmb_desc)
+void smc_ism_unregister_dmb(struct smcd_dev *smcd,
+ struct smc_buf_desc *dmb_desc)
{
- struct smcd_dmb dmb;
- int rc = 0;
+ struct dibs_dmb dmb;
if (!dmb_desc->dma_addr)
- return rc;
+ return;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_tok = dmb_desc->token;
- dmb.sba_idx = dmb_desc->sba_idx;
+ dmb.idx = dmb_desc->sba_idx;
dmb.cpu_addr = dmb_desc->cpu_addr;
dmb.dma_addr = dmb_desc->dma_addr;
dmb.dmb_len = dmb_desc->len;
- rc = smcd->ops->unregister_dmb(smcd, &dmb);
- if (!rc || rc == ISM_ERROR) {
- dmb_desc->cpu_addr = NULL;
- dmb_desc->dma_addr = 0;
- }
- return rc;
+ smcd->dibs->ops->unregister_dmb(smcd->dibs, &dmb);
+
+ return;
}
int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
struct smc_buf_desc *dmb_desc)
{
- struct smcd_dmb dmb;
+ struct dibs_dev *dibs;
+ struct dibs_dmb dmb;
int rc;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_len = dmb_len;
- dmb.sba_idx = dmb_desc->sba_idx;
+ dmb.idx = dmb_desc->sba_idx;
dmb.vlan_id = lgr->vlan_id;
- dmb.rgid = lgr->peer_gid.gid;
- rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, lgr->smcd->client);
+ copy_to_dibsgid(&dmb.rgid, &lgr->peer_gid);
+
+ dibs = lgr->smcd->dibs;
+ rc = dibs->ops->register_dmb(dibs, &dmb, &smc_dibs_client);
if (!rc) {
- dmb_desc->sba_idx = dmb.sba_idx;
+ dmb_desc->sba_idx = dmb.idx;
dmb_desc->token = dmb.dmb_tok;
dmb_desc->cpu_addr = dmb.cpu_addr;
dmb_desc->dma_addr = dmb.dma_addr;
@@ -256,38 +263,39 @@ bool smc_ism_support_dmb_nocopy(struct smcd_dev *smcd)
* merging sndbuf with peer DMB to avoid
* data copies between them.
*/
- return (smcd->ops->support_dmb_nocopy &&
- smcd->ops->support_dmb_nocopy(smcd));
+ return (smcd->dibs->ops->support_mmapped_rdmb &&
+ smcd->dibs->ops->support_mmapped_rdmb(smcd->dibs));
}
int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
struct smc_buf_desc *dmb_desc)
{
- struct smcd_dmb dmb;
+ struct dibs_dmb dmb;
int rc = 0;
- if (!dev->ops->attach_dmb)
+ if (!dev->dibs->ops->attach_dmb)
return -EINVAL;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_tok = token;
- rc = dev->ops->attach_dmb(dev, &dmb);
+ rc = dev->dibs->ops->attach_dmb(dev->dibs, &dmb);
if (!rc) {
- dmb_desc->sba_idx = dmb.sba_idx;
+ dmb_desc->sba_idx = dmb.idx;
dmb_desc->token = dmb.dmb_tok;
dmb_desc->cpu_addr = dmb.cpu_addr;
dmb_desc->dma_addr = dmb.dma_addr;
dmb_desc->len = dmb.dmb_len;
+ dmb_desc->is_attached = true;
}
return rc;
}
int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token)
{
- if (!dev->ops->detach_dmb)
+ if (!dev->dibs->ops->detach_dmb)
return -EINVAL;
- return dev->ops->detach_dmb(dev, token);
+ return dev->dibs->ops->detach_dmb(dev->dibs, token);
}
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
@@ -297,12 +305,12 @@ static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
char smc_pnet[SMC_MAX_PNETID_LEN + 1];
struct smc_pci_dev smc_pci_dev;
struct nlattr *port_attrs;
+ struct dibs_dev *dibs;
struct nlattr *attrs;
- struct ism_dev *ism;
int use_cnt = 0;
void *nlh;
- ism = smcd->priv;
+ dibs = smcd->dibs;
nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
&smc_gen_nl_family, NLM_F_MULTI,
SMC_NETLINK_GET_DEV_SMCD);
@@ -317,7 +325,7 @@ static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
if (nla_put_u8(skb, SMC_NLA_DEV_IS_CRIT, use_cnt > 0))
goto errattr;
memset(&smc_pci_dev, 0, sizeof(smc_pci_dev));
- smc_set_pci_values(to_pci_dev(ism->dev.parent), &smc_pci_dev);
+ smc_set_pci_values(to_pci_dev(dibs->dev.parent), &smc_pci_dev);
if (nla_put_u32(skb, SMC_NLA_DEV_PCI_FID, smc_pci_dev.pci_fid))
goto errattr;
if (nla_put_u16(skb, SMC_NLA_DEV_PCI_CHID, smc_pci_dev.pci_pchid))
@@ -367,7 +375,7 @@ static void smc_nl_prep_smcd_dev(struct smcd_dev_list *dev_list,
list_for_each_entry(smcd, &dev_list->list, list) {
if (num < snum)
goto next;
- if (smc_ism_is_loopback(smcd))
+ if (smc_ism_is_loopback(smcd->dibs))
goto next;
if (smc_nl_handle_smcd_dev(smcd, skb, cb))
goto errout;
@@ -385,11 +393,10 @@ int smcd_nl_get_device(struct sk_buff *skb, struct netlink_callback *cb)
return skb->len;
}
-#if IS_ENABLED(CONFIG_ISM)
struct smc_ism_event_work {
struct work_struct work;
struct smcd_dev *smcd;
- struct ism_event event;
+ struct dibs_event event;
};
#define ISM_EVENT_REQUEST 0x0001
@@ -409,25 +416,27 @@ union smcd_sw_event_info {
static void smcd_handle_sw_event(struct smc_ism_event_work *wrk)
{
- struct smcd_gid peer_gid = { .gid = wrk->event.tok,
- .gid_ext = 0 };
+ struct dibs_dev *dibs = wrk->smcd->dibs;
union smcd_sw_event_info ev_info;
+ struct smcd_gid peer_gid;
+ uuid_t ism_rgid;
- ev_info.info = wrk->event.info;
- switch (wrk->event.code) {
+ copy_to_smcdgid(&peer_gid, &wrk->event.gid);
+ ev_info.info = wrk->event.data;
+ switch (wrk->event.subtype) {
case ISM_EVENT_CODE_SHUTDOWN: /* Peer shut down DMBs */
smc_smcd_terminate(wrk->smcd, &peer_gid, ev_info.vlan_id);
break;
case ISM_EVENT_CODE_TESTLINK: /* Activity timer */
if (ev_info.code == ISM_EVENT_REQUEST &&
- wrk->smcd->ops->signal_event) {
+ dibs->ops->signal_event) {
ev_info.code = ISM_EVENT_RESPONSE;
- wrk->smcd->ops->signal_event(wrk->smcd,
- &peer_gid,
- ISM_EVENT_REQUEST_IR,
- ISM_EVENT_CODE_TESTLINK,
- ev_info.info);
- }
+ copy_to_dibsgid(&ism_rgid, &peer_gid);
+ dibs->ops->signal_event(dibs, &ism_rgid,
+ ISM_EVENT_REQUEST_IR,
+ ISM_EVENT_CODE_TESTLINK,
+ ev_info.info);
+ }
break;
}
}
@@ -437,41 +446,39 @@ static void smc_ism_event_work(struct work_struct *work)
{
struct smc_ism_event_work *wrk =
container_of(work, struct smc_ism_event_work, work);
- struct smcd_gid smcd_gid = { .gid = wrk->event.tok,
- .gid_ext = 0 };
+ struct smcd_gid smcd_gid;
+
+ copy_to_smcdgid(&smcd_gid, &wrk->event.gid);
switch (wrk->event.type) {
- case ISM_EVENT_GID: /* GID event, token is peer GID */
+ case DIBS_DEV_EVENT: /* GID event, token is peer GID */
smc_smcd_terminate(wrk->smcd, &smcd_gid, VLAN_VID_MASK);
break;
- case ISM_EVENT_DMB:
+ case DIBS_BUF_EVENT:
break;
- case ISM_EVENT_SWR: /* Software defined event */
+ case DIBS_SW_EVENT: /* Software defined event */
smcd_handle_sw_event(wrk);
break;
}
kfree(wrk);
}
-static struct smcd_dev *smcd_alloc_dev(struct device *parent, const char *name,
- const struct smcd_ops *ops, int max_dmbs)
+static struct smcd_dev *smcd_alloc_dev(const char *name, int max_dmbs)
{
struct smcd_dev *smcd;
- smcd = devm_kzalloc(parent, sizeof(*smcd), GFP_KERNEL);
+ smcd = kzalloc(sizeof(*smcd), GFP_KERNEL);
if (!smcd)
return NULL;
- smcd->conn = devm_kcalloc(parent, max_dmbs,
- sizeof(struct smc_connection *), GFP_KERNEL);
+ smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *),
+ GFP_KERNEL);
if (!smcd->conn)
- return NULL;
+ goto free_smcd;
smcd->event_wq = alloc_ordered_workqueue("ism_evt_wq-%s)",
WQ_MEM_RECLAIM, name);
if (!smcd->event_wq)
- return NULL;
-
- smcd->ops = ops;
+ goto free_conn;
spin_lock_init(&smcd->lock);
spin_lock_init(&smcd->lgr_lock);
@@ -479,28 +486,37 @@ static struct smcd_dev *smcd_alloc_dev(struct device *parent, const char *name,
INIT_LIST_HEAD(&smcd->lgr_list);
init_waitqueue_head(&smcd->lgrs_deleted);
return smcd;
+
+free_conn:
+ kfree(smcd->conn);
+free_smcd:
+ kfree(smcd);
+ return NULL;
}
-static void smcd_register_dev(struct ism_dev *ism)
+static void smcd_register_dev(struct dibs_dev *dibs)
{
- const struct smcd_ops *ops = ism_get_smcd_ops();
struct smcd_dev *smcd, *fentry;
+ int max_dmbs;
- if (!ops)
- return;
+ max_dmbs = dibs->ops->max_dmbs();
- smcd = smcd_alloc_dev(&ism->pdev->dev, dev_name(&ism->pdev->dev), ops,
- ISM_NR_DMBS);
+ smcd = smcd_alloc_dev(dev_name(&dibs->dev), max_dmbs);
if (!smcd)
return;
- smcd->priv = ism;
- smcd->client = &smc_ism_client;
- ism_set_priv(ism, &smc_ism_client, smcd);
- if (smc_pnetid_by_dev_port(&ism->pdev->dev, 0, smcd->pnetid))
+
+ smcd->dibs = dibs;
+ dibs_set_priv(dibs, &smc_dibs_client, smcd);
+
+ if (smc_pnetid_by_dev_port(dibs->dev.parent, 0, smcd->pnetid))
smc_pnetid_by_table_smcd(smcd);
- if (smcd->ops->supports_v2())
+ if (smc_ism_is_loopback(dibs) ||
+ (dibs->ops->add_vlan_id &&
+ !dibs->ops->add_vlan_id(dibs, ISM_RESERVED_VLANID))) {
smc_ism_set_v2_capable();
+ }
+
mutex_lock(&smcd_dev_list.mutex);
/* sort list:
* - devices without pnetid before devices with pnetid;
@@ -509,7 +525,7 @@ static void smcd_register_dev(struct ism_dev *ism)
if (!smcd->pnetid[0]) {
fentry = list_first_entry_or_null(&smcd_dev_list.list,
struct smcd_dev, list);
- if (fentry && smc_ism_is_loopback(fentry))
+ if (fentry && smc_ism_is_loopback(fentry->dibs))
list_add(&smcd->list, &fentry->list);
else
list_add(&smcd->list, &smcd_dev_list.list);
@@ -518,25 +534,32 @@ static void smcd_register_dev(struct ism_dev *ism)
}
mutex_unlock(&smcd_dev_list.mutex);
- pr_warn_ratelimited("smc: adding smcd device %s with pnetid %.16s%s\n",
- dev_name(&ism->dev), smcd->pnetid,
- smcd->pnetid_by_user ? " (user defined)" : "");
-
+ if (smc_pnet_is_pnetid_set(smcd->pnetid))
+ pr_warn_ratelimited("smc: adding smcd device %s with pnetid %.16s%s\n",
+ dev_name(&dibs->dev), smcd->pnetid,
+ smcd->pnetid_by_user ?
+ " (user defined)" :
+ "");
+ else
+ pr_warn_ratelimited("smc: adding smcd device %s without pnetid\n",
+ dev_name(&dibs->dev));
return;
}
-static void smcd_unregister_dev(struct ism_dev *ism)
+static void smcd_unregister_dev(struct dibs_dev *dibs)
{
- struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client);
+ struct smcd_dev *smcd = dibs_get_priv(dibs, &smc_dibs_client);
pr_warn_ratelimited("smc: removing smcd device %s\n",
- dev_name(&ism->dev));
+ dev_name(&dibs->dev));
smcd->going_away = 1;
smc_smcd_terminate_all(smcd);
mutex_lock(&smcd_dev_list.mutex);
list_del_init(&smcd->list);
mutex_unlock(&smcd_dev_list.mutex);
destroy_workqueue(smcd->event_wq);
+ kfree(smcd->conn);
+ kfree(smcd);
}
/* SMCD Device event handler. Called from ISM device interrupt handler.
@@ -550,9 +573,10 @@ static void smcd_unregister_dev(struct ism_dev *ism)
* Context:
* - Function called in IRQ context from ISM device driver event handler.
*/
-static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event)
+static void smcd_handle_event(struct dibs_dev *dibs,
+ const struct dibs_event *event)
{
- struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client);
+ struct smcd_dev *smcd = dibs_get_priv(dibs, &smc_dibs_client);
struct smc_ism_event_work *wrk;
if (smcd->going_away)
@@ -574,10 +598,10 @@ static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event)
* Context:
* - Function called in IRQ context from ISM device driver IRQ handler.
*/
-static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
+static void smcd_handle_irq(struct dibs_dev *dibs, unsigned int dmbno,
u16 dmbemask)
{
- struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client);
+ struct smcd_dev *smcd = dibs_get_priv(dibs, &smc_dibs_client);
struct smc_connection *conn = NULL;
unsigned long flags;
@@ -587,27 +611,26 @@ static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
tasklet_schedule(&conn->rx_tsklet);
spin_unlock_irqrestore(&smcd->lock, flags);
}
-#endif
int smc_ism_signal_shutdown(struct smc_link_group *lgr)
{
int rc = 0;
-#if IS_ENABLED(CONFIG_ISM)
union smcd_sw_event_info ev_info;
+ uuid_t ism_rgid;
if (lgr->peer_shutdown)
return 0;
- if (!lgr->smcd->ops->signal_event)
+ if (!lgr->smcd->dibs->ops->signal_event)
return 0;
memcpy(ev_info.uid, lgr->id, SMC_LGR_ID_SIZE);
ev_info.vlan_id = lgr->vlan_id;
ev_info.code = ISM_EVENT_REQUEST;
- rc = lgr->smcd->ops->signal_event(lgr->smcd, &lgr->peer_gid,
+ copy_to_dibsgid(&ism_rgid, &lgr->peer_gid);
+ rc = lgr->smcd->dibs->ops->signal_event(lgr->smcd->dibs, &ism_rgid,
ISM_EVENT_REQUEST_IR,
ISM_EVENT_CODE_SHUTDOWN,
ev_info.info);
-#endif
return rc;
}
@@ -618,15 +641,11 @@ int smc_ism_init(void)
smc_ism_v2_capable = false;
smc_ism_create_system_eid();
-#if IS_ENABLED(CONFIG_ISM)
- rc = ism_register_client(&smc_ism_client);
-#endif
+ rc = dibs_register_client(&smc_dibs_client);
return rc;
}
void smc_ism_exit(void)
{
-#if IS_ENABLED(CONFIG_ISM)
- ism_unregister_client(&smc_ism_client);
-#endif
+ dibs_unregister_client(&smc_dibs_client);
}
diff --git a/net/smc/smc_ism.h b/net/smc/smc_ism.h
index 6763133dd8d0..a1575e31df73 100644
--- a/net/smc/smc_ism.h
+++ b/net/smc/smc_ism.h
@@ -12,6 +12,7 @@
#include <linux/uio.h>
#include <linux/types.h>
#include <linux/mutex.h>
+#include <linux/dibs.h>
#include "smc.h"
@@ -47,7 +48,8 @@ int smc_ism_get_vlan(struct smcd_dev *dev, unsigned short vlan_id);
int smc_ism_put_vlan(struct smcd_dev *dev, unsigned short vlan_id);
int smc_ism_register_dmb(struct smc_link_group *lgr, int buf_size,
struct smc_buf_desc *dmb_desc);
-int smc_ism_unregister_dmb(struct smcd_dev *dev, struct smc_buf_desc *dmb_desc);
+void smc_ism_unregister_dmb(struct smcd_dev *dev,
+ struct smc_buf_desc *dmb_desc);
bool smc_ism_support_dmb_nocopy(struct smcd_dev *smcd);
int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
struct smc_buf_desc *dmb_desc);
@@ -67,7 +69,9 @@ static inline int smc_ism_write(struct smcd_dev *smcd, u64 dmb_tok,
{
int rc;
- rc = smcd->ops->move_data(smcd, dmb_tok, idx, sf, offset, data, len);
+ rc = smcd->dibs->ops->move_data(smcd->dibs, dmb_tok, idx, sf, offset,
+ data, len);
+
return rc < 0 ? rc : 0;
}
@@ -84,14 +88,36 @@ static inline bool __smc_ism_is_emulated(u16 chid)
static inline bool smc_ism_is_emulated(struct smcd_dev *smcd)
{
- u16 chid = smcd->ops->get_chid(smcd);
+ u16 chid = smcd->dibs->ops->get_fabric_id(smcd->dibs);
return __smc_ism_is_emulated(chid);
}
-static inline bool smc_ism_is_loopback(struct smcd_dev *smcd)
+static inline bool smc_ism_is_loopback(struct dibs_dev *dibs)
{
- return (smcd->ops->get_chid(smcd) == 0xFFFF);
+ return (dibs->ops->get_fabric_id(dibs) == DIBS_LOOPBACK_FABRIC);
+}
+
+static inline void copy_to_smcdgid(struct smcd_gid *sgid, uuid_t *dibs_gid)
+{
+ __be64 temp;
+
+ memcpy(&temp, dibs_gid, sizeof(sgid->gid));
+ sgid->gid = ntohll(temp);
+ memcpy(&temp, (uint8_t *)dibs_gid + sizeof(sgid->gid),
+ sizeof(sgid->gid_ext));
+ sgid->gid_ext = ntohll(temp);
+}
+
+static inline void copy_to_dibsgid(uuid_t *dibs_gid, struct smcd_gid *sgid)
+{
+ __be64 temp;
+
+ temp = htonll(sgid->gid);
+ memcpy(dibs_gid, &temp, sizeof(sgid->gid));
+ temp = htonll(sgid->gid_ext);
+ memcpy((uint8_t *)dibs_gid + sizeof(sgid->gid), &temp,
+ sizeof(sgid->gid_ext));
}
#endif
diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
index f865c58c3aa7..f5d5eb617526 100644
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -2157,6 +2157,8 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
init_waitqueue_head(&lgr->llc_msg_waiter);
init_rwsem(&lgr->llc_conf_mutex);
lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
+ lgr->max_send_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_send_wr));
+ lgr->max_recv_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_recv_wr));
}
/* called after lgr was removed from lgr_list */
diff --git a/net/smc/smc_loopback.c b/net/smc/smc_loopback.c
deleted file mode 100644
index 0eb00bbefd17..000000000000
--- a/net/smc/smc_loopback.c
+++ /dev/null
@@ -1,421 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Shared Memory Communications Direct over loopback-ism device.
- *
- * Functions for loopback-ism device.
- *
- * Copyright (c) 2024, Alibaba Inc.
- *
- * Author: Wen Gu <guwen@linux.alibaba.com>
- * Tony Lu <tonylu@linux.alibaba.com>
- *
- */
-
-#include <linux/device.h>
-#include <linux/types.h>
-#include <net/smc.h>
-
-#include "smc_cdc.h"
-#include "smc_ism.h"
-#include "smc_loopback.h"
-
-#define SMC_LO_V2_CAPABLE 0x1 /* loopback-ism acts as ISMv2 */
-#define SMC_LO_SUPPORT_NOCOPY 0x1
-#define SMC_DMA_ADDR_INVALID (~(dma_addr_t)0)
-
-static const char smc_lo_dev_name[] = "loopback-ism";
-static struct smc_lo_dev *lo_dev;
-
-static void smc_lo_generate_ids(struct smc_lo_dev *ldev)
-{
- struct smcd_gid *lgid = &ldev->local_gid;
- uuid_t uuid;
-
- uuid_gen(&uuid);
- memcpy(&lgid->gid, &uuid, sizeof(lgid->gid));
- memcpy(&lgid->gid_ext, (u8 *)&uuid + sizeof(lgid->gid),
- sizeof(lgid->gid_ext));
-
- ldev->chid = SMC_LO_RESERVED_CHID;
-}
-
-static int smc_lo_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid,
- u32 vid_valid, u32 vid)
-{
- struct smc_lo_dev *ldev = smcd->priv;
-
- /* rgid should be the same as lgid */
- if (!ldev || rgid->gid != ldev->local_gid.gid ||
- rgid->gid_ext != ldev->local_gid.gid_ext)
- return -ENETUNREACH;
- return 0;
-}
-
-static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
- void *client_priv)
-{
- struct smc_lo_dmb_node *dmb_node, *tmp_node;
- struct smc_lo_dev *ldev = smcd->priv;
- int sba_idx, rc;
-
- /* check space for new dmb */
- for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LO_MAX_DMBS) {
- if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
- break;
- }
- if (sba_idx == SMC_LO_MAX_DMBS)
- return -ENOSPC;
-
- dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
- if (!dmb_node) {
- rc = -ENOMEM;
- goto err_bit;
- }
-
- dmb_node->sba_idx = sba_idx;
- dmb_node->len = dmb->dmb_len;
- dmb_node->cpu_addr = kzalloc(dmb_node->len, GFP_KERNEL |
- __GFP_NOWARN | __GFP_NORETRY |
- __GFP_NOMEMALLOC);
- if (!dmb_node->cpu_addr) {
- rc = -ENOMEM;
- goto err_node;
- }
- dmb_node->dma_addr = SMC_DMA_ADDR_INVALID;
- refcount_set(&dmb_node->refcnt, 1);
-
-again:
- /* add new dmb into hash table */
- get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
- write_lock_bh(&ldev->dmb_ht_lock);
- hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
- if (tmp_node->token == dmb_node->token) {
- write_unlock_bh(&ldev->dmb_ht_lock);
- goto again;
- }
- }
- hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
- write_unlock_bh(&ldev->dmb_ht_lock);
- atomic_inc(&ldev->dmb_cnt);
-
- dmb->sba_idx = dmb_node->sba_idx;
- dmb->dmb_tok = dmb_node->token;
- dmb->cpu_addr = dmb_node->cpu_addr;
- dmb->dma_addr = dmb_node->dma_addr;
- dmb->dmb_len = dmb_node->len;
-
- return 0;
-
-err_node:
- kfree(dmb_node);
-err_bit:
- clear_bit(sba_idx, ldev->sba_idx_mask);
- return rc;
-}
-
-static void __smc_lo_unregister_dmb(struct smc_lo_dev *ldev,
- struct smc_lo_dmb_node *dmb_node)
-{
- /* remove dmb from hash table */
- write_lock_bh(&ldev->dmb_ht_lock);
- hash_del(&dmb_node->list);
- write_unlock_bh(&ldev->dmb_ht_lock);
-
- clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
- kvfree(dmb_node->cpu_addr);
- kfree(dmb_node);
-
- if (atomic_dec_and_test(&ldev->dmb_cnt))
- wake_up(&ldev->ldev_release);
-}
-
-static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
-{
- struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
- struct smc_lo_dev *ldev = smcd->priv;
-
- /* find dmb from hash table */
- read_lock_bh(&ldev->dmb_ht_lock);
- hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
- if (tmp_node->token == dmb->dmb_tok) {
- dmb_node = tmp_node;
- break;
- }
- }
- if (!dmb_node) {
- read_unlock_bh(&ldev->dmb_ht_lock);
- return -EINVAL;
- }
- read_unlock_bh(&ldev->dmb_ht_lock);
-
- if (refcount_dec_and_test(&dmb_node->refcnt))
- __smc_lo_unregister_dmb(ldev, dmb_node);
- return 0;
-}
-
-static int smc_lo_support_dmb_nocopy(struct smcd_dev *smcd)
-{
- return SMC_LO_SUPPORT_NOCOPY;
-}
-
-static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
-{
- struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
- struct smc_lo_dev *ldev = smcd->priv;
-
- /* find dmb_node according to dmb->dmb_tok */
- read_lock_bh(&ldev->dmb_ht_lock);
- hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
- if (tmp_node->token == dmb->dmb_tok) {
- dmb_node = tmp_node;
- break;
- }
- }
- if (!dmb_node) {
- read_unlock_bh(&ldev->dmb_ht_lock);
- return -EINVAL;
- }
- read_unlock_bh(&ldev->dmb_ht_lock);
-
- if (!refcount_inc_not_zero(&dmb_node->refcnt))
- /* the dmb is being unregistered, but has
- * not been removed from the hash table.
- */
- return -EINVAL;
-
- /* provide dmb information */
- dmb->sba_idx = dmb_node->sba_idx;
- dmb->dmb_tok = dmb_node->token;
- dmb->cpu_addr = dmb_node->cpu_addr;
- dmb->dma_addr = dmb_node->dma_addr;
- dmb->dmb_len = dmb_node->len;
- return 0;
-}
-
-static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token)
-{
- struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
- struct smc_lo_dev *ldev = smcd->priv;
-
- /* find dmb_node according to dmb->dmb_tok */
- read_lock_bh(&ldev->dmb_ht_lock);
- hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
- if (tmp_node->token == token) {
- dmb_node = tmp_node;
- break;
- }
- }
- if (!dmb_node) {
- read_unlock_bh(&ldev->dmb_ht_lock);
- return -EINVAL;
- }
- read_unlock_bh(&ldev->dmb_ht_lock);
-
- if (refcount_dec_and_test(&dmb_node->refcnt))
- __smc_lo_unregister_dmb(ldev, dmb_node);
- return 0;
-}
-
-static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok,
- unsigned int idx, bool sf, unsigned int offset,
- void *data, unsigned int size)
-{
- struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node;
- struct smc_lo_dev *ldev = smcd->priv;
- struct smc_connection *conn;
-
- if (!sf)
- /* since sndbuf is merged with peer DMB, there is
- * no need to copy data from sndbuf to peer DMB.
- */
- return 0;
-
- read_lock_bh(&ldev->dmb_ht_lock);
- hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
- if (tmp_node->token == dmb_tok) {
- rmb_node = tmp_node;
- break;
- }
- }
- if (!rmb_node) {
- read_unlock_bh(&ldev->dmb_ht_lock);
- return -EINVAL;
- }
- memcpy((char *)rmb_node->cpu_addr + offset, data, size);
- read_unlock_bh(&ldev->dmb_ht_lock);
-
- conn = smcd->conn[rmb_node->sba_idx];
- if (!conn || conn->killed)
- return -EPIPE;
- tasklet_schedule(&conn->rx_tsklet);
- return 0;
-}
-
-static void smc_lo_get_local_gid(struct smcd_dev *smcd,
- struct smcd_gid *smcd_gid)
-{
- struct smc_lo_dev *ldev = smcd->priv;
-
- smcd_gid->gid = ldev->local_gid.gid;
- smcd_gid->gid_ext = ldev->local_gid.gid_ext;
-}
-
-static u16 smc_lo_get_chid(struct smcd_dev *smcd)
-{
- return ((struct smc_lo_dev *)smcd->priv)->chid;
-}
-
-static struct device *smc_lo_get_dev(struct smcd_dev *smcd)
-{
- return &((struct smc_lo_dev *)smcd->priv)->dev;
-}
-
-static const struct smcd_ops lo_ops = {
- .query_remote_gid = smc_lo_query_rgid,
- .register_dmb = smc_lo_register_dmb,
- .unregister_dmb = smc_lo_unregister_dmb,
- .support_dmb_nocopy = smc_lo_support_dmb_nocopy,
- .attach_dmb = smc_lo_attach_dmb,
- .detach_dmb = smc_lo_detach_dmb,
- .add_vlan_id = NULL,
- .del_vlan_id = NULL,
- .set_vlan_required = NULL,
- .reset_vlan_required = NULL,
- .signal_event = NULL,
- .move_data = smc_lo_move_data,
- .get_local_gid = smc_lo_get_local_gid,
- .get_chid = smc_lo_get_chid,
- .get_dev = smc_lo_get_dev,
-};
-
-static struct smcd_dev *smcd_lo_alloc_dev(const struct smcd_ops *ops,
- int max_dmbs)
-{
- struct smcd_dev *smcd;
-
- smcd = kzalloc(sizeof(*smcd), GFP_KERNEL);
- if (!smcd)
- return NULL;
-
- smcd->conn = kcalloc(max_dmbs, sizeof(struct smc_connection *),
- GFP_KERNEL);
- if (!smcd->conn)
- goto out_smcd;
-
- smcd->ops = ops;
-
- spin_lock_init(&smcd->lock);
- spin_lock_init(&smcd->lgr_lock);
- INIT_LIST_HEAD(&smcd->vlan);
- INIT_LIST_HEAD(&smcd->lgr_list);
- init_waitqueue_head(&smcd->lgrs_deleted);
- return smcd;
-
-out_smcd:
- kfree(smcd);
- return NULL;
-}
-
-static int smcd_lo_register_dev(struct smc_lo_dev *ldev)
-{
- struct smcd_dev *smcd;
-
- smcd = smcd_lo_alloc_dev(&lo_ops, SMC_LO_MAX_DMBS);
- if (!smcd)
- return -ENOMEM;
- ldev->smcd = smcd;
- smcd->priv = ldev;
- smc_ism_set_v2_capable();
- mutex_lock(&smcd_dev_list.mutex);
- list_add(&smcd->list, &smcd_dev_list.list);
- mutex_unlock(&smcd_dev_list.mutex);
- pr_warn_ratelimited("smc: adding smcd device %s\n",
- dev_name(&ldev->dev));
- return 0;
-}
-
-static void smcd_lo_unregister_dev(struct smc_lo_dev *ldev)
-{
- struct smcd_dev *smcd = ldev->smcd;
-
- pr_warn_ratelimited("smc: removing smcd device %s\n",
- dev_name(&ldev->dev));
- smcd->going_away = 1;
- smc_smcd_terminate_all(smcd);
- mutex_lock(&smcd_dev_list.mutex);
- list_del_init(&smcd->list);
- mutex_unlock(&smcd_dev_list.mutex);
- kfree(smcd->conn);
- kfree(smcd);
-}
-
-static int smc_lo_dev_init(struct smc_lo_dev *ldev)
-{
- smc_lo_generate_ids(ldev);
- rwlock_init(&ldev->dmb_ht_lock);
- hash_init(ldev->dmb_ht);
- atomic_set(&ldev->dmb_cnt, 0);
- init_waitqueue_head(&ldev->ldev_release);
-
- return smcd_lo_register_dev(ldev);
-}
-
-static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
-{
- smcd_lo_unregister_dev(ldev);
- if (atomic_read(&ldev->dmb_cnt))
- wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
-}
-
-static void smc_lo_dev_release(struct device *dev)
-{
- struct smc_lo_dev *ldev =
- container_of(dev, struct smc_lo_dev, dev);
-
- kfree(ldev);
-}
-
-static int smc_lo_dev_probe(void)
-{
- struct smc_lo_dev *ldev;
- int ret;
-
- ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
- if (!ldev)
- return -ENOMEM;
-
- ldev->dev.parent = NULL;
- ldev->dev.release = smc_lo_dev_release;
- device_initialize(&ldev->dev);
- dev_set_name(&ldev->dev, smc_lo_dev_name);
-
- ret = smc_lo_dev_init(ldev);
- if (ret)
- goto free_dev;
-
- lo_dev = ldev; /* global loopback device */
- return 0;
-
-free_dev:
- put_device(&ldev->dev);
- return ret;
-}
-
-static void smc_lo_dev_remove(void)
-{
- if (!lo_dev)
- return;
-
- smc_lo_dev_exit(lo_dev);
- put_device(&lo_dev->dev); /* device_initialize in smc_lo_dev_probe */
-}
-
-int smc_loopback_init(void)
-{
- return smc_lo_dev_probe();
-}
-
-void smc_loopback_exit(void)
-{
- smc_lo_dev_remove();
-}
diff --git a/net/smc/smc_loopback.h b/net/smc/smc_loopback.h
deleted file mode 100644
index 04dc6808d2e1..000000000000
--- a/net/smc/smc_loopback.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Shared Memory Communications Direct over loopback-ism device.
- *
- * SMC-D loopback-ism device structure definitions.
- *
- * Copyright (c) 2024, Alibaba Inc.
- *
- * Author: Wen Gu <guwen@linux.alibaba.com>
- * Tony Lu <tonylu@linux.alibaba.com>
- *
- */
-
-#ifndef _SMC_LOOPBACK_H
-#define _SMC_LOOPBACK_H
-
-#include <linux/device.h>
-#include <net/smc.h>
-
-#if IS_ENABLED(CONFIG_SMC_LO)
-#define SMC_LO_MAX_DMBS 5000
-#define SMC_LO_DMBS_HASH_BITS 12
-#define SMC_LO_RESERVED_CHID 0xFFFF
-
-struct smc_lo_dmb_node {
- struct hlist_node list;
- u64 token;
- u32 len;
- u32 sba_idx;
- void *cpu_addr;
- dma_addr_t dma_addr;
- refcount_t refcnt;
-};
-
-struct smc_lo_dev {
- struct smcd_dev *smcd;
- struct device dev;
- u16 chid;
- struct smcd_gid local_gid;
- atomic_t dmb_cnt;
- rwlock_t dmb_ht_lock;
- DECLARE_BITMAP(sba_idx_mask, SMC_LO_MAX_DMBS);
- DECLARE_HASHTABLE(dmb_ht, SMC_LO_DMBS_HASH_BITS);
- wait_queue_head_t ldev_release;
-};
-
-int smc_loopback_init(void);
-void smc_loopback_exit(void);
-#else
-static inline int smc_loopback_init(void)
-{
- return 0;
-}
-
-static inline void smc_loopback_exit(void)
-{
-}
-#endif
-
-#endif /* _SMC_LOOPBACK_H */
diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c
index 76ad29e31d60..a3a1e1fde8eb 100644
--- a/net/smc/smc_pnet.c
+++ b/net/smc/smc_pnet.c
@@ -169,7 +169,7 @@ static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name)
pr_warn_ratelimited("smc: smcd device %s "
"erased user defined pnetid "
"%.16s\n",
- dev_name(smcd->ops->get_dev(smcd)),
+ dev_name(&smcd->dibs->dev),
smcd->pnetid);
memset(smcd->pnetid, 0, SMC_MAX_PNETID_LEN);
smcd->pnetid_by_user = false;
@@ -332,8 +332,11 @@ static struct smcd_dev *smc_pnet_find_smcd(char *smcd_name)
mutex_lock(&smcd_dev_list.mutex);
list_for_each_entry(smcd_dev, &smcd_dev_list.list, list) {
- if (!strncmp(dev_name(smcd_dev->ops->get_dev(smcd_dev)),
- smcd_name, IB_DEVICE_NAME_MAX - 1))
+ if (!strncmp(dev_name(&smcd_dev->dibs->dev), smcd_name,
+ IB_DEVICE_NAME_MAX - 1) ||
+ (smcd_dev->dibs->dev.parent &&
+ !strncmp(dev_name(smcd_dev->dibs->dev.parent), smcd_name,
+ IB_DEVICE_NAME_MAX - 1)))
goto out;
}
smcd_dev = NULL;
@@ -413,7 +416,6 @@ static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
bool smcddev_applied = true;
bool ibdev_applied = true;
struct smcd_dev *smcd;
- struct device *dev;
bool new_ibdev;
/* try to apply the pnetid to active devices */
@@ -431,10 +433,8 @@ static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
if (smcd) {
smcddev_applied = smc_pnet_apply_smcd(smcd, pnet_name);
if (smcddev_applied) {
- dev = smcd->ops->get_dev(smcd);
- pr_warn_ratelimited("smc: smcd device %s "
- "applied user defined pnetid "
- "%.16s\n", dev_name(dev),
+ pr_warn_ratelimited("smc: smcd device %s applied user defined pnetid %.16s\n",
+ dev_name(&smcd->dibs->dev),
smcd->pnetid);
}
}
@@ -450,7 +450,7 @@ static int smc_pnet_add_ib(struct smc_pnettable *pnettable, char *ib_name,
return -ENOMEM;
new_pe->type = SMC_PNET_IB;
memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN);
- strncpy(new_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX);
+ strscpy(new_pe->ib_name, ib_name);
new_pe->ib_port = ib_port;
new_ibdev = true;
@@ -1126,37 +1126,38 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev,
*/
void smc_pnet_find_roce_resource(struct sock *sk, struct smc_init_info *ini)
{
- struct dst_entry *dst = sk_dst_get(sk);
-
- if (!dst)
- goto out;
- if (!dst->dev)
- goto out_rel;
+ struct net_device *dev;
+ struct dst_entry *dst;
- smc_pnet_find_roce_by_pnetid(dst->dev, ini);
+ rcu_read_lock();
+ dst = __sk_dst_get(sk);
+ dev = dst ? dst_dev_rcu(dst) : NULL;
+ dev_hold(dev);
+ rcu_read_unlock();
-out_rel:
- dst_release(dst);
-out:
- return;
+ if (dev) {
+ smc_pnet_find_roce_by_pnetid(dev, ini);
+ dev_put(dev);
+ }
}
void smc_pnet_find_ism_resource(struct sock *sk, struct smc_init_info *ini)
{
- struct dst_entry *dst = sk_dst_get(sk);
+ struct net_device *dev;
+ struct dst_entry *dst;
ini->ism_dev[0] = NULL;
- if (!dst)
- goto out;
- if (!dst->dev)
- goto out_rel;
- smc_pnet_find_ism_by_pnetid(dst->dev, ini);
+ rcu_read_lock();
+ dst = __sk_dst_get(sk);
+ dev = dst ? dst_dev_rcu(dst) : NULL;
+ dev_hold(dev);
+ rcu_read_unlock();
-out_rel:
- dst_release(dst);
-out:
- return;
+ if (dev) {
+ smc_pnet_find_ism_by_pnetid(dev, ini);
+ dev_put(dev);
+ }
}
/* Lookup and apply a pnet table entry to the given ib device.
@@ -1192,7 +1193,6 @@ int smc_pnetid_by_table_ib(struct smc_ib_device *smcibdev, u8 ib_port)
*/
int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
{
- const char *ib_name = dev_name(smcddev->ops->get_dev(smcddev));
struct smc_pnettable *pnettable;
struct smc_pnetentry *tmp_pe;
struct smc_net *sn;
@@ -1205,7 +1205,13 @@ int smc_pnetid_by_table_smcd(struct smcd_dev *smcddev)
mutex_lock(&pnettable->lock);
list_for_each_entry(tmp_pe, &pnettable->pnetlist, list) {
if (tmp_pe->type == SMC_PNET_IB &&
- !strncmp(tmp_pe->ib_name, ib_name, IB_DEVICE_NAME_MAX)) {
+ (!strncmp(tmp_pe->ib_name,
+ dev_name(&smcddev->dibs->dev),
+ sizeof(tmp_pe->ib_name)) ||
+ (smcddev->dibs->dev.parent &&
+ !strncmp(tmp_pe->ib_name,
+ dev_name(smcddev->dibs->dev.parent),
+ sizeof(tmp_pe->ib_name))))) {
smc_pnet_apply_smcd(smcddev, tmp_pe->pnet_name);
rc = 0;
break;
diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c
index 2fab6456f765..b1efed546243 100644
--- a/net/smc/smc_sysctl.c
+++ b/net/smc/smc_sysctl.c
@@ -12,12 +12,14 @@
#include <linux/init.h>
#include <linux/sysctl.h>
+#include <linux/bpf.h>
#include <net/net_namespace.h>
#include "smc.h"
#include "smc_core.h"
#include "smc_llc.h"
#include "smc_sysctl.h"
+#include "smc_hs_bpf.h"
static int min_sndbuf = SMC_BUF_MIN_SIZE;
static int min_rcvbuf = SMC_BUF_MIN_SIZE;
@@ -29,6 +31,71 @@ static int links_per_lgr_min = SMC_LINKS_ADD_LNK_MIN;
static int links_per_lgr_max = SMC_LINKS_ADD_LNK_MAX;
static int conns_per_lgr_min = SMC_CONN_PER_LGR_MIN;
static int conns_per_lgr_max = SMC_CONN_PER_LGR_MAX;
+static unsigned int smcr_max_wr_min = 2;
+static unsigned int smcr_max_wr_max = 2048;
+
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+static int smc_net_replace_smc_hs_ctrl(struct net *net, const char *name)
+{
+ struct smc_hs_ctrl *ctrl = NULL;
+
+ rcu_read_lock();
+ /* null or empty name ask to clear current ctrl */
+ if (name && name[0]) {
+ ctrl = smc_hs_ctrl_find_by_name(name);
+ if (!ctrl) {
+ rcu_read_unlock();
+ return -EINVAL;
+ }
+ /* no change, just return */
+ if (ctrl == rcu_dereference(net->smc.hs_ctrl)) {
+ rcu_read_unlock();
+ return 0;
+ }
+ if (!bpf_try_module_get(ctrl, ctrl->owner)) {
+ rcu_read_unlock();
+ return -EBUSY;
+ }
+ }
+ /* xhcg old ctrl with the new one atomically */
+ ctrl = unrcu_pointer(xchg(&net->smc.hs_ctrl, RCU_INITIALIZER(ctrl)));
+ /* release old ctrl */
+ if (ctrl)
+ bpf_module_put(ctrl, ctrl->owner);
+
+ rcu_read_unlock();
+ return 0;
+}
+
+static int proc_smc_hs_ctrl(const struct ctl_table *ctl, int write,
+ void *buffer, size_t *lenp, loff_t *ppos)
+{
+ struct net *net = container_of(ctl->data, struct net, smc.hs_ctrl);
+ char val[SMC_HS_CTRL_NAME_MAX];
+ const struct ctl_table tbl = {
+ .data = val,
+ .maxlen = SMC_HS_CTRL_NAME_MAX,
+ };
+ struct smc_hs_ctrl *ctrl;
+ int ret;
+
+ rcu_read_lock();
+ ctrl = rcu_dereference(net->smc.hs_ctrl);
+ if (ctrl)
+ memcpy(val, ctrl->name, sizeof(ctrl->name));
+ else
+ val[0] = '\0';
+ rcu_read_unlock();
+
+ ret = proc_dostring(&tbl, write, buffer, lenp, ppos);
+ if (ret)
+ return ret;
+
+ if (write)
+ ret = smc_net_replace_smc_hs_ctrl(net, val);
+ return ret;
+}
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
static struct ctl_table smc_table[] = {
{
@@ -99,6 +166,33 @@ static struct ctl_table smc_table[] = {
.extra1 = SYSCTL_ZERO,
.extra2 = SYSCTL_ONE,
},
+ {
+ .procname = "smcr_max_send_wr",
+ .data = &init_net.smc.sysctl_smcr_max_send_wr,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &smcr_max_wr_min,
+ .extra2 = &smcr_max_wr_max,
+ },
+ {
+ .procname = "smcr_max_recv_wr",
+ .data = &init_net.smc.sysctl_smcr_max_recv_wr,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &smcr_max_wr_min,
+ .extra2 = &smcr_max_wr_max,
+ },
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+ {
+ .procname = "hs_ctrl",
+ .data = &init_net.smc.hs_ctrl,
+ .mode = 0644,
+ .maxlen = SMC_HS_CTRL_NAME_MAX,
+ .proc_handler = proc_smc_hs_ctrl,
+ },
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
};
int __net_init smc_sysctl_net_init(struct net *net)
@@ -109,6 +203,16 @@ int __net_init smc_sysctl_net_init(struct net *net)
table = smc_table;
if (!net_eq(net, &init_net)) {
int i;
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+ struct smc_hs_ctrl *ctrl;
+
+ rcu_read_lock();
+ ctrl = rcu_dereference(init_net.smc.hs_ctrl);
+ if (ctrl && ctrl->flags & SMC_HS_CTRL_FLAG_INHERITABLE &&
+ bpf_try_module_get(ctrl, ctrl->owner))
+ rcu_assign_pointer(net->smc.hs_ctrl, ctrl);
+ rcu_read_unlock();
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
table = kmemdup(table, sizeof(smc_table), GFP_KERNEL);
if (!table)
@@ -130,6 +234,8 @@ int __net_init smc_sysctl_net_init(struct net *net)
WRITE_ONCE(net->smc.sysctl_rmem, net_smc_rmem_init);
net->smc.sysctl_max_links_per_lgr = SMC_LINKS_PER_LGR_MAX_PREFER;
net->smc.sysctl_max_conns_per_lgr = SMC_CONN_PER_LGR_PREFER;
+ net->smc.sysctl_smcr_max_send_wr = SMCR_MAX_SEND_WR_DEF;
+ net->smc.sysctl_smcr_max_recv_wr = SMCR_MAX_RECV_WR_DEF;
/* disable handshake limitation by default */
net->smc.limit_smc_hs = 0;
@@ -139,6 +245,9 @@ err_reg:
if (!net_eq(net, &init_net))
kfree(table);
err_alloc:
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+ smc_net_replace_smc_hs_ctrl(net, NULL);
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
return -ENOMEM;
}
@@ -148,6 +257,10 @@ void __net_exit smc_sysctl_net_exit(struct net *net)
table = net->smc.smc_hdr->ctl_table_arg;
unregister_net_sysctl_table(net->smc.smc_hdr);
+#if IS_ENABLED(CONFIG_SMC_HS_CTRL_BPF)
+ smc_net_replace_smc_hs_ctrl(net, NULL);
+#endif /* CONFIG_SMC_HS_CTRL_BPF */
+
if (!net_eq(net, &init_net))
kfree(table);
}
diff --git a/net/smc/smc_sysctl.h b/net/smc/smc_sysctl.h
index eb2465ae1e15..8538915af7af 100644
--- a/net/smc/smc_sysctl.h
+++ b/net/smc/smc_sysctl.h
@@ -25,6 +25,8 @@ static inline int smc_sysctl_net_init(struct net *net)
net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE;
net->smc.sysctl_max_links_per_lgr = SMC_LINKS_PER_LGR_MAX_PREFER;
net->smc.sysctl_max_conns_per_lgr = SMC_CONN_PER_LGR_PREFER;
+ net->smc.sysctl_smcr_max_send_wr = SMCR_MAX_SEND_WR_DEF;
+ net->smc.sysctl_smcr_max_recv_wr = SMCR_MAX_RECV_WR_DEF;
return 0;
}
diff --git a/net/smc/smc_tx.c b/net/smc/smc_tx.c
index 214ac3cbcf9a..3144b4b1fe29 100644
--- a/net/smc/smc_tx.c
+++ b/net/smc/smc_tx.c
@@ -426,6 +426,9 @@ static int smcd_tx_rdma_writes(struct smc_connection *conn, size_t len,
int srcchunk, dstchunk;
int rc;
+ if (conn->sndbuf_desc->is_attached)
+ return 0;
+
for (dstchunk = 0; dstchunk < 2; dstchunk++) {
for (srcchunk = 0; srcchunk < 2; srcchunk++) {
void *data = conn->sndbuf_desc->cpu_addr + src_off;
diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c
index b04a21b8c511..5feafa98ab1a 100644
--- a/net/smc/smc_wr.c
+++ b/net/smc/smc_wr.c
@@ -547,9 +547,9 @@ void smc_wr_remember_qp_attr(struct smc_link *lnk)
IB_QP_DEST_QPN,
&init_attr);
- lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
+ lnk->wr_tx_cnt = min_t(size_t, lnk->max_send_wr,
lnk->qp_attr.cap.max_send_wr);
- lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
+ lnk->wr_rx_cnt = min_t(size_t, lnk->max_recv_wr,
lnk->qp_attr.cap.max_recv_wr);
}
@@ -741,50 +741,51 @@ int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr)
int smc_wr_alloc_link_mem(struct smc_link *link)
{
/* allocate link related memory */
- link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
+ link->wr_tx_bufs = kcalloc(link->max_send_wr,
+ SMC_WR_BUF_SIZE, GFP_KERNEL);
if (!link->wr_tx_bufs)
goto no_mem;
- link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, link->wr_rx_buflen,
+ link->wr_rx_bufs = kcalloc(link->max_recv_wr, link->wr_rx_buflen,
GFP_KERNEL);
if (!link->wr_rx_bufs)
goto no_mem_wr_tx_bufs;
- link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
- GFP_KERNEL);
+ link->wr_tx_ibs = kcalloc(link->max_send_wr,
+ sizeof(link->wr_tx_ibs[0]), GFP_KERNEL);
if (!link->wr_tx_ibs)
goto no_mem_wr_rx_bufs;
- link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
+ link->wr_rx_ibs = kcalloc(link->max_recv_wr,
sizeof(link->wr_rx_ibs[0]),
GFP_KERNEL);
if (!link->wr_rx_ibs)
goto no_mem_wr_tx_ibs;
- link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
+ link->wr_tx_rdmas = kcalloc(link->max_send_wr,
sizeof(link->wr_tx_rdmas[0]),
GFP_KERNEL);
if (!link->wr_tx_rdmas)
goto no_mem_wr_rx_ibs;
- link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
+ link->wr_tx_rdma_sges = kcalloc(link->max_send_wr,
sizeof(link->wr_tx_rdma_sges[0]),
GFP_KERNEL);
if (!link->wr_tx_rdma_sges)
goto no_mem_wr_tx_rdmas;
- link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
+ link->wr_tx_sges = kcalloc(link->max_send_wr, sizeof(link->wr_tx_sges[0]),
GFP_KERNEL);
if (!link->wr_tx_sges)
goto no_mem_wr_tx_rdma_sges;
- link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
+ link->wr_rx_sges = kcalloc(link->max_recv_wr,
sizeof(link->wr_rx_sges[0]) * link->wr_rx_sge_cnt,
GFP_KERNEL);
if (!link->wr_rx_sges)
goto no_mem_wr_tx_sges;
- link->wr_tx_mask = bitmap_zalloc(SMC_WR_BUF_CNT, GFP_KERNEL);
+ link->wr_tx_mask = bitmap_zalloc(link->max_send_wr, GFP_KERNEL);
if (!link->wr_tx_mask)
goto no_mem_wr_rx_sges;
- link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
+ link->wr_tx_pends = kcalloc(link->max_send_wr,
sizeof(link->wr_tx_pends[0]),
GFP_KERNEL);
if (!link->wr_tx_pends)
goto no_mem_wr_tx_mask;
- link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT,
+ link->wr_tx_compl = kcalloc(link->max_send_wr,
sizeof(link->wr_tx_compl[0]),
GFP_KERNEL);
if (!link->wr_tx_compl)
@@ -905,7 +906,7 @@ int smc_wr_create_link(struct smc_link *lnk)
goto dma_unmap;
}
smc_wr_init_sge(lnk);
- bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT);
+ bitmap_zero(lnk->wr_tx_mask, lnk->max_send_wr);
init_waitqueue_head(&lnk->wr_tx_wait);
rc = percpu_ref_init(&lnk->wr_tx_refs, smcr_wr_tx_refs_free, 0, GFP_KERNEL);
if (rc)
diff --git a/net/smc/smc_wr.h b/net/smc/smc_wr.h
index f3008dda222a..aa4533af9122 100644
--- a/net/smc/smc_wr.h
+++ b/net/smc/smc_wr.h
@@ -19,8 +19,6 @@
#include "smc.h"
#include "smc_core.h"
-#define SMC_WR_BUF_CNT 16 /* # of ctrl buffers per link */
-
#define SMC_WR_TX_WAIT_FREE_SLOT_TIME (10 * HZ)
#define SMC_WR_TX_SIZE 44 /* actual size of wr_send data (<=SMC_WR_BUF_SIZE) */
generated by cgit (git 2.34.1) at 2025-12-11 21:35:40 +0000