From 0128aee2c26e71e14a7454cea4b4ca6e81eddbc4 Mon Sep 17 00:00:00 2001 From: Sylvain Didelot Date: Wed, 3 Jun 2020 10:44:54 -0400 Subject: [PATCH] prov/verbs: fi_getinfo() returns fi_sockaddr_ib interfaces. This is the final patch of the series that adds the support of GID-base connection establishment. The Verbs provider now can directly connect to the network adapters using the GID. In other words, the patch allows to use Libfabric even if there is no IP address set for the Infiniband interfaces. There are significant issues of issues IP addresses for connection establishment: - It requires to set up/maintain IP addresses for every IB interfaces. - In the context of multirail (multiple local interfaces that belong to the same network subnet), it requires specific IP routes to prevent an interface to reply for another one. Connection establishment would fail otherwise. The GID can be accessed by looking at the field src_addr returned by "fi_info -p verbs -v". Example of output: src_addr: fi_sockaddr_ib://[fe80::248a:703:1c:dc0c]:ffff:13f:0 The patch also modifies fabtest so anybody can start testing this new feature. A new option -F allows to specify the address format that is use for the source/destination addresses. After figuring out the GID of interface that will be used for the server, one can run the following commands with fabtest: Server: fi_msg_bw -s [fe80::248a:703:1c:dc0c]:ffff:13f:0 -e msg \ -p verbs -F fi_sockaddr_ib Client: fi_msg_bw -e msg -p verbs \ -F fi_sockaddr_ib [fe80::248a:703:1c:dc0c]:ffff:13f:0 Signed-off-by: Sylvain Didelot --- prov/verbs/src/fi_verbs.c | 73 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/prov/verbs/src/fi_verbs.c b/prov/verbs/src/fi_verbs.c index 0d7f6624eb4..86cbdf55d61 100644 --- a/prov/verbs/src/fi_verbs.c +++ b/prov/verbs/src/fi_verbs.c @@ -103,7 +103,7 @@ int vrb_sockaddr_len(struct sockaddr *addr) } static int -vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags, +vrb_get_rdmacm_rai(const char *node, const char *service, uint64_t flags, const struct fi_info *hints, struct rdma_addrinfo **rai) { struct rdma_addrinfo rai_hints, *_rai; @@ -154,6 +154,77 @@ vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags, return ret; } +static int vrb_get_sib_rai(const char *node, const char *service, uint64_t flags, + const struct fi_info *hints, struct rdma_addrinfo **rai) +{ + struct sockaddr_ib *sib; + size_t sib_len; + char *straddr; + uint32_t fmt; + int ret; + bool has_prefix; + const char *prefix = "fi_sockaddr_ib://"; + + if (!node) + return -FI_EINVAL; + + fmt = ofi_addr_format(node); + if (fmt == FI_SOCKADDR_IB) + has_prefix = true; + else if (fmt == FI_FORMAT_UNSPEC) + has_prefix = false; + else + return -FI_EINVAL; + + if (service) { + ret = asprintf(&straddr, "%s%s:%s", has_prefix ? "" : prefix, node, + service); + } else { + ret = asprintf(&straddr, "%s%s", has_prefix ? "" : prefix, node); + } + + if (ret) + return -FI_ENOMEM; + + ret = ofi_str_toaddr(straddr, &fmt, (void **)&sib, &sib_len); + free(straddr); + + if (ret || fmt != FI_SOCKADDR_IB) { + return -FI_EINVAL; + } + + *rai = calloc(1, sizeof(struct rdma_addrinfo)); + if (*rai == NULL) + return -FI_ENOMEM; + + (*rai)->ai_family = AF_IB; + (*rai)->ai_port_space = RDMA_PS_IB; + (*rai)->ai_qp_type = IBV_QPT_RC; + + if (flags & FI_SOURCE) { + (*rai)->ai_flags |= RAI_PASSIVE; + (*rai)->ai_src_addr = (void *)sib; + (*rai)->ai_src_len = sizeof(struct sockaddr_ib); + } else { + (*rai)->ai_dst_addr = (void *)sib; + (*rai)->ai_dst_len = sizeof(struct sockaddr_ib); + } + + ofi_straddr_log(&vrb_prov, FI_LOG_INFO, FI_LOG_FABRIC, + "src addr", sib); + + return 0; +} + +int vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags, + const struct fi_info *hints, struct rdma_addrinfo **rai) +{ + if (node && hints && hints->addr_format == FI_SOCKADDR_IB) + return vrb_get_sib_rai(node, service, flags, hints, rai); + else + return vrb_get_rdmacm_rai(node, service, flags, hints, rai); +} + int vrb_get_rai_id(const char *node, const char *service, uint64_t flags, const struct fi_info *hints, struct rdma_addrinfo **rai, struct rdma_cm_id **id)