prov/verbs: Allow RDMACM to connect using GIDs

fabtests/benchmarks/msg_bw.c

@@ -107,6 +107,7 @@ int main(int argc, char **argv)
 	hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
 	hints->domain_attr->mr_mode = opts.mr_mode;
 	hints->domain_attr->threading = FI_THREAD_DOMAIN;
+	hints->addr_format = opts.address_format;
 
 	ret = run();
 

fabtests/benchmarks/msg_pingpong.c

@@ -107,6 +107,7 @@ int main(int argc, char **argv)
 	hints->caps = FI_MSG;
 	hints->domain_attr->mr_mode = opts.mr_mode;
 	hints->domain_attr->threading = FI_THREAD_DOMAIN;
+	hints->addr_format = opts.address_format;
 
 	ret = run();
 

fabtests/benchmarks/rma_bw.c

@@ -95,6 +95,7 @@ int main(int argc, char **argv)
 	hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
 	hints->mode = FI_CONTEXT;
 	hints->domain_attr->threading = FI_THREAD_DOMAIN;
+	hints->addr_format = opts.address_format;
 
 	while ((op = getopt(argc, argv, "ho:" CS_OPTS INFO_OPTS BENCHMARK_OPTS)) != -1) {
 		switch (op) {

fabtests/common/shared.c

@@ -2764,6 +2764,7 @@ void ft_addr_usage()
 			"over the, optional, port");
 	FT_PRINT_OPTS_USAGE("-C <number>", "number of connections to accept before "
 			"cleaning up a server");
+	FT_PRINT_OPTS_USAGE("-F <addr_format>", "Address format (default:FI_FORMAT_UNSPEC)");
 }
 
 void ft_usage(char *name, char *desc)
@@ -2923,6 +2924,16 @@ void ft_parse_addr_opts(int op, char *optarg, struct ft_opts *opts)
 		else
 			opts->oob_port = default_oob_port;
 		break;
+	case 'F':
+		if (!strncasecmp("fi_sockaddr_in", optarg, 14))
+			opts->address_format = FI_SOCKADDR_IN;
+		else if (!strncasecmp("fi_sockaddr_in6", optarg, 15))
+			opts->address_format = FI_SOCKADDR_IN6;
+		else if (!strncasecmp("fi_sockaddr_ib", optarg, 14))
+			opts->address_format = FI_SOCKADDR_IB;
+		else if (!strncasecmp("fi_sockaddr", optarg, 11)) /* keep me last */
+			opts->address_format = FI_SOCKADDR;
+		break;
 	case 'C':
 		opts->options |= FT_OPT_SERVER_PERSIST;
 		opts->num_connections = atoi(optarg);

fabtests/include/shared.h

@@ -165,13 +165,14 @@ struct ft_opts {
 	char *oob_port;
 	int argc;
 	int num_connections;
+	int address_format;
 
 	uint64_t mr_mode;
 	/* Fail if the selected provider does not support FI_MSG_PREFIX.  */
 	int force_prefix;
 	enum fi_hmem_iface iface;
 	uint64_t device;
-	
+
 	char **argv;
 };
 
@@ -241,7 +242,7 @@ extern int ft_parent_proc;
 extern int ft_socket_pair[2];
 extern int sock;
 extern int listen_sock;
-#define ADDR_OPTS "B:P:s:a:b::E::C:"
+#define ADDR_OPTS "B:P:s:a:b::E::C:F:"
 #define FAB_OPTS "f:d:p:D:i:H"
 #define INFO_OPTS FAB_OPTS "e:M:"
 #define CS_OPTS ADDR_OPTS "I:S:mc:t:w:l"
@@ -265,7 +266,8 @@ extern char default_port[8];
 		.mr_mode = FI_MR_LOCAL | OFI_MR_BASIC_MAP, \
 		.iface = FI_HMEM_SYSTEM, \
 		.device = 0, \
-		.argc = argc, .argv = argv \
+		.argc = argc, .argv = argv, \
+		.address_format = FI_FORMAT_UNSPEC \
 	}
 
 #define FT_STR_LEN 32

fabtests/man/fabtests.7.md

@@ -382,6 +382,9 @@ the list available for that test.
 *-s <address>*
 : Specifies the address of the local endpoint.
 
+*-F <address_format>
+: Specifies the address format.
+
 *-b[=oob_port]*
 : Enables out-of-band (via sockets) address exchange and test
   synchronization.  A port for the out-of-band connection may be specified

include/ofi_net.h

@@ -127,11 +127,33 @@ int ofi_discard_socket(SOCKET sock, size_t len);
 
 #define OFI_ADDRSTRLEN (INET6_ADDRSTRLEN + 50)
 
+/*  values taken from librdmacm/rdma_cma.h */
+#define OFI_IB_IP_PS_MASK   0xFFFFFFFFFFFF0000ULL
+#define OFI_IB_IP_PORT_MASK   0x000000000000FFFFULL
+
+struct ofi_sockaddr_ib {
+	unsigned short int  sib_family; /* AF_IB */
+	uint16_t            sib_pkey;
+	uint32_t            sib_flowinfo;
+	uint8_t             sib_addr[16];
+	uint64_t            sib_sid;
+	uint64_t            sib_sid_mask;
+	uint64_t            sib_scope_id;
+};
+
+enum ofi_rdma_port_space {
+	OFI_RDMA_PS_IPOIB = 0x0002,
+	OFI_RDMA_PS_IB    = 0x013F,
+	OFI_RDMA_PS_TCP   = 0x0106,
+	OFI_RDMA_PS_UDP   = 0x0111,
+};
+
 union ofi_sock_ip {
-	struct sockaddr		sa;
-	struct sockaddr_in	sin;
-	struct sockaddr_in6	sin6;
-	uint8_t			align[32];
+	struct sockaddr			sa;
+	struct sockaddr_in		sin;
+	struct sockaddr_in6		sin6;
+	struct ofi_sockaddr_ib		sib;
+	uint8_t				align[48];
 };
 
 struct ofi_addr_list_entry {
@@ -160,6 +182,7 @@ void ofi_free_list_of_addr(struct slist *addr_list);
 #define ofi_sin6_addr(addr) (((struct sockaddr_in6 *)(addr))->sin6_addr)
 #define ofi_sin6_port(addr) (((struct sockaddr_in6 *)(addr))->sin6_port)
 
+#define ofi_sib_addr(addr) (((struct ofi_sockaddr_ib *)(addr))->sib_addr)
 
 static inline size_t ofi_sizeofaddr(const struct sockaddr *addr)
 {
@@ -168,6 +191,8 @@ static inline size_t ofi_sizeofaddr(const struct sockaddr *addr)
 		return sizeof(struct sockaddr_in);
 	case AF_INET6:
 		return sizeof(struct sockaddr_in6);
+	case AF_IB:
+		return sizeof(struct ofi_sockaddr_ib);
 	default:
 		FI_WARN(&core_prov, FI_LOG_CORE, "Unknown address format\n");
 		return 0;
@@ -181,6 +206,8 @@ static inline size_t ofi_sizeofip(const struct sockaddr *addr)
 		return sizeof(struct in_addr);
 	case AF_INET6:
 		return sizeof(struct in6_addr);
+	case AF_IB:
+		return sizeof(ofi_sib_addr(addr));
 	default:
 		FI_WARN(&core_prov, FI_LOG_CORE, "Unknown address format\n");
 		return 0;
@@ -203,7 +230,7 @@ static inline int ofi_translate_addr_format(int family)
 
 uint16_t ofi_get_sa_family(const struct fi_info *info);
 
-static inline int ofi_ipv4_is_any_addr(struct sockaddr *sa)
+static inline int ofi_sin_is_any_addr(struct sockaddr *sa)
 {
 	struct in_addr ia_any = {
 		.s_addr = INADDR_ANY,
@@ -216,7 +243,7 @@ static inline int ofi_ipv4_is_any_addr(struct sockaddr *sa)
 
 }
 
-static inline int ofi_ipv6_is_any_addr(struct sockaddr *sa)
+static inline int ofi_sin6_is_any_addr(struct sockaddr *sa)
 {
 	struct in6_addr ia6_any = IN6ADDR_ANY_INIT;
 
@@ -226,16 +253,28 @@ static inline int ofi_ipv6_is_any_addr(struct sockaddr *sa)
 	return !memcmp(&ofi_sin6_addr(sa), &ia6_any, sizeof(ia6_any));
 }
 
+static inline int ofi_sib_is_any_addr(struct sockaddr *sa)
+{
+	struct in6_addr ia6_any = IN6ADDR_ANY_INIT;
+
+	if (!sa)
+		return 0;
+
+	return !memcmp(&ofi_sib_addr(sa), &ia6_any, sizeof(ia6_any));
+}
+
 static inline int ofi_is_any_addr(struct sockaddr *sa)
 {
 	if (!sa)
 		return 0;
 
 	switch(sa->sa_family) {
 	case AF_INET:
-		return ofi_ipv4_is_any_addr(sa);
+		return ofi_sin_is_any_addr(sa);
 	case AF_INET6:
-		return ofi_ipv6_is_any_addr(sa);
+		return ofi_sin6_is_any_addr(sa);
+	case AF_IB:
+		return ofi_sib_is_any_addr(sa);
 	default:
 		FI_WARN(&core_prov, FI_LOG_CORE, "Unknown address format!\n");
 		return 0;
@@ -252,6 +291,8 @@ static inline uint16_t ofi_addr_get_port(const struct sockaddr *addr)
 		return ntohs(ofi_sin_port((const struct sockaddr_in *) addr));
 	case AF_INET6:
 		return ntohs(ofi_sin6_port((const struct sockaddr_in6 *) addr));
+	case AF_IB:
+		return (uint16_t)ntohll(((const struct ofi_sockaddr_ib *)addr)->sib_sid);
 	default:
 		FI_WARN(&core_prov, FI_LOG_FABRIC, "Unknown address format\n");
 		assert(0);
@@ -261,13 +302,20 @@ static inline uint16_t ofi_addr_get_port(const struct sockaddr *addr)
 
 static inline void ofi_addr_set_port(struct sockaddr *addr, uint16_t port)
 {
+	struct ofi_sockaddr_ib *sib;
+
 	switch (ofi_sa_family(addr)) {
 	case AF_INET:
 		ofi_sin_port(addr) = htons(port);
 		break;
 	case AF_INET6:
 		ofi_sin6_port(addr) = htons(port);
 		break;
+    case AF_IB:
+		sib = (struct ofi_sockaddr_ib *)addr;
+		sib->sib_sid = htonll(((uint64_t)OFI_RDMA_PS_IB << 16) + ntohs(port));
+		sib->sib_sid_mask = htonll(OFI_IB_IP_PS_MASK | OFI_IB_IP_PORT_MASK);
+		break;
 	default:
 		FI_WARN(&core_prov, FI_LOG_FABRIC, "Unknown address format\n");
 		assert(0);
@@ -281,6 +329,8 @@ static inline void * ofi_get_ipaddr(const struct sockaddr *addr)
 		return &ofi_sin_addr((const struct sockaddr_in *) addr);
 	case AF_INET6:
 		return &ofi_sin6_addr((const struct sockaddr_in6 *) addr);
+	case AF_IB:
+		return &ofi_sib_addr((const struct ofi_sockaddr_ib *) addr);
 	default:
 		return NULL;
 	}
@@ -299,6 +349,9 @@ static inline int ofi_equals_ipaddr(const struct sockaddr *addr1,
 	case AF_INET6:
 	        return !memcmp(&ofi_sin6_addr(addr1), &ofi_sin6_addr(addr2),
 				sizeof(ofi_sin6_addr(addr1)));
+	case AF_IB:
+	        return !memcmp(&ofi_sib_addr(addr1), &ofi_sib_addr(addr2),
+				sizeof(ofi_sib_addr(addr1)));
 	default:
 		return 0;
 	}
@@ -323,6 +376,7 @@ size_t ofi_mask_addr(struct sockaddr *maskaddr, const struct sockaddr *srcaddr,
  */
 const char *ofi_straddr(char *buf, size_t *len,
 			uint32_t addr_format, const void *addr);
+uint32_t ofi_addr_format(const char *str);
 
 /* Returns allocated address to caller.  Caller must free.  */
 int ofi_str_toaddr(const char *str, uint32_t *addr_format,

include/windows/osd.h

@@ -261,6 +261,7 @@ do						\
 #define strcasecmp _stricmp
 #define snprintf _snprintf
 #define sleep(x) Sleep(x * 1000)
+#define strtok_r strtok_s
 
 #define __PRI64_PREFIX "ll"
 

prov/verbs/src/fi_verbs.c

@@ -103,7 +103,7 @@ int vrb_sockaddr_len(struct sockaddr *addr)
 }
 
 static int
-vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags,
+vrb_get_rdmacm_rai(const char *node, const char *service, uint64_t flags,
 		 const struct fi_info *hints, struct rdma_addrinfo **rai)
 {
 	struct rdma_addrinfo rai_hints, *_rai;
@@ -154,6 +154,97 @@ vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags,
 	return ret;
 }
 
+static int vrb_get_sib_rai(const char *node, const char *service, uint64_t flags,
+		      const struct fi_info *hints, struct rdma_addrinfo **rai)
+{
+	struct sockaddr_ib *sib;
+	size_t sib_len;
+	char *straddr;
+	uint32_t fmt;
+	int ret;
+	bool has_prefix;
+	const char *prefix = "fi_sockaddr_ib://";
+
+	*rai = calloc(1, sizeof(struct rdma_addrinfo));
+	if (*rai == NULL)
+		return -FI_ENOMEM;
+
+	ret = vrb_fi_to_rai(hints, flags, *rai);
+	if (ret)
+		return ret;
+
+	if (node) {
+		fmt = ofi_addr_format(node);
+		if (fmt == FI_SOCKADDR_IB)
+			has_prefix = true;
+		else if (fmt == FI_FORMAT_UNSPEC)
+			has_prefix = false;
+		else
+			return -FI_EINVAL;
+
+		if (service) {
+			ret = asprintf(&straddr, "%s%s:%s", has_prefix ? "" : prefix,
+			                                    node, service);
+		} else {
+			ret = asprintf(&straddr, "%s%s", has_prefix ? "" : prefix, node);
+		}
+
+		if (ret == -1)
+			return -FI_ENOMEM;
+
+		ret = ofi_str_toaddr(straddr, &fmt, (void **)&sib, &sib_len);
+		free(straddr);
+
+		if (ret || fmt != FI_SOCKADDR_IB) {
+			return -FI_EINVAL;
+		}
+
+		if (flags & FI_SOURCE) {
+			(*rai)->ai_flags |= RAI_PASSIVE;
+			if ((*rai)->ai_src_addr)
+				free((*rai)->ai_src_addr);
+			(*rai)->ai_src_addr = (void *)sib;
+			(*rai)->ai_src_len = sizeof(struct sockaddr_ib);
+		} else {
+			if ((*rai)->ai_dst_addr)
+				free((*rai)->ai_dst_addr);
+			(*rai)->ai_dst_addr = (void *)sib;
+			(*rai)->ai_dst_len = sizeof(struct sockaddr_ib);
+		}
+
+	} else if (service) {
+		if ((flags & FI_SOURCE) && (*rai)->ai_src_addr) {
+			if ((*rai)->ai_src_len < sizeof(struct sockaddr_ib))
+				return -FI_EINVAL;
+
+			(*rai)->ai_src_len = sizeof(struct sockaddr_ib);
+			sib = (struct sockaddr_ib *)(*rai)->ai_src_addr;
+		} else {
+			if ((*rai)->ai_dst_len < sizeof(struct sockaddr_ib))
+				return -FI_EINVAL;
+
+			(*rai)->ai_dst_len = sizeof(struct sockaddr_ib);
+			sib = (struct sockaddr_ib *)(*rai)->ai_dst_addr;
+		}
+
+		sib->sib_sid = htonll(((uint64_t) RDMA_PS_IB << 16) + (uint16_t)atoi(service));
+		sib->sib_sid_mask = htonll(OFI_IB_IP_PS_MASK | OFI_IB_IP_PORT_MASK);
+	}
+
+	return 0;
+}
+
+int vrb_get_rdma_rai(const char *node, const char *service, uint64_t flags,
+		 const struct fi_info *hints, struct rdma_addrinfo **rai)
+{
+	if (hints && hints->addr_format == FI_SOCKADDR_IB &&
+	    (node || hints->src_addr || hints->dest_addr)) {
+		return vrb_get_sib_rai(node, service, flags, hints, rai);
+	}
+
+	return vrb_get_rdmacm_rai(node, service, flags, hints, rai);
+}
+
 int vrb_get_rai_id(const char *node, const char *service, uint64_t flags,
 		      const struct fi_info *hints, struct rdma_addrinfo **rai,
 		      struct rdma_cm_id **id)
@@ -165,7 +256,7 @@ int vrb_get_rai_id(const char *node, const char *service, uint64_t flags,
 	if (ret)
 		return ret;
 
-	ret = rdma_create_id(NULL, id, NULL, RDMA_PS_TCP);
+	ret = rdma_create_id(NULL, id, NULL, vrb_get_port_space(hints));
 	if (ret) {
 		VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_create_id", errno);
 		ret = -errno;