bnxt_re/lib: Direct Verbs: Support CQ and QP verbs

The following Direct Verb (DV) APIs have been implemented in
this patch.

CQ Direct Verbs:
----------------
- bnxt_re_dv_create_cq():
  Create a CQ of requested size (cqe). The application must have
  already registered this memory using bnxt_re_dv_umem_reg().
  The CQ umem-handle and umem-offset provided by the application
  are passed to the driver. Save the CQ-handle returned by the driver.

- bnxt_re_dv_destroy_cq():
  Pass the CQ-handle to the driver and destroy the DV_CQ.

QP Direct Verbs:
----------------
- bnxt_re_dv_create_qp():
  Create a QP using specified params (struct bnxt_re_dv_qp_init_attr).
  The application must have already registered SQ/RQ memory using
  bnxt_re_dv_umem_reg(). The SQ/RQ umem-handle and umem-offset provided
  by the application are passed to the driver. Save the QP-handle
  returned by the driver.

- bnxt_re_dv_destroy_qp():
  Pass the QP-handle to the driver and destroy the DV_QP.

- bnxt_re_dv_modify_qp():
  Modify QP attributes for the DV_QP specified by the QP-handle.

- bnxt_re_dv_query_qp():
  Return QP attributes for the DV_QP specified by the QP-handle.

Note:
-----
Some applications might want to allocate memory for all resources of a
given type (CQ/QP) in one big chunk and then register that entire
memory once using dv_umem_reg(). At the time of creating each individual
resource, the application should pass a specific offset/length in the
umem registered memory.

Signed-off-by: Sriharsha Basavapatna <[email protected]>
Co-developed-by: Kalesh AP <[email protected]>
Signed-off-by: Kalesh AP <[email protected]>
Co-developed-by: Selvin Xavier <[email protected]>
Signed-off-by: Selvin Xavier <[email protected]>

Author: sbasavapatna

providers/bnxt_re/CMakeLists.txt

@@ -1,7 +1,19 @@
-rdma_provider(bnxt_re
+rdma_shared_provider(bnxt_re bnxt_re.map
+	1 1.1.${PACKAGE_VERSION}
 	db.c
 	main.c
 	memory.c
 	verbs.c
 	dv.c
 )
+
+publish_headers(infiniband
+  bnxt_re_dv.h
+)
+
+rdma_pkg_config("bnxt_re" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
+
+if (ENABLE_LTTNG AND LTTNGUST_FOUND)
+  target_include_directories(bnxt_re PUBLIC ".")
+  target_link_libraries(bnxt_re LINK_PRIVATE LTTng::UST)
+endif()

providers/bnxt_re/bnxt_re.map

@@ -1,10 +1,17 @@
-{
+/* Export symbols should be added below according to
+   Documentation/versioning.md document. */
+BNXT_RE_1.1 {
 	global:
-		openib_driver_init;
 		bnxt_re_dv_alloc_db_region;
 		bnxt_re_dv_free_db_region;
 		bnxt_re_dv_umem_reg;
 		bnxt_re_dv_umem_dereg;
 		bnxt_re_dv_get_default_db_region;
+		bnxt_re_dv_create_cq;
+		bnxt_re_dv_destroy_cq;
+		bnxt_re_dv_create_qp;
+		bnxt_re_dv_destroy_qp;
+		bnxt_re_dv_modify_qp;
+		bnxt_re_dv_query_qp;
 	local: *;
 };

providers/bnxt_re/bnxt_re_dv.h

@@ -63,6 +63,48 @@ struct bnxt_re_dv_umem_reg_attr {
 	int dmabuf_fd;
 };
 
+struct bnxt_re_dv_cq_init_attr {
+	uint64_t cq_handle;
+	void *umem_handle; /* umem_handle from umem_reg */
+	uint64_t cq_umem_offset;	/* offset into umem */
+	uint32_t ncqe;
+};
+
+struct bnxt_re_dv_cq_attr {
+	uint32_t ncqe; /* no. of entries */
+	uint32_t cqe_size; /* size of entries */
+};
+
+struct bnxt_re_dv_qp_init_attr {
+	/* Standard ibv params */
+	enum ibv_qp_type qp_type;
+	uint32_t max_send_wr;
+	uint32_t max_recv_wr;
+	uint32_t max_send_sge;
+	uint32_t max_recv_sge;
+	uint32_t max_inline_data;
+	struct ibv_cq *send_cq;
+	struct ibv_cq *recv_cq;
+	struct ibv_srq *srq;
+
+	/* DV params */
+	uint64_t qp_handle;	/* to match with cqe */
+	void *dbr_handle;	/* dbr_handle from alloc_dbr */
+	void *sq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t sq_umem_offset;	/* offset into umem */
+	uint32_t sq_len;	/* sq length including MSN area */
+	uint32_t sq_slots;	/* sq length in slots */
+	void *rq_umem_handle;	/* umem_handle from umem_reg */
+	uint64_t rq_umem_offset;	/* offset into umem */
+	uint32_t sq_wqe_sz;     /* sq wqe size */
+	uint32_t sq_psn_sz;     /* sq psn size */
+	uint32_t sq_npsn;       /* sq num psn entries */
+	uint32_t rq_len;	/* rq length */
+	uint32_t rq_slots;	/* rq length in slots */
+	uint32_t rq_wqe_sz;     /* rq wqe size */
+	uint64_t comp_mask;	/* compatibility mask for future updates */
+};
+
 struct bnxt_re_dv_db_region_attr *
 bnxt_re_dv_alloc_db_region(struct ibv_context *ctx);
 int bnxt_re_dv_free_db_region(struct ibv_context *ctx,
@@ -72,6 +114,15 @@ int bnxt_re_dv_get_default_db_region(struct ibv_context *ibvctx,
 void *bnxt_re_dv_umem_reg(struct ibv_context *ibvctx,
 			  struct bnxt_re_dv_umem_reg_attr *in);
 int bnxt_re_dv_umem_dereg(void *umem_handle);
+struct ibv_cq *bnxt_re_dv_create_cq(struct ibv_context *ibvctx,
+				    struct bnxt_re_dv_cq_init_attr *cq_attr);
+int bnxt_re_dv_destroy_cq(struct ibv_cq *ibv_cq);
+struct ibv_qp *bnxt_re_dv_create_qp(struct ibv_pd *pd,
+				    struct bnxt_re_dv_qp_init_attr *qp_attr);
+int bnxt_re_dv_destroy_qp(struct ibv_qp *ibvqp);
+int bnxt_re_dv_modify_qp(struct ibv_qp *ibv_qp, struct ibv_qp_attr *attr,
+			 int attr_mask);
+int bnxt_re_dv_query_qp(void *qp_handle, struct ib_uverbs_qp_attr *attr);
 #ifdef __cplusplus
 }
 #endif