diff --git a/SOURCES/10000-add-sxe2-rdma-core.patch b/SOURCES/10000-add-sxe2-rdma-core.patch new file mode 100755 index 0000000000000000000000000000000000000000..2fac59ac309565e9dd8458fd23ce33c7d25d5eb3 --- /dev/null +++ b/SOURCES/10000-add-sxe2-rdma-core.patch @@ -0,0 +1,9668 @@ +diff -Naur rdma-core-48.0/CMakeLists.txt rdma-core-48.0.bak/CMakeLists.txt +--- rdma-core-48.0/CMakeLists.txt 2026-05-26 10:44:41.360128441 +0800 ++++ rdma-core-48.0.bak/CMakeLists.txt 2026-05-26 10:31:41.449978895 +0800 +@@ -733,7 +733,6 @@ + #add_subdirectory(providers/ocrdma) + add_subdirectory(providers/qedr) + add_subdirectory(providers/vmw_pvrdma) +-add_subdirectory(providers/sxe2) + endif() + + add_subdirectory(providers/hfi1verbs) +diff -Naur rdma-core-48.0/kernel-headers/CMakeLists.txt rdma-core-48.0.bak/kernel-headers/CMakeLists.txt +--- rdma-core-48.0/kernel-headers/CMakeLists.txt 2026-05-26 10:39:16.261981850 +0800 ++++ rdma-core-48.0.bak/kernel-headers/CMakeLists.txt 2026-05-26 10:31:41.421978710 +0800 +@@ -26,7 +26,6 @@ + rdma/rvt-abi.h + rdma/siw-abi.h + rdma/vmw_pvrdma-abi.h +- rdma/sxe2-abi.h + ) + + publish_internal_headers(rdma/hfi +@@ -81,7 +80,6 @@ + rdma/rdma_user_rxe.h + rdma/siw-abi.h + rdma/vmw_pvrdma-abi.h +- rdma/sxe2-abi.h + ) + + publish_headers(infiniband +diff -Naur rdma-core-48.0/kernel-headers/rdma/ib_user_ioctl_verbs.h rdma-core-48.0.bak/kernel-headers/rdma/ib_user_ioctl_verbs.h +--- rdma-core-48.0/kernel-headers/rdma/ib_user_ioctl_verbs.h 2026-05-26 10:35:17.353404424 +0800 ++++ rdma-core-48.0.bak/kernel-headers/rdma/ib_user_ioctl_verbs.h 2026-05-26 10:31:41.422978717 +0800 +@@ -254,7 +254,6 @@ + RDMA_DRIVER_SIW, + RDMA_DRIVER_ERDMA, + RDMA_DRIVER_MANA, +- RDMA_DRIVER_SXE2, + }; + + enum ib_uverbs_gid_type { +diff -Naur rdma-core-48.0/kernel-headers/rdma/sxe2-abi.h rdma-core-48.0.bak/kernel-headers/rdma/sxe2-abi.h +--- rdma-core-48.0/kernel-headers/rdma/sxe2-abi.h 2026-05-26 10:35:55.120653790 +0800 ++++ rdma-core-48.0.bak/kernel-headers/rdma/sxe2-abi.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,117 +0,0 @@ +-/** +- * Copyright (C) +- * +- * @file sxe2_abi.h +- * @author zhaozy +- * @date 2024年6月6日 +- * @brief verbs扩展命令结构定义 +- * @note 创建文件 +- */ +-#ifndef __SXE2_ABI_STRUCT_H__ +-#define __SXE2_ABI_STRUCT_H__ +- +-#include +- +-#define SXE2_RDMA_ABI_VER (5) +- +-struct sxe2_create_ah_resp { +- __u32 ah_id; +- __u8 rsvd[4]; +-}; +- +-struct sxe2_modify_qp_req { +- __u8 sq_flush; +- __u8 rq_flush; +- __u8 llwqe_enable; +- __u8 new_page_alloc; +- __u32 llwqe_page_index; +-}; +- +-struct sxe2_modify_qp_resp { +- __u8 rd_fence_rate; +- __u8 rsvd[3]; +- __u32 db_mmap_size; +- __u32 db_page_id; +- __u32 rsvd1; +- __u64 db_mmap_offset; +-}; +- +-struct sxe2_create_qp_req { +- __u32 sq_depth; +- __u32 rq_depth; +- __u8 sq_shift; +- __u8 rq_shift; +- __u8 rsvd[6]; +- __u64 user_wqe_bufs; +- __u64 doorbell_note; +- __u64 user_compl_ctx; +-}; +- +-struct sxe2_create_cq_req { +- __aligned_u64 user_cq_buf; +- __aligned_u64 user_cq_db_note; +-}; +- +-struct sxe2_create_cq_resp { +- __u32 cq_id; +- __u32 ncqe; +-}; +- +-struct sxe2_alloc_pd_resp { +- __u32 pd_id; +- __u8 rsvd[4]; +-}; +- +-struct sxe2_create_qp_resp { +- __u32 qpn; +- __u32 qp_caps; +-}; +- +-struct sxe2_alloc_ucontext_req { +- __u32 rsvd32; +- __u8 userspace_ver; +- __u8 rsvd8[3]; +- __aligned_u64 comp_mask; +-}; +- +-struct sxe2_alloc_ucontext_resp { +- __u32 max_pds; +- __u32 max_qps; +- __u32 wq_size; /* size of the WQs (SQ+RQ) in the mmaped area */ +- __u8 kernel_ver; +- __u8 rsvd[3]; +- __aligned_u64 feature_flags; +- __aligned_u64 db_mmap_key; +- __u32 max_hw_wq_frags; +- __u32 max_hw_read_sges; +- __u32 max_hw_inline; +- __u32 max_hw_rq_quanta; +- __u32 max_hw_wq_quanta; +- __u32 min_hw_cq_size; +- __u32 max_hw_cq_size; +- __u16 max_hw_sq_chunk; +- __u8 hw_rev; +- __u8 is_pf; +- __aligned_u64 comp_mask; +- __u16 min_hw_wq_size; +- __u32 max_db; +- __u8 rsvd3[2]; +- __u8 bdf[16]; +- __u32 max_hw_srq_quanta; +- __u32 max_hw_srq_wr; +-}; +- +-struct sxe2_create_srq_req { +- __aligned_u64 user_srq_buf; /* SRQ Buf虚拟地址 */ +- __aligned_u64 user_srq_db_note; /* SRQ DB Note虚拟地址 */ +- __aligned_u64 srq_cmpl_ctx; /* SRQ Completion Context指针地址 */ +- __u32 srq_buf_size; /* SRQ Buf大小 */ +- __u32 srq_size; /* SRQ深度(包含SRQE数量) */ +- __u32 max_wr_cal; /* 用户态计算后的max_wr,已减去预留wr,Query SRQ使用 */ +-}; +- +-struct sxe2_create_srq_resp { +- __u32 srq_id; /* SRQN */ +-}; +- +-#endif +diff -Naur rdma-core-48.0/libibverbs/verbs.h rdma-core-48.0.bak/libibverbs/verbs.h +--- rdma-core-48.0/libibverbs/verbs.h 2026-05-26 10:41:09.999732819 +0800 ++++ rdma-core-48.0.bak/libibverbs/verbs.h 2026-05-26 10:31:41.521979371 +0800 +@@ -2273,7 +2273,6 @@ + extern const struct verbs_device_ops verbs_provider_vmw_pvrdma; + extern const struct verbs_device_ops verbs_provider_all; + extern const struct verbs_device_ops verbs_provider_none; +-extern const struct verbs_device_ops verbs_provider_sxe2; + void ibv_static_providers(void *unused, ...); + + static inline struct ibv_device **__ibv_get_device_list(int *num_devices) +diff -Naur rdma-core-48.0/providers/sxe2/ah.c rdma-core-48.0.bak/providers/sxe2/ah.c +--- rdma-core-48.0/providers/sxe2/ah.c 2026-05-26 10:42:01.855075215 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/ah.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,62 +0,0 @@ +- +-#include +-#include +- +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "log.h" +- +-struct ibv_ah *sxe2_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr) +-{ +- struct sxe2_uah *ah; +- union ibv_gid sgid; +- struct sxe2_ucreate_ah_resp resp; +- int err; +- struct sxe2_rdma_ucontext *sctx = to_sctx(ibpd->context); +- +- memset(&resp, 0, sizeof(resp)); +- err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, +- &sgid); +- if (err) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv query gid failed, ret %d.\n", err); +- errno = err; +- return NULL; +- } +- +- ah = calloc(1, sizeof(*ah)); +- if (!ah) { +- SXE2_VERBS_LOG_ERROR_BDF("calloc ah buf failed.\n"); +- return NULL; +- } +- +- err = ibv_cmd_create_ah(ibpd, &ah->ibv_ah, attr, &resp.ibv_resp, +- sizeof(resp)); +- if (err) { +- free(ah); +- SXE2_VERBS_LOG_ERROR_BDF("ibv create ah failed, ret %d.\n", err); +- errno = err; +- return NULL; +- } +- +- ah->ah_id = resp.ah_id; +- +- return &ah->ibv_ah; +-} +- +-int sxe2_udestroy_ah(struct ibv_ah *ibah) +-{ +- struct sxe2_uah *ah; +- int ret; +- +- ah = container_of(ibah, struct sxe2_uah, ibv_ah); +- +- ret = ibv_cmd_destroy_ah(ibah); +- if (ret) { +- SXE2_VERBS_LOG_ERROR("ibv destroy ah failed, ret %d.\n", ret); +- return ret; +- } +- +- free(ah); +- +- return 0; +-} +diff -Naur rdma-core-48.0/providers/sxe2/buf.c rdma-core-48.0.bak/providers/sxe2/buf.c +--- rdma-core-48.0/providers/sxe2/buf.c 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/buf.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,313 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "util/bitmap.h" +- +-#include "buf.h" +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "log.h" +- +- void sxe2_huge_mem_free(struct sxe2_rdma_ucontext *ctx, sxe2_hugetlb_mem_s *hmem) +-{ +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- if (hmem->bitmap) { +- free(hmem->bitmap); +- } +- +- if (shmdt(hmem->shmaddr) == -1) { +- SXE2_VERBS_LOG_ERROR_BDF("shmdt fail, err(%s)", strerror(errno)); +- } +- shmctl(hmem->shmid, IPC_RMID, NULL); +- free(hmem); +- +- return; +-} +- +- sxe2_hugetlb_mem_s *sxe2_huge_mem_alloc(struct sxe2_rdma_ucontext *ctx, size_t size) +-{ +- sxe2_hugetlb_mem_s *hmem; +- size_t shm_len; +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- hmem = malloc(sizeof(*hmem)); +- if (hmem == NULL) { +- SXE2_VERBS_LOG_ERROR_BDF("hmem malloc error"); +- goto end; +- } +- memset(hmem, 0, sizeof(*hmem)); +- +- shm_len = align(size, SXE2_SHM_LENGTH); +- hmem->shmid = shmget(IPC_PRIVATE, shm_len, SHM_HUGETLB | SHM_R | SHM_W); +- if (hmem->shmid == -1) { +- SXE2_VERBS_LOG_ERROR_BDF("shmget fail, err(%s)", strerror(errno)); +- goto out_free; +- } +- +- hmem->shmaddr = shmat(hmem->shmid, SXE2_SHM_ADDR, SXE2_SHMAT_FLAGS); +- if (hmem->shmaddr == (void *)-1) { +- SXE2_VERBS_LOG_ERROR_BDF("shmat fail, err(%s)", strerror(errno)); +- goto out_rmid; +- } +- +- hmem->bitmap = bitmap_alloc0(shm_len / SXE2_SHM_SINGLE_CHUNK_SIZE); +- if (!hmem->bitmap) { +- SXE2_VERBS_LOG_ERROR_BDF("bitmap_alloc0 fail, err(%s)", strerror(errno)); +- goto out_shmdt; +- } +- +- hmem->bmp_size = shm_len / SXE2_SHM_SINGLE_CHUNK_SIZE; +- +- shmctl(hmem->shmid, IPC_RMID, NULL); +- +- goto end; +- +-out_shmdt: +- if (shmdt(hmem->shmaddr) == -1) { +- SXE2_VERBS_LOG_ERROR_BDF("shmdt fail, err(%s)", strerror(errno)); +- } +-out_rmid: +- shmctl(hmem->shmid, IPC_RMID, NULL); +-out_free: +- free(hmem); +- hmem = NULL; +-end: +- return hmem; +-} +- +- void sxe2_huge_buf_free(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, bool dofork_range) +-{ +- size_t nchunk; +- +- nchunk = buf->length / SXE2_SHM_SINGLE_CHUNK_SIZE; +- +- if (buf->hmem == NULL) { +- goto end; +- } +- +- if (dofork_range == true) { +- ibv_dofork_range(buf->buf, buf->length); +- } +- +- sxe2_spin_lock(&ctx->hugetlb_lock); +- bitmap_zero_region(buf->hmem->bitmap, buf->base, buf->base + nchunk); +- if (bitmap_empty(buf->hmem->bitmap, buf->hmem->bmp_size)) { +- list_del(&buf->hmem->entry); +- sxe2_spin_unlock(&ctx->hugetlb_lock); +- sxe2_huge_mem_free(ctx, buf->hmem); +- buf->hmem = NULL; +- goto free; +- } +- +- sxe2_spin_unlock(&ctx->hugetlb_lock); +-free: +- buf->buf = NULL; +-end: +- return; +-} +- +- int sxe2_huge_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size) +-{ +- int found = 0; +- size_t nchunk; +- sxe2_hugetlb_mem_s *hmem = NULL; +- int ret; +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- buf->length = align(size, SXE2_SHM_SINGLE_CHUNK_SIZE); +- nchunk = buf->length / SXE2_SHM_SINGLE_CHUNK_SIZE; +- +- if (!nchunk) { +- ret = 0; +- SXE2_VERBS_LOG_WARN_BDF("not need huge"); +- goto end; +- } +- +- sxe2_spin_lock(&ctx->hugetlb_lock); +- list_for_each(&ctx->hugetlb_list, hmem, entry) { +- if (!bitmap_full(hmem->bitmap, hmem->bmp_size)) { +- buf->base = bitmap_find_free_region(hmem->bitmap, hmem->bmp_size, nchunk); +- if (buf->base != hmem->bmp_size) { +- bitmap_fill_region(hmem->bitmap, buf->base, buf->base + nchunk); +- buf->hmem = hmem; +- found = true; +- break; +- } +- } +- } +- sxe2_spin_unlock(&ctx->hugetlb_lock); +- +- if (!found) { +- hmem = sxe2_huge_mem_alloc(ctx, buf->length); +- if (NULL == hmem) { +- SXE2_VERBS_LOG_ERROR_BDF("sxe2_huge_mem_alloc error"); +- ret = ENOMEM; +- goto end; +- } +- +- buf->base = 0; +- assert(nchunk <= hmem->bmp_size); +- bitmap_fill_region(hmem->bitmap, 0, nchunk); +- +- buf->hmem = hmem; +- +- sxe2_spin_lock(&ctx->hugetlb_lock); +- if (nchunk != hmem->bmp_size) { +- list_add(&ctx->hugetlb_list, &hmem->entry); +- } else { +- list_add_tail(&ctx->hugetlb_list, &hmem->entry); +- } +- sxe2_spin_unlock(&ctx->hugetlb_lock); +- } +- +- buf->buf = hmem->shmaddr + buf->base * SXE2_SHM_SINGLE_CHUNK_SIZE; +- +- ret = ibv_dontfork_range(buf->buf, buf->length); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("dontfork_range error,buf:%p,length:%zu,ret:%d", buf->buf, buf->length, ret); +- ret = EPERM; +- goto out_fork; +- } +- buf->type = SXE2_ALLOC_TYPE_HUGE; +- +- goto end; +- +-out_fork: +- sxe2_huge_buf_free(ctx, buf, false); +-end: +- return ret; +-} +- +-int sxe2_prefered_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size, size_t page_size, +- sxe2_alloc_type_e type) +-{ +- int ret; +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- if (type == SXE2_ALLOC_TYPE_HUGE || +- type == SXE2_ALLOC_TYPE_PREFER_HUGE ) { +- ret = sxe2_huge_buf_alloc(ctx, buf, size); +- if (!ret) { +- goto end; +- } +- +- if (type == SXE2_ALLOC_TYPE_HUGE) { +- SXE2_VERBS_LOG_ERROR_BDF("Huge mode allocation fail, page_size:%zu, size:%zu", +- page_size, size); +- goto end; +- } +- +- SXE2_VERBS_LOG_ERROR_BDF("Huge mode allocation fail, fallback to default mode, type:%d, \ +- page_size:%zu, size:%zu", type, page_size, size); +- } +- +- ret = sxe2_buf_alloc(ctx, buf, size, page_size); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("cmr_buf_alloc error, page_size:%zu, size:%zu,ret:%d", +- page_size, size, ret); +- } +- +-end: +- return ret; +- +-} +- +-void sxe2_actual_buf_free(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf) +-{ +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- if ((buf == NULL) || +- (buf->buf == NULL) || +- (ctx == NULL)) { +- SXE2_VERBS_LOG_ERROR_BDF( "context or buf or buf->buf is NULL"); +- errno = EINVAL; +- goto end; +- } +- +- switch (buf->type) { +- case SXE2_ALLOC_TYPE_ANON: +- sxe2_buf_free(buf); +- break; +- +- case SXE2_ALLOC_TYPE_HUGE: +- sxe2_huge_buf_free(ctx, buf, true); +- break; +- +- default: +- SXE2_VERBS_LOG_ERROR_BDF("Bad allocation type:%d", buf->type); +- } +-end: +- return; +-} +- +-void sxe2_alloc_type_get(const char *component, sxe2_alloc_type_e *alloc_type, sxe2_alloc_type_e default_type) +-{ +- char *env_value; +- char name[SXE2_ALLOC_ENV_NAME]; +- +- snprintf(name, sizeof(name), "%s_ALLOC_TYPE", component); +- +- *alloc_type = default_type; +- +- env_value = getenv(name); +- if (env_value) { +- if (!strcasecmp(env_value, "ANON")) { +- *alloc_type = SXE2_ALLOC_TYPE_ANON; +- } else if (!strcasecmp(env_value, "HUGE")){ +- *alloc_type = SXE2_ALLOC_TYPE_HUGE; +- } else if (!strcasecmp(env_value, "PREFER_HUGE")){ +- *alloc_type = SXE2_ALLOC_TYPE_PREFER_HUGE; +- } +- } +-} +- +-int sxe2_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size, size_t page_size) +-{ +- int ret; +- size_t al_size; +- struct sxe2_rdma_ucontext *sctx = ctx; +- +- if ((buf == NULL) || (ctx == NULL)) { +- SXE2_VERBS_LOG_ERROR_BDF("context of buf is NULL"); +- ret = EPERM; +- goto end; +- } +- +- al_size = align(size, page_size); +- ret = posix_memalign(&buf->buf, page_size, al_size); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("posix_memalign error,page_size:%zu,al_size:%zu, ret:%d", page_size, al_size, ret); +- goto end; +- } +- +- ret = ibv_dontfork_range(buf->buf, al_size); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("dontfork_range error,buf:%p,al_size:%zu, ret:%d", buf->buf, al_size, ret); +- free(buf->buf); +- buf->buf = NULL; +- ret = EPERM; +- goto end; +- } +- +- buf->length = al_size; +- buf->type = SXE2_ALLOC_TYPE_ANON; +- +-end: +- return ret; +-} +- +-void sxe2_buf_free(sxe2_buf_s *buf) +-{ +- ibv_dofork_range(buf->buf, buf->length); +- free(buf->buf); +- buf->buf = NULL; +- +- return; +-} +- +diff -Naur rdma-core-48.0/providers/sxe2/buf.h rdma-core-48.0.bak/providers/sxe2/buf.h +--- rdma-core-48.0/providers/sxe2/buf.h 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/buf.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,42 +0,0 @@ +- +-#ifndef __BUF_H__ +-#define __BUF_H__ +- +-#include "sxe2_common.h" +-#include +-#include +-#include +- +-#define SXE2_SHM_ADDR NULL +-#define SXE2_SHMAT_FLAGS 0 +-#define SXE2_ALLOC_ENV_NAME (128) +- +-#ifndef HPAGE_SIZE +-#define HPAGE_SIZE (2UL * 1024 * 1024) +-#endif +- +-#define SXE2_SHM_LENGTH HPAGE_SIZE +-#define SXE2_SHM_SINGLE_CHUNK_SIZE 32768 +- +-int sxe2_prefered_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size, size_t page_size, +- sxe2_alloc_type_e type); +- +-void sxe2_actual_buf_free(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf); +- +-void sxe2_alloc_type_get(const char *component, sxe2_alloc_type_e *alloc_type, +- sxe2_alloc_type_e default_type); +- +-int sxe2_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size, size_t page_size); +- +-void sxe2_buf_free(sxe2_buf_s *buf); +- +- sxe2_hugetlb_mem_s *sxe2_huge_mem_alloc(struct sxe2_rdma_ucontext *ctx, size_t size); +- +- void sxe2_huge_buf_free(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, bool dofork_range); +- +- void sxe2_huge_mem_free(struct sxe2_rdma_ucontext *ctx, sxe2_hugetlb_mem_s *hmem); +- +- int sxe2_huge_buf_alloc(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf, size_t size); +- +-#endif +- +diff -Naur rdma-core-48.0/providers/sxe2/CMakeLists.txt rdma-core-48.0.bak/providers/sxe2/CMakeLists.txt +--- rdma-core-48.0/providers/sxe2/CMakeLists.txt 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/CMakeLists.txt 1970-01-01 08:00:00.000000000 +0800 +@@ -1,51 +0,0 @@ +-# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +-if(DEFINED ENV{MD_MAKEMODE}) +- set(makemode $ENV{MD_MAKEMODE}) +-else() +- set(makemode release) +-endif() +-if(makemode STREQUAL "release") +- add_definitions(-DSXE2_CFG_RELEASE) +-else() +- add_definitions(-DSXE2_CFG_DEBUG) +-endif() +-if(DEFINED ENV{ASAN}) +- set(asan $ENV{ASAN}) +-else() +- set(asan no) +-endif() +-if(asan STREQUAL "yes") +- add_definitions(-DSXE2_SUPPORT_ASAN) +-endif() +-if(DEFINED ENV{INJECT}) +- set(inject $ENV{INJECT}) +-else() +- set(inject no) +-endif() +-if(inject STREQUAL "yes") +- add_definitions(-DSXE2_SUPPORT_INJECT) +-endif() +-if(DEFINED ENV{IO_STAT}) +- set(iostat $ENV{IO_STAT}) +-else() +- set(iostat no) +-endif() +-if(iostat STREQUAL "yes") +- add_definitions(-DSXE2_SUPPORT_IO_STAT) +-endif() +-add_compile_options(-Werror) +-rdma_provider(sxe2 +- sxe2_common.c +- mr.c +- io.c +- ah.c +- cq.c +- pd.c +- qp.c +- db.c +- device_port.c +- log.c +- srq.c +- mc.c +- buf.c +-) +diff -Naur rdma-core-48.0/providers/sxe2/cq.c rdma-core-48.0.bak/providers/sxe2/cq.c +--- rdma-core-48.0/providers/sxe2/cq.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/cq.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,405 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "sxe2-abi.h" +-#include "log.h" +-#include "io.h" +-#include "buf.h" +- +-#define SXE2_HCA_CORE_CLOCK_800_MHZ (800) +- +-#define SXE2_VERBS_MIN_CQ_SIZE 4 +-#define SXE2_VERBS_MAX_CQ_SIZE 2097152 +- +-#define SXE2_UCQ_DB_NOTE_CMD_SN_SHIFT 29 +-#define SXE2_UCQ_DB_NOTE_CMD_SN 3 +- +-enum { +- UCREATE_CQ_SUPPORTED_FLAGS = +- IBV_CREATE_CQ_ATTR_SINGLE_THREADED | +- IBV_CREATE_CQ_ATTR_IGNORE_OVERRUN +-}; +- +-enum { +- UCREATE_CQ_SUPPORTED_COMP_MASK = IBV_CQ_INIT_ATTR_MASK_FLAGS | +- IBV_CQ_INIT_ATTR_MASK_PD +-}; +- +-static inline __u32 get_cqe_count(__u32 ncqe) +-{ +- ncqe++; +- +- ncqe = (__u32)roundup_pow_of_two(ncqe); +- +- if (ncqe < SXE2_U_MINCQ_SIZE) { +- ncqe = SXE2_U_MINCQ_SIZE; +- } +- return ncqe; +-} +- +-static inline size_t get_cq_total_bytes(unsigned int ncqe) +-{ +- const long page_size = sysconf(_SC_PAGE_SIZE); +- return roundup(ncqe * sizeof(struct sxe2_cqe), page_size); +-} +- +-static void sxe2_uk_cq_init(struct sxe2_cq_uk *cq, struct sxe2_cq_uk_init_info *info) +-{ +- cq->cq_base = info->cq_base; +- cq->cqe_alloc_db = info->cqe_alloc_db; +- cq->cq_id = info->cq_id; +- cq->ncqe = info->ncqe; +- cq->doorbell_note = info->doorbell_note; +- SXE2_RING_INIT(cq->cq_ring, cq->ncqe); +- cq->polarity = 1; +- cq->arm_sn = 0; +- return; +-} +- +-static void sxe2_ibvcq_ex_fill_priv_funcs(struct sxe2_ucq *cq, struct ibv_cq_init_attr_ex *cq_attr) +-{ +- struct ibv_cq_ex *ibvcq_ex = &cq->verbs_cq.cq_ex; +- +- ibvcq_ex->start_poll = sxe2_start_poll; +- ibvcq_ex->end_poll = sxe2_end_poll; +- ibvcq_ex->next_poll = sxe2_next_poll; +- +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) { +- ibvcq_ex->read_completion_ts = sxe2_wc_read_completion_ts; +- cq->report_rtt = true; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK) { +- ibvcq_ex->read_completion_wallclock_ns = sxe2_wc_read_completion_wallclock_ns; +- cq->report_rtt = true; +- } +- +- ibvcq_ex->read_opcode = sxe2_wc_read_opcode; +- ibvcq_ex->read_vendor_err = sxe2_wc_read_vendor_err; +- ibvcq_ex->read_wc_flags = sxe2_wc_read_wc_flags; +- +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_BYTE_LEN) { +- ibvcq_ex->read_byte_len = sxe2_wc_read_byte_len; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_IMM) { +- ibvcq_ex->read_imm_data = sxe2_wc_read_imm_data; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_QP_NUM) { +- ibvcq_ex->read_qp_num = sxe2_wc_read_qp_num; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_SRC_QP) { +- ibvcq_ex->read_src_qp = sxe2_wc_read_src_qp; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_SLID) { +- ibvcq_ex->read_slid = sxe2_wc_read_slid; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_SL) { +- ibvcq_ex->read_sl = sxe2_wc_read_sl; +- } +- if (cq_attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) { +- ibvcq_ex->read_dlid_path_bits = sxe2_wc_read_dlid_path_bits; +- } +-} +- +-static bool ucreate_cq_attr_check(struct sxe2_rdma_ucontext *sctx, +- const struct ibv_cq_init_attr_ex *cq_attr) +-{ +- bool isFail = true; +- +- if (cq_attr->comp_mask & (~UCREATE_CQ_SUPPORTED_COMP_MASK)) { +- SXE2_VERBS_LOG_ERROR_BDF("Unsupported comp_mask(%u) for create cq\n", cq_attr->comp_mask); +- goto end; +- } +- +- if ((cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS) && +- (cq_attr->flags & (~UCREATE_CQ_SUPPORTED_FLAGS))) { +- SXE2_VERBS_LOG_ERROR_BDF("Unsupported creation flags(%u) requested for create cq\n", cq_attr->flags); +- goto end; +- } +- +- isFail = false; +-end: +- return isFail; +-} +-static int sxe2_alloc_cq_buf(struct sxe2_rdma_ucontext *sctx, sxe2_buf_s *buf, size_t size) +-{ +- int ret; +- sxe2_alloc_type_e type; +- sxe2_alloc_type_e default_type = SXE2_ALLOC_TYPE_ANON; +- long page_size = 0; +- +- page_size = sysconf(_SC_PAGE_SIZE); +- if (page_size < 0) { +- SXE2_VERBS_LOG_ERROR_BDF("get system page size failed."); +- ret = EPERM; +- goto end; +- } +- +- sxe2_alloc_type_get(SXE2_CQ_PREFIX, &type, default_type); +- +- ret = sxe2_prefered_buf_alloc( +- sctx, buf, size, page_size, type); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("cq buf alloc err ret (%d)", ret); +- goto end; +- } +- +- memset(buf->buf, 0, buf->length); +- +-end: +- return ret; +-} +-static void sxe2_free_cq_buf(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf) +-{ +- sxe2_actual_buf_free(ctx, buf); +-} +- +-static struct ibv_cq_ex *ucreate_cq(struct ibv_context *context, +- struct ibv_cq_init_attr_ex *cq_attr, +- bool ext_cq) +-{ +- int ret; +- unsigned int ncqe; +- struct sxe2_ucreate_cq_ex cmd_ex; +- struct sxe2_ucreate_cq_ex_resp resp_ex; +- struct sxe2_rdma_ucontext *sctx; +- struct sxe2_ucq *ucq; +- struct sxe2_cq_uk_init_info info; +- size_t total_size = 0; +- +- sctx = container_of(context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- SXE2_VERBS_LOG_INFO_BDF("ucreate_cq start \n"); +- +- if ((cq_attr->cqe <= 0) || (cq_attr->cqe > sctx->uk_attrs.max_hw_cq_size - 1)) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("inv cqe:%d or vector:%d is invalid\n", cq_attr->cqe, cq_attr->comp_vector); +- goto null; +- } +- +- if (ucreate_cq_attr_check(sctx, cq_attr)) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("invalid param\n"); +- goto null; +- } +- +- ncqe = cq_attr->cqe; +- ucq = calloc(1, sizeof(*ucq)); +- if (!ucq) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("cq user mem alloc failed, mem size:%ld\n", sizeof(*ucq)); +- goto null; +- } +- +- if (pthread_spin_init(&ucq->lock, PTHREAD_PROCESS_SHARED)) { +- errno = EFAULT; +- SXE2_VERBS_LOG_ERROR_BDF("cq lock init failed\n"); +- goto err_lock_init; +- } +- +- memset(&cmd_ex, 0, sizeof(cmd_ex)); +- memset(&resp_ex, 0, sizeof(resp_ex)); +- memset(&info, 0, sizeof(info)); +- +- ucq->comp_vector = cq_attr->comp_vector; +- info.ncqe = get_cqe_count(cq_attr->cqe); +- if ((info.ncqe < SXE2_VERBS_MIN_CQ_SIZE) || (info.ncqe > SXE2_VERBS_MAX_CQ_SIZE)) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("cq size(%d)", info.ncqe); +- goto err_cq_base; +- } +- +- total_size = get_cq_total_bytes(info.ncqe); +- +- ucq->buf_size = total_size; +- if (sxe2_alloc_cq_buf(sctx, &ucq->buf, total_size)) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("ncqe %#x total_size %zu", ncqe, total_size); +- goto err_cq_base; +- } +- +- info.cq_base = ucq->buf.buf; +- if (!info.cq_base) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("cq buf alloc failed, ncqe:%d size:%ld\n", info.ncqe, total_size); +- goto err_cq_base; +- } +- memset(info.cq_base, 0, total_size); +- +- info.doorbell_note = sxe2_alloc_hw_buf(SXE2_DB_NOTE_SIZE); +- if (!info.doorbell_note) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("cq db note buf alloc failed, ncqe:%d size:%d\n", info.ncqe, SXE2_DB_NOTE_SIZE); +- goto err_alloc_db; +- } +- memset(info.doorbell_note, 0, SXE2_DB_NOTE_SIZE); +- set_32bit_val(info.doorbell_note, 4, (__u32)(SXE2_UCQ_DB_NOTE_CMD_SN) << SXE2_UCQ_DB_NOTE_CMD_SN_SHIFT); +- +- cq_attr->cqe = info.ncqe; +- cmd_ex.user_cq_buf = (__u64)((uintptr_t)info.cq_base); +- cmd_ex.user_cq_db_note = (__u64)((uintptr_t)info.doorbell_note); +- +- SXE2_VERBS_LOG_INFO_BDF("create cq comp_vector%d cqe:%d \n", cq_attr->comp_vector, cq_attr->cqe); +- +- ret = ibv_cmd_create_cq_ex(context, cq_attr, &ucq->verbs_cq, &cmd_ex.ibv_cmd, +- sizeof(cmd_ex), &resp_ex.ibv_resp, sizeof(resp_ex), 0); +- if (ret != 0) { +- errno = ret; +- SXE2_VERBS_LOG_ERROR_BDF("ibv cmd create cq err(%d)\n", ret); +- goto err_create_cq; +- } +- +- if (ext_cq) { +- sxe2_ibvcq_ex_fill_priv_funcs(ucq, cq_attr); +- } +- +- info.cq_id = resp_ex.cq_id; +- ucq->verbs_cq.cq.cqe = (int)ncqe; +- SXE2_VERBS_LOG_INFO_BDF("create cq cqn(%#x) cqsize %d\n", info.cq_id, info.ncqe); +- +- info.cqe_alloc_db = (__le64 *)sctx->cq_db_arm; +- sxe2_uk_cq_init(&ucq->cq, &info); +- +- return &ucq->verbs_cq.cq_ex; +- +-err_create_cq: +- sxe2_free_hw_buf(info.doorbell_note, SXE2_DB_NOTE_SIZE); +- +-err_alloc_db: +- sxe2_free_cq_buf(sctx, &ucq->buf); +- +-err_cq_base: +- pthread_spin_destroy(&ucq->lock); +- +-err_lock_init: +- free(ucq); +- +-null: +- return NULL; +-} +- +-struct ibv_cq *sxe2_ucreate_cq(struct ibv_context *context, int cqe, +- struct ibv_comp_channel *channel, int comp_vector) +-{ +- struct ibv_cq_ex *cq; +- struct ibv_cq_init_attr_ex cq_attr; +- struct sxe2_rdma_ucontext *sctx; +- +- sctx = container_of(context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- SXE2_VERBS_LOG_INFO_BDF("sxe2_ucreate_cq comp_vector%d cqe:%d \n", comp_vector, cqe); +- SXE2_VERBS_LOG_INFO_BDF("sxe2_ucreate_cq ctx num_comp_vectors:%d\n", context->num_comp_vectors); +- memset(&cq_attr, 0, sizeof(cq_attr)); +- cq_attr.cqe = cqe; +- cq_attr.channel = channel; +- cq_attr.comp_vector = comp_vector; +- +- cq = ucreate_cq(context, &cq_attr, false); +- if (!cq) { +- SXE2_VERBS_LOG_ERROR_BDF("user create cq failed\n"); +- } +- +- return cq ? ibv_cq_ex_to_cq(cq) : NULL; +-} +- +-struct ibv_cq_ex *sxe2_ucreate_cq_ex(struct ibv_context *context, +- struct ibv_cq_init_attr_ex *cq_attr) +-{ +- struct ibv_cq_ex *cq; +- struct sxe2_rdma_ucontext *sctx; +- +- sctx = container_of(context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (cq_attr->wc_flags & ~SXE2_CQ_SUPPORTED_WC_FLAGS) { +- SXE2_VERBS_LOG_ERROR_BDF("user create cq ex flags err\n"); +- errno = EOPNOTSUPP; +- cq = NULL; +- goto end; +- } +- +- cq = ucreate_cq(context, cq_attr, true); +- +-end: +- return cq; +-} +- +-int sxe2_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr) +-{ +- int ret; +- struct ibv_modify_cq cmd; +- struct sxe2_rdma_ucontext *sctx; +- +- if (!cq || !attr) { +- SXE2_VERBS_LOG_ERROR("user modify cq para err\n"); +- ret = EINVAL; +- goto end; +- } +- sctx = container_of(cq->context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- memset(&cmd, 0, sizeof(cmd)); +- +- ret = ibv_cmd_modify_cq(cq, attr, &cmd, sizeof(cmd)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv cmd modify cq err(%d)\n", ret); +- } +- +-end: +- return ret; +-} +- +-int sxe2_udestroy_cq(struct ibv_cq *cq) +-{ +- struct sxe2_ucq *ucq; +- int ret; +- struct sxe2_rdma_ucontext *sctx; +- +- sctx = container_of(cq->context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- ucq = container_of(cq, struct sxe2_ucq, verbs_cq.cq); +- +- ret = pthread_spin_destroy(&ucq->lock); +- if (ret != 0) { +- SXE2_VERBS_LOG_ERROR("user destroy cq lock failed\n"); +- goto end; +- } +- +- ret = ibv_cmd_destroy_cq(cq); +- if (ret != 0) { +- SXE2_VERBS_LOG_ERROR("ibv cmd destory cq err(%d)\n", ret); +- goto end; +- } +- +- sxe2_free_cq_buf(sctx, &ucq->buf); +- sxe2_free_hw_buf(ucq->cq.doorbell_note, SXE2_DB_NOTE_SIZE); +- ucq->cq.doorbell_note = NULL; +- free(ucq); +-end: +- return ret; +-} +- +diff -Naur rdma-core-48.0/providers/sxe2/db.c rdma-core-48.0.bak/providers/sxe2/db.c +--- rdma-core-48.0/providers/sxe2/db.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/db.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,622 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "log.h" +-#include "sxe2_common.h" +- +-int gsingle_threaded = 0; +- +-#define SXE2_VERBS_STRTOL_BASE 0 +-#define SXE2_VERBS_LLWQE_PER_DB 15 +-#define SXE2_VERBS_PF_DB_PAGE 2 +-#define SXE2_VERBS_VF_DB_PAGE 1 +-#define SXE2_VERBS_DED_LLWQE_PERCENT \ +- 3 +-#define SXE2_VERBS_LLWQE_OFFSET 0x100 +-#define SXE2_VERBS_LLWQE_DB_OFFSET \ +- 0x10 +-#define SXE2_VERBS_LLWQE_SIZE 256 +-#define SXE2_VERBS_LLWQE_DB_SIZE 16 +-#define SXE2_VERBS_CQ_ARM_OFFSET 0x8 +-#define SXE2_VERBS_CQ_INFO_OFFSET 0xC +- +-enum sxe2_verbs_cmd_attr_num { +- SXE2_VERBS_ONE_CMD_ATTR = 1, +- SXE2_VERBS_TWO_CMD_ATTRS = 2, +- SXE2_VERBS_THREE_CMD_ATTRS = 3, +- SXE2_VERBS_FOUR_CMD_ATTRS = 4, +- SXE2_VERBS_FIVE_CMD_ATTRS = 5, +-}; +- +- +-static struct sxe2_verbs_llwqe *db_ualloc_page(struct ibv_context *context, +- __u32 flags, +- __u32 db_mmap_size, +- __u32 db_page_id, +- __u64 db_mmap_offset) +-{ +- struct sxe2_verbs_llwqe *llwqe = NULL; +- struct sxe2_rdma_ucontext *sctx = to_sctx(context); +- struct sxe2_db_mmap_db_page_addr *page_addr_entry = NULL; +- struct sxe2_db_mmap_db_page_addr *mmap_addr_entry = NULL; +- struct sxe2_db_mmap_db_page_addr *tmp_mmap_addr_entry = NULL; +- __u32 db_page_id_allign; +- __u32 page_id_mod; +- long page_size = sysconf(_SC_PAGE_SIZE); +- __u32 db_page_multiplier = page_size/SXE2_DB_PAGE_SIZE; +- +- llwqe = calloc(1, sizeof(*llwqe)); +- if (!llwqe) { +- SXE2_VERBS_LOG_ERROR_BDF("llwqe calloc fail"); +- goto end; +- } +- llwqe->db_mmap_size = db_mmap_size; +- llwqe->db_page_id = db_page_id; +- llwqe->db_mmap_offset = db_mmap_offset; +- llwqe->db_page_addr = NULL; +- +- page_id_mod = llwqe->db_page_id % db_page_multiplier; +- if (page_id_mod != 0) { +- list_for_each_safe(&sctx->mmap_page_addr_list, mmap_addr_entry, tmp_mmap_addr_entry, +- list_entry) +- { +- db_page_id_allign = llwqe->db_page_id - page_id_mod; +- if (mmap_addr_entry->db_page_id == db_page_id_allign) { +- llwqe->db_page_addr = (void*)((__u64)mmap_addr_entry->db_page_addr + page_id_mod * SXE2_DB_PAGE_SIZE); +- llwqe->db_page_id = mmap_addr_entry->db_page_id + page_id_mod; +- break; +- } +- } +- } +- if (!llwqe->db_page_addr) { +- llwqe->db_page_addr = +- mmap(NULL, db_mmap_size, PROT_WRITE | PROT_READ, MAP_SHARED, +- context->cmd_fd, db_mmap_offset); +- if (llwqe->db_page_addr == MAP_FAILED) { +- SXE2_VERBS_LOG_ERROR_BDF("mmap db fail"); +- goto free_llwqe; +- } +- page_addr_entry = calloc(1, sizeof(*page_addr_entry)); +- if (!page_addr_entry) { +- SXE2_VERBS_LOG_ERROR_BDF("page addr entry alloc fail"); +- goto unmap_page; +- } +- page_addr_entry->db_page_id = llwqe->db_page_id - page_id_mod; +- page_addr_entry->db_page_addr = llwqe->db_page_addr; +- page_addr_entry->mmap_size = db_mmap_size; +- list_add_tail(&sctx->mmap_page_addr_list, &page_addr_entry->list_entry); +- +- if (page_id_mod != 0) { +- llwqe->db_page_addr = (void*)((__u64)llwqe->db_page_addr + page_id_mod * SXE2_DB_PAGE_SIZE); +- } +- } +- llwqe->mmaped_entry = true; +- +- SXE2_VERBS_LOG_DEBUG_BDF( +- "DB ALLOC:llwqe->db_handle:%#x, db_mmap_offset:%#llx, length:%#x, " +- "pageid:%#x, over", +- llwqe->db_handle, (__u64)llwqe->db_mmap_offset, llwqe->db_mmap_size, +- llwqe->db_page_id); +- +-end: +- return llwqe; +-unmap_page: +- munmap(llwqe->db_page_addr, db_mmap_size); +-free_llwqe: +- free(llwqe); +- llwqe = NULL; +- goto end; +-} +- +-static void db_uinsert_llwqe(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_verbs_llwqe *llwqe_first) +-{ +- struct list_head *head; +- struct sxe2_verbs_llwqe *llwqe = llwqe_first; +- __u64 idx; +- +- if (llwqe_first->qp_dedicated) { +- head = &sctx->dedicated_llwqe_list; +- } else { +- head = &sctx->shared_llwqe_list; +- } +- +- for (idx = 0; idx < SXE2_VERBS_LLWQE_PER_DB; idx++) { +- if (idx != 0) { +- llwqe = calloc(1, sizeof(*llwqe)); +- } +- if (!llwqe) { +- SXE2_VERBS_LOG_ERROR_BDF("llwqe is NULL"); +- goto end; +- } +- +- llwqe->wqe_addr = llwqe_first->db_page_addr + SXE2_VERBS_LLWQE_OFFSET + +- (idx * SXE2_VERBS_LLWQE_SIZE); +- llwqe->db_addr = llwqe_first->db_page_addr + +- SXE2_VERBS_LLWQE_DB_OFFSET + +- (idx * SXE2_VERBS_LLWQE_DB_SIZE); +- llwqe->wqe_buf_size = +- llwqe_first->no_ll_mode ? 0 : SXE2_VERBS_LLWQE_SIZE; +- llwqe->num_llwqe = llwqe_first->no_ll_mode ? 0 : 1; +- list_node_init(&llwqe->list_entry); +- list_add_tail(head, &llwqe->list_entry); +- pthread_spin_init(&llwqe->lock, PTHREAD_PROCESS_SHARED); +- if (idx != 0) { +- llwqe->db_page_addr = llwqe_first->db_page_addr; +- llwqe->db_page_id = llwqe_first->db_page_id; +- llwqe->db_handle = llwqe_first->db_handle; +- llwqe->no_ll_mode = llwqe_first->no_ll_mode; +- llwqe->db_mmap_offset = llwqe_first->db_mmap_offset; +- } +- if (llwqe_first->qp_dedicated) { +- sctx->alloc_dedicated_llwqes++; +- llwqe->qp_dedicated = true; +- } else { +- sctx->alloc_shared_llwqes++; +- llwqe->qp_shared = true; +- } +- llwqe->need_lock = llwqe_first->qp_shared && (!gsingle_threaded); +- } +- +-end: +- return; +-} +- +-int sxe2_uget_single_threaded_env(void) +-{ +- char *env; +- int ret_code = 0; +- +- env = getenv(SXE2_SINGLE_THREADED); +- if (env) { +- ret_code = strncmp(env, "1", 1) ? 0 : 1; +- } +- return ret_code; +-} +- +-int sxe2_uget_ll_mode(void) +-{ +- char *env; +- int ret_code = 0; +- +- env = getenv(SXE2_LL_MODE); +- if (env) { +- ret_code = strncmp(env, "0", 1) ? 1 : 0; +- } +- return ret_code; +-} +- +-int sxe2_uget_tot_llwqe(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_get_context_resp *resp) +-{ +- errno = 0; +- char *env; +- int total_llwqe = 0; +- int config_tot_llwqe = 0; +- int max_tot_llwqe = 0; +- int total_db_page = 0; +- int cal_db_page = 0; +- +- max_tot_llwqe = (int)(resp->max_db * SXE2_VERBS_LLWQE_PER_DB); +- +- if (resp->is_pf) { +- cal_db_page = (SXE2_VERBS_PF_DB_PAGE < resp->max_db) +- ? SXE2_VERBS_PF_DB_PAGE +- : (int)resp->max_db; +- } else { +- cal_db_page = (SXE2_VERBS_VF_DB_PAGE < resp->max_db) +- ? SXE2_VERBS_VF_DB_PAGE +- : (int)resp->max_db; +- } +- +- env = getenv(SXE2_TOTAL_LL_WQE); +- if (env) { +- config_tot_llwqe = (int)strtol(env, NULL, SXE2_VERBS_STRTOL_BASE); +- if (errno == ERANGE || config_tot_llwqe < 1) { +- total_db_page = cal_db_page; +- SXE2_VERBS_LOG_WARN_BDF( +- "Config total_llwqe %d invalid, modify total_db_page %d", +- config_tot_llwqe, total_db_page); +- } else if (config_tot_llwqe > max_tot_llwqe) { +- total_db_page = (int)resp->max_db; +- SXE2_VERBS_LOG_WARN_BDF("Config total_llwqe %d > max_tot_llwqe " +- "%d, modify total_db_page %d", +- config_tot_llwqe, max_tot_llwqe, +- total_db_page); +- } else { +- total_db_page = (config_tot_llwqe + SXE2_VERBS_LLWQE_PER_DB - 1) / +- SXE2_VERBS_LLWQE_PER_DB; +- SXE2_VERBS_LOG_INFO_BDF( +- "Config and align total_llwqe %d, total_db_page %d", +- config_tot_llwqe, total_db_page); +- } +- } else { +- total_db_page = cal_db_page; +- SXE2_VERBS_LOG_WARN_BDF( +- "No config total_llwqe, modify total_db_page %d", total_db_page); +- } +- +- total_llwqe = total_db_page * SXE2_VERBS_LLWQE_PER_DB; +- return total_llwqe; +-} +- +-int sxe2_uget_ded_llwqe(struct sxe2_rdma_ucontext *sctx, int total_llwqe) +-{ +- errno = 0; +- char *env; +- int cal_ded_llwqe = 0; +- int config_ded_llwqe = 0; +- int ded_llwqe = 0; +- +- cal_ded_llwqe = ((total_llwqe / SXE2_VERBS_LLWQE_PER_DB) / +- SXE2_VERBS_DED_LLWQE_PERCENT) * +- SXE2_VERBS_LLWQE_PER_DB; +- +- env = getenv(SXE2_DEDICATED_LL_WQE); +- if (env) { +- config_ded_llwqe = (int)strtol(env, NULL, SXE2_VERBS_STRTOL_BASE); +- if (errno == ERANGE || config_ded_llwqe < 0) { +- ded_llwqe = cal_ded_llwqe; +- SXE2_VERBS_LOG_WARN_BDF( +- "Config ded_llwqe %d invalid, modify ded_llwqe %d", +- config_ded_llwqe, ded_llwqe); +- } else if (config_ded_llwqe > total_llwqe) { +- ded_llwqe = total_llwqe; +- SXE2_VERBS_LOG_WARN_BDF( +- "Config ded_llwqe %d > total_llwqe %d, modify ded_llwqe %d", +- config_ded_llwqe, total_llwqe, total_llwqe); +- } else { +- ded_llwqe = ((config_ded_llwqe + SXE2_VERBS_LLWQE_PER_DB - 1) / +- SXE2_VERBS_LLWQE_PER_DB) * +- SXE2_VERBS_LLWQE_PER_DB; +- SXE2_VERBS_LOG_INFO_BDF("Config and align ded_llwqe %d", +- config_ded_llwqe); +- } +- } else { +- ded_llwqe = cal_ded_llwqe; +- SXE2_VERBS_LOG_WARN_BDF("No config ded_llwqe, modify ded_llwqe %d", +- ded_llwqe); +- } +- +- return ded_llwqe; +-} +- +-struct sxe2_verbs_llwqe * +-db_ualloc_page_and_llwqes(struct ibv_context *context, bool dedicated, +- __u32 db_mmap_size, +- __u32 db_page_id, +- __u64 db_mmap_offset) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_verbs_llwqe *llwqe = NULL; +- +- sctx = to_sctx(context); +- if (db_page_id == 0) { +- SXE2_VERBS_LOG_ERROR_BDF("have no llwqe page"); +- goto end; +- } +- llwqe = db_ualloc_page(context, SXE2_VERBS_DB_PAGE_TYPE_LLWQE, db_mmap_size, db_page_id, db_mmap_offset); +- if (!llwqe) { +- SXE2_VERBS_LOG_ERROR_BDF("alloc db page fail"); +- goto end; +- } +- +- if (dedicated) { +- llwqe->qp_dedicated = true; +- } else { +- llwqe->qp_shared = true; +- } +- +- db_uinsert_llwqe(sctx, llwqe); +- +-end: +- return llwqe; +-} +- +-static void *db_ummap(int fd, off_t offset) +-{ +- void *map = NULL; +- long page_size = sysconf(_SC_PAGE_SIZE); +- +- if (page_size < 0) { +- map = MAP_FAILED; +- goto end; +- } +- map = mmap(NULL, (size_t)page_size, PROT_WRITE | PROT_READ, MAP_SHARED, fd, +- offset); +- if (map == MAP_FAILED) { +- return map; +- } +- +- if (ibv_dontfork_range(map, page_size)) { +- munmap(map, page_size); +- return MAP_FAILED; +- } +-end: +- return map; +-} +- +-static void db_umunmap(void *map) +-{ +- long page_size = sysconf(_SC_PAGE_SIZE); +- if (page_size < 0) { +- goto end; +- } +- +- ibv_dofork_range(map, (size_t)page_size); +- munmap(map, page_size); +-end: +- return; +-} +- +-int sxe2_uinit_doorbell(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_get_context_resp *resp) +-{ +- int ret_code = 0; +- int config_ll_mode = 0; +- int total_llwqe = 0; +- int ded_llwqe = 0; +- __u64 mmap_key; +- long page_size = sysconf(_SC_PAGE_SIZE); +- +- gsingle_threaded = sxe2_uget_single_threaded_env( ); +- +- config_ll_mode = sxe2_uget_ll_mode( ); +- +- if (config_ll_mode) { +- total_llwqe = sxe2_uget_tot_llwqe(sctx, resp); +- if (total_llwqe) { +- ded_llwqe = sxe2_uget_ded_llwqe(sctx, total_llwqe); +- } else { +- ret_code = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF( +- "config ll_mode but total_llwqe=0, ret_code %d", ret_code); +- goto end; +- } +- } +- +- sctx->ll_mode = config_ll_mode; +- sctx->tot_ll_wqes = (__u32)total_llwqe; +- sctx->ded_ll_wqes = (__u32)ded_llwqe; +- sctx->shared_ll_wqes = (__u32)(total_llwqe - ded_llwqe); +- list_head_init(&sctx->shared_llwqe_list); +- list_head_init(&sctx->dedicated_llwqe_list); +- list_head_init(&sctx->mmap_page_addr_list); +- pthread_mutex_init(&sctx->alloc_llwqe_mutex, NULL); +- sctx->alloc_dedicated_llwqes = 0; +- sctx->alloc_shared_llwqes = 0; +- +- mmap_key = resp->db_mmap_key; +- sctx->qp_db_no_llwqe = db_ummap(sctx->ibv_ctx.context.cmd_fd, (off_t)mmap_key); +- if (sctx->qp_db_no_llwqe == MAP_FAILED) { +- ret_code = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("mmap db page fail, ret_code %d", ret_code); +- goto end; +- } +- if (page_size > SXE2_DB_PAGE_SIZE) +- sctx->qp_db_no_llwqe = (void*)((__u64)sctx->qp_db_no_llwqe + SXE2_DB_PAGE_SIZE); +- +- sctx->cq_db_arm = sctx->qp_db_no_llwqe + SXE2_VERBS_CQ_ARM_OFFSET; +- sctx->cq_db_info = sctx->qp_db_no_llwqe + SXE2_VERBS_CQ_INFO_OFFSET; +- +- SXE2_VERBS_LOG_INFO_BDF( +- "init doorbell over: single_thread %d, ll_mode %d, " +- "total_llwqes %#x, ded_llwqes %#x, shared_llwqes %#x, " +- "qp_db_no_llwqe addr %p, cq_db_arm addr %p", +- gsingle_threaded, sctx->ll_mode, sctx->tot_ll_wqes, sctx->ded_ll_wqes, +- sctx->shared_ll_wqes, sctx->qp_db_no_llwqe, sctx->cq_db_arm); +- +-end: +- return ret_code; +-} +- +-void sxe2_ufree_doorbell(struct ibv_context *context) +-{ +- struct sxe2_verbs_llwqe *llwqe = NULL; +- struct sxe2_verbs_llwqe *tmp_llwqe = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_db_mmap_db_page_addr *mmap_addr_entry = NULL; +- struct sxe2_db_mmap_db_page_addr *tmp_mmap_addr_entry = NULL; +- long page_size = sysconf(_SC_PAGE_SIZE); +- +- if (context == NULL) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv context is NULL"); +- goto end; +- } +- sctx = to_sctx(context); +- +- list_for_each_safe(&sctx->dedicated_llwqe_list, llwqe, tmp_llwqe, +- list_entry) +- { +- SXE2_VERBS_LOG_DEBUG_BDF( +- "DB FREE:llwqe->db_handle:%#x, db_mmap_offset:%#llx, " +- "db_mmap_size:%#x, db_page_id:%#x, start(dedicated)", +- llwqe->db_handle, (__u64)llwqe->db_mmap_offset, llwqe->db_mmap_size, +- llwqe->db_page_id); +- list_del(&llwqe->list_entry); +- free(llwqe); +- llwqe = NULL; +- } +- +- list_for_each_safe(&sctx->shared_llwqe_list, llwqe, tmp_llwqe, list_entry) +- { +- SXE2_VERBS_LOG_DEBUG_BDF( +- "DB FREE:llwqe->db_handle:%#x, db_mmap_offset:%#llx, " +- "db_mmap_size:%#x, db_page_id:%#x, start(shared)", +- llwqe->db_handle, (__u64)llwqe->db_mmap_offset, llwqe->db_mmap_size, +- llwqe->db_page_id); +- list_del(&llwqe->list_entry); +- free(llwqe); +- llwqe = NULL; +- } +- +- list_for_each_safe(&sctx->mmap_page_addr_list, mmap_addr_entry, tmp_mmap_addr_entry, +- list_entry) +- { +- if (mmap_addr_entry->db_page_addr && +- munmap(mmap_addr_entry->db_page_addr, mmap_addr_entry->mmap_size)) { +- SXE2_VERBS_LOG_ERROR_BDF("munmap db fail"); +- } +- SXE2_VERBS_LOG_DEBUG_BDF("DB FREE: munmap page_addr(%p) page_size(%u)", +- mmap_addr_entry->db_page_addr, mmap_addr_entry->mmap_size); +- list_del(&mmap_addr_entry->list_entry); +- free(mmap_addr_entry); +- mmap_addr_entry = NULL; +- } +- +- if (page_size > SXE2_DB_PAGE_SIZE) +- sctx->qp_db_no_llwqe = (void*)((__u64)sctx->qp_db_no_llwqe - SXE2_DB_PAGE_SIZE); +- db_umunmap(sctx->qp_db_no_llwqe); +- sctx->qp_db_no_llwqe = NULL; +- +- SXE2_VERBS_LOG_INFO_BDF("free doorbell over"); +- +-end: +- return; +-} +- +-static struct sxe2_verbs_llwqe *get_idle_shared_llwqe(struct sxe2_rdma_ucontext *sctx) +-{ +- struct sxe2_verbs_llwqe *llwqe = NULL; +- struct sxe2_verbs_llwqe *llwqe_entry = NULL; +- +- list_for_each(&sctx->shared_llwqe_list, llwqe_entry, list_entry) +- { +- if (!llwqe) { +- llwqe = llwqe_entry; +- } else { +- if (llwqe_entry->count < llwqe->count) { +- llwqe = llwqe_entry; +- } +- } +- } +- if (llwqe) { +- llwqe->count++; +- } +- +- return llwqe; +-} +- +-struct sxe2_verbs_llwqe *alloc_db_page_and_get_qp_llwqe(struct ibv_context *context, +- __u32 db_mmap_size, +- __u32 db_page_id, +- __u64 db_mmap_offset, +- __u8 alloc_page_type) +-{ +- struct sxe2_verbs_llwqe *llwqe = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- if (context == NULL) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv context is NULL"); +- goto end; +- } +- sctx = to_sctx(context); +- +- pthread_mutex_lock(&sctx->alloc_llwqe_mutex); +- +- if (alloc_page_type == SXE2_VERBS_DB_PAGE_TYPE_DEDICATED) { +- if (!db_ualloc_page_and_llwqes(context, true, db_mmap_size, db_page_id, db_mmap_offset)) { +- SXE2_VERBS_LOG_ERROR_BDF( +- "LL_WQE:Alloc new ded db page failed"); +- goto end; +- } +- llwqe = list_pop(&sctx->dedicated_llwqe_list, struct sxe2_verbs_llwqe, +- list_entry); +- if (llwqe) { +- llwqe->count++; +- } +- } else { +- if (!db_ualloc_page_and_llwqes(context, false, db_mmap_size, db_page_id, db_mmap_offset)) { +- SXE2_VERBS_LOG_ERROR_BDF("LLWQE:Alloc new shared db page failed"); +- goto end; +- } +- llwqe = get_idle_shared_llwqe(sctx); +- } +- +-end: +- pthread_mutex_unlock(&sctx->alloc_llwqe_mutex); +- if (llwqe) { +- SXE2_VERBS_LOG_DEBUG_BDF( +- "DB:Get LL_WQE:db_page_id %#x, ll_wqe_count %#x, dedicated %#x, " +- "shared %#x, wqe_addr %p, db_addr %p", +- llwqe->db_page_id, llwqe->count, llwqe->qp_dedicated, +- llwqe->qp_shared, llwqe->wqe_addr, llwqe->db_addr); +- } else { +- SXE2_VERBS_LOG_ERROR_BDF("DB:Get LL_WQE failed"); +- } +- return llwqe; +-} +- +-struct sxe2_verbs_llwqe *db_uget_qp_llwqe(struct ibv_context *context, +- bool *need_alloc_page, +- __u8 *alloc_page_type) +-{ +- struct sxe2_verbs_llwqe *llwqe = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- if (context == NULL) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv context is NULL"); +- goto end; +- } +- sctx = to_sctx(context); +- +- pthread_mutex_lock(&sctx->alloc_llwqe_mutex); +- +- llwqe = list_pop(&sctx->dedicated_llwqe_list, struct sxe2_verbs_llwqe, +- list_entry); +- if (llwqe) { +- llwqe->count++; +- *need_alloc_page = false; +- goto end; +- } +- if (sctx->alloc_dedicated_llwqes < sctx->ded_ll_wqes) { +- *need_alloc_page = true; +- *alloc_page_type = SXE2_VERBS_DB_PAGE_TYPE_DEDICATED; +- goto end; +- } +- +- llwqe = get_idle_shared_llwqe(sctx); +- +- if (sctx->alloc_shared_llwqes < sctx->shared_ll_wqes) { +- *need_alloc_page = true; +- *alloc_page_type = SXE2_VERBS_DB_PAGE_TYPE_SHARED; +- } +- +-end: +- pthread_mutex_unlock(&sctx->alloc_llwqe_mutex); +- if (llwqe) { +- SXE2_VERBS_LOG_DEBUG_BDF( +- "DB:Get LL_WQE:db_page_id %#x, ll_wqe_count %#x, dedicated %#x, " +- "shared %#x, wqe_addr %p, db_addr %p", +- llwqe->db_page_id, llwqe->count, llwqe->qp_dedicated, +- llwqe->qp_shared, llwqe->wqe_addr, llwqe->db_addr); +- } else { +- SXE2_VERBS_LOG_INFO_BDF("DB:No LL_WQE left, will alloc new db page."); +- } +- return llwqe; +-} +- +-void db_uput_qp_llwqe(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_verbs_llwqe *llwqe) +-{ +- if (!llwqe || (!llwqe->qp_dedicated && !llwqe->qp_shared)) { +- SXE2_VERBS_LOG_ERROR_BDF("llwqe:invalid argument"); +- goto end; +- } +- +- pthread_mutex_lock(&sctx->alloc_llwqe_mutex); +- if (llwqe->qp_dedicated) { +- list_add_tail(&sctx->dedicated_llwqe_list, &llwqe->list_entry); +- } else { +- llwqe->count--; +- } +- pthread_mutex_unlock(&sctx->alloc_llwqe_mutex); +- +-end: +- return; +-} +diff -Naur rdma-core-48.0/providers/sxe2/device_port.c rdma-core-48.0.bak/providers/sxe2/device_port.c +--- rdma-core-48.0/providers/sxe2/device_port.c 2026-05-26 10:42:01.852075195 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/device_port.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,277 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "sxe2_common.h" +-#include "device_port.h" +-#include "log.h" +-#include "verbs.h" +-#include +- +-#define INTEL_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL) +-static const struct verbs_match_ent sxe2_hca_table[] = { +- VERBS_DRIVER_ID(RDMA_DRIVER_SXE2), +- INTEL_HCA(SXE2_PCI_VENDOR_ID, SXE2_RDMA_DEVICE_ID_PF), +- INTEL_HCA(SXE2_PCI_VENDOR_ID, SXE2_RDMA_DEVICE_ID_VF), +- INTEL_HCA(SXE2_PCI_VENDOR_ID, SXE2_RDMA_DEVICE_ID_PF_PCIE_2_10G), +- INTEL_HCA(SXE2_PCI_VENDOR_ID, SXE2_RDMA_DEVICE_ID_VF_PCIE_2_10G), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_TL, SXE2_RDMA_DEVICE_ID_PF), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_TL, SXE2_RDMA_DEVICE_ID_VF), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_TL, SXE2_RDMA_DEVICE_ID_PF_PCIE_2_10G), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_TL, SXE2_RDMA_DEVICE_ID_VF_PCIE_2_10G), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_POP, SXE2_RDMA_DEVICE_ID_PF_POP), +- INTEL_HCA(SXE2_PCI_VENDOR_ID_FOR_POP, SXE2_RDMA_DEVICE_ID_VF_POP), +- {} +-}; +- +-void sxe2_ufree_context(struct ibv_context *context) +-{ +- struct sxe2_rdma_ucontext *sctx; +- +- sctx = container_of(context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- sxe2_ufree_pd(&sctx->sxe2_upd->ibv_pd); +- sctx->sxe2_upd = NULL; +- sxe2_ufree_doorbell(context); +- +- verbs_uninit_context(&sctx->ibv_ctx); +- if(sctx != NULL){ +- free(sctx); +- } +-} +- +-int sxe2_uquery_device_ex(struct ibv_context *ibctx, +- const struct ibv_query_device_ex_input *input, +- struct ibv_device_attr_ex *attr, size_t attr_size) +-{ +- int ret = SXE2_OK; +- struct ib_uverbs_ex_query_device_resp resp = {}; +- size_t resp_size = sizeof(resp); +- struct sxe2_rdma_ucontext *sctx; +- uint16_t major_ver_id; +- uint16_t sub_ver_id; +- uint16_t fix_ver_id; +- uint16_t build_id; +- +- sctx = to_sctx(ibctx); +- +- ret = ibv_cmd_query_device_any(ibctx, input, attr, attr_size, +- &resp, &resp_size); +- if (ret != SXE2_OK) { +- SXE2_VERBS_LOG_ERROR_BDF("device:uquery device err ret=%d\n", ret); +- goto end; +- } +- +- major_ver_id = FIELD_GET(SXE2_RDMA_FW_MAIN_VERSION_BITS, resp.base.fw_ver); +- sub_ver_id = FIELD_GET(SXE2_RDMA_FW_SUB_VERSION_BITS, resp.base.fw_ver); +- fix_ver_id = FIELD_GET(SXE2_RDMA_FW_FIX_VERSION_BITS, resp.base.fw_ver); +- build_id = FIELD_GET(SXE2_RDMA_FW_BUILD_NUMBER_BITS, resp.base.fw_ver); +- +- snprintf(attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver), +- "%u.%u.%u.%u", major_ver_id, sub_ver_id, fix_ver_id, build_id); +-end: +- return ret; +-} +- +-int sxe2_uquery_port(struct ibv_context *ibctx, uint8_t port, +- struct ibv_port_attr *attr) +-{ +- int ret = SXE2_OK; +- struct ibv_query_port cmd; +- struct sxe2_rdma_ucontext *sctx; +- sctx = to_sctx(ibctx); +- ret = ibv_cmd_query_port(ibctx, port, attr, &cmd, sizeof(cmd)); +- if (ret != SXE2_OK) { +- SXE2_VERBS_LOG_ERROR_BDF("device:uquery port err ret=%d\n", ret); +- goto end; +- } +- +-end: +- return ret; +-} +- +-static const struct verbs_context_ops sxe2_uctx_ops = { +- .alloc_pd = sxe2_ualloc_pd, +- .create_ah = sxe2_ucreate_ah, +- .create_cq = sxe2_ucreate_cq, +- .create_cq_ex = sxe2_ucreate_cq_ex, +- .create_qp = sxe2_ucreate_qp, +- .create_qp_ex = sxe2_ucreate_qp_ex, +- .dealloc_pd = sxe2_ufree_pd, +- .dereg_mr = sxe2_udereg_mr, +- .destroy_ah = sxe2_udestroy_ah, +- .destroy_cq = sxe2_udestroy_cq, +- .destroy_qp = sxe2_udestroy_qp, +- .modify_qp = sxe2_umodify_qp, +- .modify_cq = sxe2_umodify_cq, +- .poll_cq = sxe2_upoll_cq, +- .post_recv = sxe2_upost_recv, +- .post_send = sxe2_upost_send, +- .post_srq_recv = sxe2_upost_srq_recv, +- .query_device_ex = sxe2_uquery_device_ex, +- .query_port = sxe2_uquery_port, +- .query_qp = sxe2_uquery_qp, +- .reg_mr = sxe2_ureg_mr, +- .rereg_mr = sxe2_urereg_mr, +- .free_context = sxe2_ufree_context, +- .cq_event = sxe2_ucq_event, +- .reg_dmabuf_mr = sxe2_ureg_dmabuf_mr, +- .req_notify_cq = sxe2_uarm_cq, +- .create_srq = sxe2_ucreate_srq, +- .modify_srq = sxe2_umodify_srq, +- .query_srq = sxe2_uquery_srq, +- .destroy_srq = sxe2_udestroy_srq, +- .create_srq_ex = sxe2_ucreate_srq_ex, +- .get_srq_num = sxe2_uget_srq_num, +- .attach_mcast = sxe2_uattach_mcast, +- .detach_mcast = sxe2_udetach_mcast, +-#if 0 +- .resize_cq = sxe2_uresize_cq, +-#endif +-}; +- +-struct verbs_context *sxe2_ualloc_context(struct ibv_device *ib_dev, +- int cmd_fd, void *private_data) +-{ +- int ret = SXE2_OK; +- struct ibv_pd *ibv_pd; +- struct sxe2_rdma_ucontext *sctx; +- struct sxe2_get_context cmd = {}; +- struct sxe2_get_context_resp resp = {}; +- __u8 user_ver = SXE2_RDMA_USER_ABI_VER; +- char *env; +- +- sctx = verbs_init_and_alloc_context(ib_dev, cmd_fd, sctx, ibv_ctx, +- RDMA_DRIVER_SXE2); +- if (!sctx) +- { +- ret = -ENOMEM; +- goto end; +- } +- sctx->enable_io_log = 1; +- env = getenv(SXE2_LOG_IOLOG_ON); +- if ((env) && (0 == strncmp(env, "OFF", 3))){ +- SXE2_VERBS_LOG_INFO_BDF("device:disable user io log.\n"); +- sctx->enable_io_log = 0; +- } +-#if defined SXE2_CFG_DEBUG +- sctx->log_level = LOG_LEVEL_DEBUG; +-#else +- sctx->log_level = LOG_LEVEL_INVALID; +-#endif +- env = getenv(SXE2_VERBS_LOG_LEVEL); +- if ((env) && (0 == strncmp(env, "OFF", 3))){ +- SXE2_VERBS_LOG_INFO_BDF("device:disable user log.\n"); +- sctx->log_level = LOG_LEVEL_INVALID; +- } else if ((env) && (0 == strncmp(env, "FATAL", 5))) { +- sctx->log_level = LOG_LEVEL_FATAL; +- } else if ((env) && (0 == strncmp(env, "ERROR", 5))) { +- sctx->log_level = LOG_LEVEL_ERROR; +- } else if ((env) && (0 == strncmp(env, "WARN", 4))) { +- sctx->log_level = LOG_LEVEL_WARN; +- } else if ((env) && (0 == strncmp(env, "INFO", 4))) { +- sctx->log_level = LOG_LEVEL_INFO; +- } else if ((env) && (0 == strncmp(env, "DEBUG", 5))) { +- sctx->log_level = LOG_LEVEL_DEBUG; +- } else if ((env) && (0 == strncmp(env, "TRACE", 5))) { +- sctx->log_level = LOG_LEVEL_TRACE; +- } +- cmd.userspace_ver = user_ver; +- ret = ibv_cmd_get_context(&sctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp)); +- if (ret != SXE2_OK) { +- SXE2_VERBS_LOG_ERROR_BDF("device:cmd get context err ret=%d\n", ret); +- goto free_ctx; +- } +- verbs_set_ops(&sctx->ibv_ctx, &sxe2_uctx_ops); +- sctx->uk_attrs.feature_flags = resp.feature_flags; +- sctx->uk_attrs.hw_rev = resp.hw_rev; +- sctx->uk_attrs.max_hw_wq_frags = resp.max_hw_wq_frags; +- sctx->uk_attrs.max_hw_read_sges = resp.max_hw_read_sges; +- sctx->uk_attrs.max_hw_inline = resp.max_hw_inline; +- sctx->uk_attrs.max_hw_rq_quanta = resp.max_hw_rq_quanta; +- sctx->uk_attrs.max_hw_wq_quanta = resp.max_hw_wq_quanta; +- sctx->uk_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk; +- sctx->uk_attrs.max_hw_cq_size = resp.max_hw_cq_size; +- sctx->uk_attrs.min_hw_cq_size = resp.min_hw_cq_size; +- sctx->uk_attrs.max_hw_srq_quanta = resp.max_hw_srq_quanta; +- sctx->uk_attrs.max_hw_srq_wr = resp.max_hw_srq_wr; +- sctx->abi_ver = user_ver; +- if (resp.comp_mask & SXE2_ALLOC_UCTX_MIN_HW_WQ_SIZE) { +- sctx->uk_attrs.min_hw_wq_size = resp.min_hw_wq_size; +- } else { +- sctx->uk_attrs.min_hw_wq_size = SXE2_QP_SW_MIN_WQSIZE; +- } +- memcpy(sctx->bdf, (const char *)resp.bdf, sizeof(sctx->bdf) - 1); +- +- ret = sxe2_uinit_doorbell(sctx, &resp); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("device:init door bell err ret=%d\n", ret); +- goto free_ctx; +- } +- +- ibv_pd = sxe2_ualloc_pd(&sctx->ibv_ctx.context); +- if (!ibv_pd) { +- SXE2_VERBS_LOG_ERROR_BDF("device:alloc pd err\n"); +- goto free_db; +- } +- +- ibv_pd->context = &sctx->ibv_ctx.context; +- sctx->sxe2_upd = container_of(ibv_pd, struct sxe2_upd, ibv_pd); +- +- sxe2_spinlock_init(&sctx->hugetlb_lock, true); +- list_head_init(&sctx->hugetlb_list); +- +- return &sctx->ibv_ctx; +- +-free_db: +- sxe2_ufree_doorbell(&sctx->ibv_ctx.context); +-free_ctx: +- free(sctx); +- sctx = NULL; +-end: +- return NULL; +-} +- +-void sxe2_uninit_device(struct verbs_device *verbs_device) +-{ +- struct sxe2_rdma_udevice *dev; +- +- log_uninit0(); +- +- dev = container_of(&verbs_device->device, struct sxe2_rdma_udevice, +- ibv_dev.device); +- free(dev); +-} +- +-struct verbs_device *sxe2_ualloc_device(struct verbs_sysfs_dev *sysfs_dev) +-{ +- struct sxe2_rdma_udevice *dev; +- +- log_init0(false); +- +- dev = calloc(1, sizeof(*dev)); +- if (!dev){ +- return NULL; +- } +- return &dev->ibv_dev; +-} +- +-static const struct verbs_device_ops sxe2_udev_ops = { +- .alloc_context = sxe2_ualloc_context, +- .alloc_device = sxe2_ualloc_device, +- .match_max_abi_version = SXE2_RDMA_MAX_ABI_VERSION, +- .match_min_abi_version = SXE2_RDMA_MIN_ABI_VERSION, +- .match_table = sxe2_hca_table, +- .name = "sxe2_rdma", +- .uninit_device = sxe2_uninit_device, +-}; +- +-PROVIDER_DRIVER(sxe2, sxe2_udev_ops); +diff -Naur rdma-core-48.0/providers/sxe2/device_port.h rdma-core-48.0.bak/providers/sxe2/device_port.h +--- rdma-core-48.0/providers/sxe2/device_port.h 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/device_port.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,48 +0,0 @@ +-#ifndef DEVICE_H +-#define DEVICE_H +-#define SXE2_PCI_VENDOR_ID 0x1FF2 +-#define SXE2_PCI_VENDOR_ID_FOR_TL 0x206F +-#define SXE2_PCI_VENDOR_ID_FOR_POP 0x1D94 +- +-#define SXE2_RDMA_DEVICE_ID_PF 0x10B1 +-#define SXE2_RDMA_DEVICE_ID_VF 0x10B2 +-#define SXE2_RDMA_DEVICE_ID_PF_PCIE_2_10G 0x10B3 +-#define SXE2_RDMA_DEVICE_ID_VF_PCIE_2_10G 0x10B4 +-#define SXE2_RDMA_DEVICE_ID_PF_POP 0x1260 +-#define SXE2_RDMA_DEVICE_ID_VF_POP 0x126F +- +-#define SXE2_RDMA_USER_ABI_VER 1 +-#define SXE2_RDMA_MIN_ABI_VERSION 1 +-#define SXE2_RDMA_MAX_ABI_VERSION 1 +-#define SXE2_OK 0 +-#define SXE2_QP_SW_MIN_WQSIZE 8 +-#define SXE2_RDMA_FW_BUILD_NUMBER_BITS GENMASK(7, 0) +-#define SXE2_RDMA_FW_FIX_VERSION_BITS GENMASK(15, 8) +-#define SXE2_RDMA_FW_SUB_VERSION_BITS GENMASK(23, 16) +-#define SXE2_RDMA_FW_MAIN_VERSION_BITS GENMASK(31, 24) +- +-enum { +- SXE2_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1, +-}; +- +-struct sxe2_rdma_udevice { +- struct verbs_device ibv_dev; +-}; +- +-void sxe2_ufree_context(struct ibv_context *context); +- +-int sxe2_uquery_device_ex(struct ibv_context *ibctx, +- const struct ibv_query_device_ex_input *input, +- struct ibv_device_attr_ex *attr, size_t attr_size); +- +-int sxe2_uquery_port(struct ibv_context *ibctx, uint8_t port, +- struct ibv_port_attr *attr); +- +-struct verbs_context *sxe2_ualloc_context(struct ibv_device *ib_dev, +- int cmd_fd, void *private_data); +- +-void sxe2_uninit_device(struct verbs_device *verbs_device); +- +-struct verbs_device *sxe2_ualloc_device(struct verbs_sysfs_dev *sysfs_dev); +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/io.c rdma-core-48.0.bak/providers/sxe2/io.c +--- rdma-core-48.0/providers/sxe2/io.c 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/io.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,3829 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include "sxe2_common.h" +-#include "io.h" +-#include "sxe2_abi.h" +-#include "ring.h" +-#include "log.h" +-#include +- +-static const rdma_disp_func g_rdma_op[SXE2_RDMA_MAX_ID] = { +- sxe2_hw_send, +- sxe2_hw_inline_send, +- sxe2_hw_rdma_write, +- sxe2_hw_inline_rdma_write, +- sxe2_hw_rdma_read, +- sxe2_hw_mw_bind, +- sxe2_hw_local_invalidate, +-}; +- +-static const int g_frag2quanta[SXE2_MAX_FRAGCNT] = { 1, 1, 2, 2, 3, 3, 4, 4, +- 5, 5, 6, 6, 7, 7, 8, 8 }; +-#define DUMP_RDMA_SEND_HDR(psnd) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("SND_WQE_HDR: (%#llx)\n" \ +- "remote_inv_key: %#x \n" \ +- "op : %#x \n" \ +- "addfragcnt : %d \n" \ +- "report_rtt : %d \n" \ +- "imme_data_flag: %d \n" \ +- "push_wqe : %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64((psnd)->val), \ +- (psnd)->field.remote_inv_rkey, \ +- (psnd)->field.op, \ +- (psnd)->field.addfragcnt, \ +- (psnd)->field.report_rtt, \ +- (psnd)->field.imme_data_flag, \ +- (psnd)->field.push_wqe, \ +- (psnd)->field.read_fence, \ +- (psnd)->field.local_fence, \ +- (psnd)->field.signaled_completion, \ +- (psnd)->field.wqe_valid); \ +- } while (0) +-#define DUMP_RDMA_SEND_INLINE_HDR(pinsnd) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("INSND_WQE_HDR: (%#llx)\n" \ +- "remote_inv_key: %#x \n" \ +- "op : %#x \n" \ +- "report_rtt : %d \n" \ +- "imme_data_flag: %d \n" \ +- "inline_datalen: %d \n" \ +- "push_wqe : %d \n" \ +- "inline_dataflg: %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64((pinsnd)->val), \ +- (pinsnd)->field.remote_inv_rkey, \ +- (pinsnd)->field.op, \ +- (pinsnd)->field.report_rtt, \ +- (pinsnd)->field.imme_data_flag, \ +- (pinsnd)->field.inline_data_len, \ +- (pinsnd)->field.push_wqe, \ +- (pinsnd)->field.inline_data_flag, \ +- (pinsnd)->field.read_fence, \ +- (pinsnd)->field.local_fence, \ +- (pinsnd)->field.signaled_completion, \ +- (pinsnd)->field.wqe_valid); \ +- } while (0) +-#define DUMP_RDMA_WRITE_HDR(pwrite) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("WRITE_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n" \ +- "op : %#x \n" \ +- "addfragcnt : %d \n" \ +- "report_rtt : %d \n" \ +- "imme_data_flag: %d \n" \ +- "push_wqe : %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64(pwrite->val), \ +- pwrite->field.remote_key, \ +- pwrite->field.op, \ +- pwrite->field.addfragcnt, \ +- pwrite->field.report_rtt, \ +- pwrite->field.imme_data_flag, \ +- pwrite->field.push_wqe, \ +- pwrite->field.read_fence, \ +- pwrite->field.local_fence, \ +- pwrite->field.signaled_completion, \ +- pwrite->field.wqe_valid); \ +- } while (0) +-#define DUMP_RDMA_WRITE_INLINE_HDR(pinwrite) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("INWRITE_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n" \ +- "op : %#x \n" \ +- "report_rtt : %d \n" \ +- "imme_data_flag: %d \n" \ +- "inline_datalen: %d \n" \ +- "push_wqe : %d \n" \ +- "inline_dataflg: %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64((pinwrite)->val), \ +- (pinwrite)->field.remote_key, \ +- (pinwrite)->field.op, \ +- (pinwrite)->field.report_rtt, \ +- (pinwrite)->field.imme_data_flag, \ +- (pinwrite)->field.inline_data_len, \ +- (pinwrite)->field.push_wqe, \ +- (pinwrite)->field.inline_data_flag, \ +- (pinwrite)->field.read_fence, \ +- (pinwrite)->field.local_fence, \ +- (pinwrite)->field.signaled_completion, \ +- (pinwrite)->field.wqe_valid); \ +- } while (0) +-#define DUMP_RDMA_READ_HDR(pread) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("READ_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n" \ +- "op : %#x \n" \ +- "addfragcnt : %d \n" \ +- "report_rtt : %d \n" \ +- "push_wqe : %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64((pread)->val), \ +- (pread)->field.remote_key, \ +- (pread)->field.op, \ +- (pread)->field.addfragcnt, \ +- (pread)->field.report_rtt, \ +- (pread)->field.push_wqe, \ +- (pread)->field.read_fence, \ +- (pread)->field.local_fence, \ +- (pread)->field.signaled_completion, \ +- (pread)->field.wqe_valid); \ +- } while (0) +-#define DUMP_RDMA_LOCAL_INV_HDR(pinv) \ +- do { \ +- SXE2_VERBS_LOG_DEBUG_BDF("LOCALINVKEY_WQE_HDR: (%#llx)\n" \ +- "op : %#x \n" \ +- "push_wqe : %d \n" \ +- "read_fence : %d \n" \ +- "local_fence : %d \n" \ +- "signaled : %d \n" \ +- "wqe_valid : %d\n", \ +- (unsigned long long)htole64((pinv)->val), \ +- (pinv)->field.op, \ +- (pinv)->field.push_wqe, \ +- (pinv)->field.read_fence, \ +- (pinv)->field.local_fence, \ +- (pinv)->field.signaled_completion, \ +- (pinv)->field.wqe_valid); \ +- } while (0) +-static void sxe2_dump_wqe_hdr(struct sxe2_uqp *uqp, __u64 *hdr, +- enum sxe2_disp_id func) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- union sxe2_send_hdr *psnd; +- union sxe2_send_inline_hdr *pinsnd; +- union sxe2_write_hdr *pwrite; +- union sxe2_write_inline_hdr *pinwrite; +- union sxe2_read_hdr *pread; +- union sxe2_inval_hdr *pinv; +- +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (sctx->enable_io_log == false) { +- return; +- } +- +- switch (func) { +- case SXE2_RDMA_SEND: +- psnd = (union sxe2_send_hdr *)hdr; +- DUMP_RDMA_SEND_HDR(psnd); +- break; +- case SXE2_RDMA_SEND_INLINE: +- pinsnd = (union sxe2_send_inline_hdr *)hdr; +- DUMP_RDMA_SEND_INLINE_HDR(pinsnd); +- break; +- case SXE2_RDMA_WRITE: +- pwrite = (union sxe2_write_hdr *)hdr; +- DUMP_RDMA_WRITE_HDR(pwrite); +- break; +- case SXE2_RDMA_WRITE_INLINE: +- pinwrite = (union sxe2_write_inline_hdr *)hdr; +- DUMP_RDMA_WRITE_INLINE_HDR(pinwrite); +- break; +- case SXE2_RDMA_READ: +- pread = (union sxe2_read_hdr *)hdr; +- DUMP_RDMA_READ_HDR(pread); +- break; +- case SXE2_RDMA_LOCAL_INV: +- pinv = (union sxe2_inval_hdr *)hdr; +- DUMP_RDMA_LOCAL_INV_HDR(pinv); +- break; +- default: +- SXE2_VERBS_LOG_ERROR_BDF("UNSUPPORT OP CODE\n"); +- } +-} +- +-static void sxe2_dump_wqe(struct sxe2_qp_common *qp, __le64 *wqe, +- __u16 quanta, __u32 wqe_idx, const char *desc) +-{ +- struct sxe2_uqp *uqp =container_of(qp, struct sxe2_uqp, qp); +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le32 *p = (__le32 *)wqe; +- int i, offset = 0; +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (sctx->enable_io_log) { +- if (desc) { +- SXE2_VERBS_LOG_DEBUG_BDF("SQWQE DUMP TYPE=[%s], qpn [0x%x], wqe_idx"\ +- "[0x%x] quanta [%u]:\n", desc, qp->qpn, wqe_idx, quanta); +- } +- else { +- SXE2_VERBS_LOG_DEBUG_BDF("RQWQE DUMP, qpn [0x%x], wqe_idx [0x%x]:\n", +- qp->qpn, wqe_idx); +- } +- for (i = 0; i < quanta * SXE2_QP_WQE_MIN_SIZE; i += 32) { +- SXE2_VERBS_LOG_DEBUG_BDF("[qpn 0x%x][offset %u] %08X %08X %08X %08X " +- "%08X %08X %08X %08X\n", +- qp->qpn, offset, +- le32toh(p[0]), le32toh(p[1]), +- le32toh(p[2]), le32toh(p[3]), +- le32toh(p[4]), le32toh(p[5]), +- le32toh(p[6]), le32toh(p[7])); +- p += 8; +- offset += 32; +- } +- } +-} +- +-static inline int sxe2_fragcnt_to_quanta_cnt(__u32 frag_cnt, __u16 *quanta) +-{ +- if (frag_cnt >= SXE2_MAX_FRAGCNT) { +- return EINVAL; +- } +- +- *quanta = (__u16)g_frag2quanta[frag_cnt]; +- return 0; +-} +- +-static inline __u16 sxe2_inline_to_quanta_cnt(__u32 data_size) +-{ +- if (data_size <= 8) { +- return SXE2_QP_WQE_MIN_QUANTA; +- } else if (data_size <= 39) { +- return 2; +- } else if (data_size <= 70) { +- return 3; +- } else if (data_size <= 101) { +- return 4; +- } else if (data_size <= 132) { +- return 5; +- } else if (data_size <= 163) { +- return 6; +- } else if (data_size <= 194) { +- return 7; +- } else { +- return 8; +- } +-} +- +-static void sxe2_qp_ring_normal_db(struct sxe2_qp_common *qp) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (SXE2_RING_CURRENT_HEAD(qp->sq_ring) != qp->initial_ring.head) { +- udma_to_device_barrier(); +- db_wr32(qp->qpn, qp->qp_db_no_llwqe); +- +- if (qp->push_dropped) +- qp->push_dropped = false; +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("DB NOTIFY: QPN (%#x -> %p) PI %#x\n", +- qp->qpn, qp->qp_db_no_llwqe, qp->sq_ring.head); +- } +- } +- qp->initial_ring.head = qp->sq_ring.head; +-} +- +-static void sxe2_qp_push_wqe(struct sxe2_qp_common *qp, __le64 *wqe, +- __u16 quanta, __u32 wqe_idx) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *push; +- struct sxe2_verbs_llwqe *llwqe; +-#if defined(SXE2_CFG_DEBUG) && defined(SXE2_SUPPORT_INJECT) +- char *env; +-#endif +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- if (!qp->llwqe_mode) { +- sxe2_qp_ring_normal_db(qp); +- } else { +- llwqe = qp->verbs_llwqe; +- if ((llwqe != NULL) && (llwqe->need_lock)) { +- pthread_spin_lock(&llwqe->lock); +- } +- push = (__le64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x7) * 0x20); +-#if defined(SXE2_CFG_DEBUG) && defined(SXE2_SUPPORT_INJECT) +- env = getenv(SXE2_INJECT_LLWQE_ERR); +- if ((env) && (0 == strncmp(env, "yes", 3))){ +- memcpy(push, wqe, 4); +- } +-#else +- memcpy(push, wqe, quanta * SXE2_QP_WQE_MIN_SIZE); +-#endif +- set_32bit_val(qp->push_db, 0, +- FIELD_PREP(SXE2_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | +- qp->qpn); +- qp->initial_ring.head = qp->sq_ring.head; +- qp->llwqe_mode = true; +- qp->push_dropped = false; +- if ((llwqe != NULL) && (llwqe->need_lock)) { +- pthread_spin_unlock(&llwqe->lock); +- } +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("DB NOTIFY(LLWQE): QPN (%#x ->db %#lx llwqe %#lx) idx %#x\n", +- qp->qpn, (uintptr_t)qp->push_db, (uintptr_t)qp->push_wqe, wqe_idx >> 3); +- } +- } +-} +- +-static void sxe2_qp_flush_wqe(struct ibv_qp *qp, bool flush_sq, bool flush_rq) +-{ +- int ret; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_umodify_qp cmd = {}; +- struct sxe2_umodify_qp_resp resp = {}; +- struct ibv_qp_attr qp_attr = {}; +- +- uqp = container_of(qp, struct sxe2_uqp, verbs_qp.qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- qp_attr.qp_state = IBV_QPS_ERR; +- cmd.sq_flush = flush_sq; +- cmd.rq_flush = flush_rq; +- +- ret = ibv_cmd_modify_qp_ex(qp, &qp_attr, IBV_QP_STATE, +- &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); +- if (ret || sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("ibv_modify_qp_ex ret(%d)\n", ret); +- } +-} +- +-static inline void sxe2_set_qkeyqpn(__le64 *wqe, __u32 offset, __u32 qkey, +- __u32 qpn) +-{ +- union sxe2_dqpn_data msg; +- +- msg.val = 0; +- msg.field.dest_qkey = qkey; +- msg.field.dest_qpn = qpn; +- +- wqe[offset >> 3] = htole64(msg.val); +-} +- +-static inline void sxe2_set_remote_offset(__le64 *wqe, __u32 offset, +- __u64 remote_offset) +-{ +- wqe[offset >> 3] = htole64(remote_offset); +-} +- +-static void sxe2_set_send_hdr(__le64 *wqe, __u32 value, +- struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_send_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.remote_inv_rkey = +- wr_info->rkey_to_inv | wr_info->op_info.send.ah_id; +- hdr.field.op = wr_info->op_type; +- hdr.field.addfragcnt = value; +- hdr.field.report_rtt = wr_info->report_rtt; +- hdr.field.imme_data_flag = wr_info->imm_data_valid; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("SND_WQE_HDR: (%#llx)\n" \ +- "remote_inv_key: %#x \n"\ +- "op : %#x \n"\ +- "addfragcnt : %d \n"\ +- "report_rtt : %d \n"\ +- "imme_data_flag: %d \n"\ +- "push_wqe : %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.remote_inv_rkey, +- hdr.field.op, +- hdr.field.addfragcnt, +- hdr.field.report_rtt, +- hdr.field.imme_data_flag, +- hdr.field.push_wqe, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_inlinesnd_hdr(__le64 *wqe, __u32 value, +- struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_send_inline_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.remote_inv_rkey = +- wr_info->rkey_to_inv | wr_info->op_info.send.ah_id; +- hdr.field.op = wr_info->op_type; +- hdr.field.report_rtt = wr_info->report_rtt; +- hdr.field.imme_data_flag = wr_info->imm_data_valid; +- hdr.field.inline_data_len = value; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.inline_data_flag = 1; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("INSND_WQE_HDR: (%#llx)\n" \ +- "remote_inv_key: %#x \n"\ +- "op : %#x \n"\ +- "report_rtt : %d \n"\ +- "imme_data_flag: %d \n"\ +- "inline_datalen: %d \n"\ +- "push_wqe : %d \n"\ +- "inline_dataflg: %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.remote_inv_rkey, +- hdr.field.op, +- hdr.field.report_rtt, +- hdr.field.imme_data_flag, +- hdr.field.inline_data_len, +- hdr.field.push_wqe, +- hdr.field.inline_data_flag, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_write_hdr(__le64 *wqe, __u32 value, +- struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_write_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.remote_key = wr_info->op_info.rdma_write.rem_addr.lkey; +- hdr.field.op = wr_info->op_type; +- hdr.field.addfragcnt = value; +- hdr.field.report_rtt = wr_info->report_rtt; +- hdr.field.imme_data_flag = wr_info->imm_data_valid; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("WRITE_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n"\ +- "op : %#x \n"\ +- "addfragcnt : %d \n"\ +- "report_rtt : %d \n"\ +- "imme_data_flag: %d \n"\ +- "push_wqe : %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.remote_key, +- hdr.field.op, +- hdr.field.addfragcnt, +- hdr.field.report_rtt, +- hdr.field.imme_data_flag, +- hdr.field.push_wqe, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_inlinewrite_hdr(__le64 *wqe, __u32 value, +- struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_write_inline_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.remote_key = wr_info->op_info.rdma_write.rem_addr.lkey; +- hdr.field.op = wr_info->op_type; +- hdr.field.report_rtt = wr_info->report_rtt; +- hdr.field.imme_data_flag = wr_info->imm_data_valid; +- hdr.field.inline_data_len = value; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.inline_data_flag = 1; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("INWRITE_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n"\ +- "op : %#x \n"\ +- "report_rtt : %d \n"\ +- "imme_data_flag: %d \n"\ +- "inline_datalen: %d \n"\ +- "push_wqe : %d \n"\ +- "inline_dataflg: %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.remote_key, +- hdr.field.op, +- hdr.field.report_rtt, +- hdr.field.imme_data_flag, +- hdr.field.inline_data_len, +- hdr.field.push_wqe, +- hdr.field.inline_data_flag, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_read_hdr(__le64 *wqe, __u32 addfragcnt, +- struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp, bool ord_fence) +-{ +- union sxe2_read_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.remote_key = wr_info->op_info.rdma_read.rem_addr.lkey; +- hdr.field.op = wr_info->op_type; +- hdr.field.addfragcnt = addfragcnt; +- hdr.field.report_rtt = wr_info->report_rtt; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.read_fence = wr_info->read_fence || ord_fence ? 1 : 0; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("READ_WQE_HDR: (%#llx)\n" \ +- "remote_key : %#x \n"\ +- "op : %#x \n"\ +- "addfragcnt : %d \n"\ +- "report_rtt : %d \n"\ +- "push_wqe : %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.remote_key, +- hdr.field.op, +- hdr.field.addfragcnt, +- hdr.field.report_rtt, +- hdr.field.push_wqe, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_bindmw_hdr(__le64 *wqe, struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_bindmw_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.op = wr_info->op_type; +- hdr.field.access = wr_info->op_info.bind_window.ena_reads << 2 | +- wr_info->op_info.bind_window.ena_writes << 3; +- hdr.field.va_base_flag = +- (wr_info->op_info.bind_window.addressing_type == +- SXE2_ADDR_TYPE_VA_BASED ? 1 : 0); +- hdr.field.mw_type = +- wr_info->op_info.bind_window.mem_window_type_1 ? 0 : 1; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("BINDMW_WQE_HDR: (%#llx)\n" \ +- "op : %#x \n"\ +- "access : %d \n"\ +- "va_base_flag : %d \n"\ +- "mw_type : %d \n"\ +- "push_wqe : %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.op, +- hdr.field.access, +- hdr.field.va_base_flag, +- hdr.field.mw_type, +- hdr.field.push_wqe, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_invalidate_hdr(__le64 *wqe, struct sxe2_wr_info *wr_info, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_inval_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.op = wr_info->op_type; +- hdr.field.push_wqe = wr_info->push_wqe; +- hdr.field.read_fence = wr_info->read_fence; +- hdr.field.local_fence = wr_info->local_fence; +- hdr.field.signaled_completion = wr_info->signaled; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("LOCALINVKEY_WQE_HDR: (%#llx)\n" \ +- "op : %#x \n"\ +- "push_wqe : %d \n"\ +- "read_fence : %d \n"\ +- "local_fence : %d \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.op, +- hdr.field.push_wqe, +- hdr.field.read_fence, +- hdr.field.local_fence, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static void sxe2_set_nop_hdr(__le64 *wqe, __u32 offset, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_nop_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.op = SXE2_OP_TYPE_NOP; +- hdr.field.signaled_completion = false; +- hdr.field.wqe_valid = qp->swqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, offset, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("NOP_WQE_HDR: (%#llx)\n" \ +- "op : %#x \n"\ +- "signaled : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.op, +- hdr.field.signaled_completion, +- hdr.field.wqe_valid); +- } +-} +- +-static int sxe2_hw_nop(struct sxe2_qp_common *qp) +-{ +- __le64 *wqe; +- __u32 wqe_idx; +- +- if (!qp->sq_ring.head) +- return EINVAL; +- +- wqe_idx = SXE2_RING_CURRENT_HEAD(qp->sq_ring); +- wqe = qp->sq_base[wqe_idx].elem; +- +- qp->sq_wrtrk_array[wqe_idx].quanta = SXE2_QP_WQE_MIN_QUANTA; +- +- set_64bit_val(wqe, 0, 0); +- set_64bit_val(wqe, 8, 0); +- set_64bit_val(wqe, 16, 0); +- +- sxe2_set_nop_hdr(wqe, 24, qp); +- +- return 0; +-} +- +-static void sxe2_set_rcvq_hdr(__le64 *wqe, __u32 addl_frag_cnt, +- struct sxe2_qp_common *qp) +-{ +- union sxe2_rq_hdr hdr; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- hdr.val = 0; +- hdr.field.addfragcnt = addl_frag_cnt; +- hdr.field.wqe_valid = qp->rwqe_polarity; +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("RCV_WQE_HDR: (%#llx)\n" \ +- "addfragcnt : %d \n"\ +- "wqe_valid : %d\n", +- (unsigned long long)htole64(hdr.val), +- hdr.field.addfragcnt, +- hdr.field.wqe_valid); +- } +-} +- +-static inline void sxe2_set_immedata(__le64 *wqe, __u32 offset, __u64 imm_data) +-{ +- wqe[offset >> 3] = htole64(imm_data); +-} +- +-static inline void sxe2_set_sgelist_data(void *wqe, __u32 offset, +- struct ibv_sge *sge, __u8 valid) +-{ +- struct sxe2_frag_data *pmsg = (struct sxe2_frag_data *)wqe; +- uint32_t len; +- +- pmsg = pmsg + offset / sizeof(*pmsg); +- if (sge) { +- len = (sge->length & ((uint32_t)1 << 31)) ? 0 : sge->length; +- pmsg->tag_offset = htole64(sge->addr); +- pmsg->offset8.field.frag_valid = valid; +- pmsg->offset8.field.frag_len = len; +- pmsg->offset8.field.stag = sge->lkey; +- pmsg->offset8.val = htole64(pmsg->offset8.val); +- } else { +- pmsg->tag_offset = 0; +- pmsg->offset8.field.frag_valid = valid; +- pmsg->offset8.val = htole64(pmsg->offset8.val); +- } +-} +- +-static void sxe2_set_inline_data_seg_list(__u8 *wqe, struct ibv_sge *sge_list, +- __u32 num_sges, __u8 polarity) +-{ +- __u8 inline_valid = (__u8)(polarity << SXE2_INLINE_VALID_S); +- __u32 quanta_bytes_remaining = 8; +- __u32 i; +- bool first_quanta = true; +- +- wqe += 8; +- +- for (i = 0; i < num_sges; i++) { +- __u8 *cur_sge = (__u8 *)(uintptr_t)sge_list[i].addr; +- __u32 sge_len = sge_list[i].length; +- +- while (sge_len) { +- __u32 bytes_copied; +- +- bytes_copied = min(sge_len, quanta_bytes_remaining); +- memcpy(wqe, cur_sge, bytes_copied); +- wqe += bytes_copied; +- cur_sge += bytes_copied; +- quanta_bytes_remaining -= bytes_copied; +- sge_len -= bytes_copied; +- +- if (!quanta_bytes_remaining) { +- quanta_bytes_remaining = 31; +- +- if (first_quanta) { +- first_quanta = false; +- wqe += 16; +- } else { +- *wqe = inline_valid; +- wqe++; +- } +- } +- } +- } +- if (!first_quanta && quanta_bytes_remaining < 31) { +- *(wqe + quanta_bytes_remaining) = inline_valid; +- } +-} +- +-static void sxe2_set_wqe_mw_bind(__le64 *wqe, +- struct sxe2_bind_window *op_info, +- struct sxe2_qp_common *qp) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_bindmw_info *pmsg; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- pmsg = (struct sxe2_bindmw_info *)wqe; +- pmsg->mw_va_base = htole64((uintptr_t)op_info->va); +- pmsg->offset8.field.mw_key = op_info->mw_stag; +- pmsg->offset8.field.mr_key = op_info->mr_stag; +- pmsg->offset8.val = htole64(pmsg->offset8.val); +- pmsg->offset16.field.mw_len = op_info->bind_len; +- pmsg->offset16.val = htole64(pmsg->offset16.val); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("BINDMW_WQE_INFO:\n" \ +- "mw_va_base : %p \n"\ +- "mw_key|mr_key : 0x%x|0x%x \n"\ +- "mw_len : %#llx \n", +- op_info->va, +- op_info->mw_stag, +- op_info->mr_stag, +- op_info->bind_len); +- } +-} +- +-static __le64 *sxe2_qp_get_next_send_wqe(struct sxe2_qp_common *qp, +- __u32 *wqe_idx, __u16 *quanta, __u32 total_size, +- __u64 wr_id, bool push_wqe) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *wqe; +- __u32 nop_wqe_idx; +- __u16 wqe_quanta = *quanta; +- bool push_wqe_pad = false; +- __u16 avail_quanta; +- __u16 i; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- if (push_wqe && (*quanta & 0x1)) { +- *quanta = *quanta + 1; +- push_wqe_pad = true; +- } +- +- avail_quanta = qp->common_attrs->max_hw_sq_chunk - +- (SXE2_RING_CURRENT_HEAD(qp->sq_ring) % +- qp->common_attrs->max_hw_sq_chunk); +- if (*quanta <= avail_quanta) { +- if (*quanta > SXE2_SQ_RING_FREE_QUANTA(qp->sq_ring)) { +- return NULL; +- } +- } else { +- if (*quanta + avail_quanta > +- SXE2_SQ_RING_FREE_QUANTA(qp->sq_ring)) { +- return NULL; +- } +- +- nop_wqe_idx = SXE2_RING_CURRENT_HEAD(qp->sq_ring); +- for (i = 0; i < avail_quanta; i++) { +- (void)sxe2_hw_nop(qp); +- SXE2_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); +- } +- if (qp->push_db && push_wqe) { +- sxe2_qp_push_wqe(qp, qp->sq_base[nop_wqe_idx].elem, +- avail_quanta, nop_wqe_idx); +- } +- } +- +- *wqe_idx = SXE2_RING_CURRENT_HEAD(qp->sq_ring); +- if (!*wqe_idx) { +- qp->swqe_polarity = !qp->swqe_polarity; +- } +- +- SXE2_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta); +- +- wqe = qp->sq_base[*wqe_idx].elem; +- +- qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id; +- qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; +- qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta; +- +- if (push_wqe_pad) { +- __le64 *nop_wqe; +- +- nop_wqe_idx = *wqe_idx + wqe_quanta; +- nop_wqe = qp->sq_base[nop_wqe_idx].elem; +- qp->sq_wrtrk_array[nop_wqe_idx].quanta = SXE2_QP_WQE_MIN_QUANTA; +- set_64bit_val(nop_wqe, 0, 0); +- set_64bit_val(nop_wqe, 8, 0); +- set_64bit_val(nop_wqe, 16, 0); +- sxe2_set_nop_hdr(nop_wqe, 24, qp); +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("POST SEND(qpn = %u): wqe_idx %u wr_id %llu"\ +- " ring_size %u sq_chunk %d pad %d push_mode %d\n", +- qp->qpn, *wqe_idx, qp->sq_wrtrk_array[*wqe_idx].wrid, +- SXE2_RING_SIZE(qp->sq_ring), qp->common_attrs->max_hw_sq_chunk, +- push_wqe_pad, push_wqe); +- } +- +- return wqe; +-} +- +-static __le64 *sxe2_qp_get_next_recv_wqe(struct sxe2_qp_common *qp, +- __u32 *wqe_idx) +-{ +- __le64 *wqe; +- int ret_code; +- +- if (SXE2_RING_FULL_ERR(qp->rq_ring)) { +- return NULL; +- } +- +- SXE2_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code); +- if (ret_code) { +- return NULL; +- } +- +- if (!*wqe_idx) { +- qp->rwqe_polarity = !qp->rwqe_polarity; +- } +- wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem; +- +- return wqe; +-} +- +-int sxe2_hw_send(struct sxe2_qp_common *qp, struct sxe2_wr_info *wr_info, +- bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_post_send *op_info; +- __u32 i, wqe_idx, total_size = 0, byte_off; +- int ret_code; +- __u32 frag_cnt, addl_frag_cnt; +- __u64 frag_info = 0; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- __u16 quanta; +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- op_info = &wr_info->op_info.send; +- if (qp->max_sq_sge_cnt < op_info->num_sges) { +- return EINVAL; +- } +- +- for (i = 0; i < op_info->num_sges; i++) { +- total_size += op_info->sg_list[i].length; +- } +- +- if (wr_info->imm_data_valid) { +- frag_cnt = op_info->num_sges + 1; +- } else { +- frag_cnt = op_info->num_sges; +- } +- +- ret_code = sxe2_fragcnt_to_quanta_cnt(frag_cnt, &quanta); +- if (ret_code) { +- return ret_code; +- } +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; +- +- if (wr_info->imm_data_valid) { +- sxe2_set_immedata(wqe, 0, wr_info->imm_data); +- i = 0; +- } else { +- sxe2_set_sgelist_data(wqe, 0, +- frag_cnt ? op_info->sg_list : NULL, +- qp->swqe_polarity); +- i = 1; +- } +- +- if (total_size == 0) { +- get_64bit_val(wqe, 8, &frag_info); +- frag_info = frag_info & (~SXE2_WQE_FRAG_VALID); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_VALID, !qp->swqe_polarity); +- set_64bit_val(wqe, 8, frag_info); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set frag_info [%#llx] field [%d]\n", +- frag_info, !qp->swqe_polarity); +- } +- } +- +- sxe2_set_qkeyqpn(wqe, 16, op_info->qkey, op_info->dest_qp); +- +- for (byte_off = 32; i < op_info->num_sges;) { +- sxe2_set_sgelist_data(wqe, byte_off, &op_info->sg_list[i], +- qp->swqe_polarity); +- byte_off += 16; +- i++; +- } +- +- if (!(frag_cnt & 0x01) && frag_cnt) { +- sxe2_set_sgelist_data(wqe, byte_off, NULL, qp->swqe_polarity); +- } +- +- sxe2_set_send_hdr(wqe, addl_frag_cnt, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "send"); +- +- return 0; +-} +- +-int sxe2_hw_inline_send(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_post_send *op_info; +- __u32 wqe_idx; +- __u32 i, total_size = 0; +- __u16 quanta; +- +- op_info = &wr_info->op_info.send; +- +- if (unlikely(qp->max_sq_sge_cnt < op_info->num_sges)) { +- return EINVAL; +- } +- +- for (i = 0; i < op_info->num_sges; i++) { +- total_size += op_info->sg_list[i].length; +- } +- +- if (unlikely(total_size > qp->max_inline_data)) { +- return EINVAL; +- } +- +- quanta = sxe2_inline_to_quanta_cnt(total_size); +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- if (wr_info->imm_data_valid) { +- sxe2_set_immedata(wqe, 0, wr_info->imm_data); +- } +- +- sxe2_set_qkeyqpn(wqe, 16, op_info->qkey, op_info->dest_qp); +- +- sxe2_set_inline_data_seg_list((__u8 *)wqe, op_info->sg_list, +- op_info->num_sges, qp->swqe_polarity); +- +- sxe2_set_inlinesnd_hdr(wqe, total_size, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- +-#ifdef SXE2_CFG_DEBUG +- if ((wr_info->op_type == SXE2_OP_TYPE_SEND_INV) || +- (wr_info->op_type == SXE2_OP_TYPE_SEND_SOL_INV)) { +- qp->statistics.send_inv_cnt++; +- } else { +- qp->statistics.send_cnt++; +- } +- +- if (wr_info->signaled) { +- qp->statistics.total_signal_cnt++; +- } +- +- qp->statistics.total_sqe_cnt++; +- qp->statistics.last_send_sqwrid = wr_info->wr_id; +-#endif +- +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "inline_send"); +- +- return 0; +-} +- +-int sxe2_hw_rdma_write(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_rdma_write *op_info; +- __u32 i, wqe_idx; +- __u32 total_size = 0, byte_off; +- int ret_code; +- __u32 frag_cnt, addl_frag_cnt; +- __u16 quanta; +- __u64 frag_info = 0; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- op_info = &wr_info->op_info.rdma_write; +- if (op_info->num_lo_sges > qp->max_sq_sge_cnt) { +- return EINVAL; +- } +- +- for (i = 0; i < op_info->num_lo_sges; i++) { +- total_size += op_info->lo_sg_list[i].length; +- } +- +- if (wr_info->imm_data_valid) { +- frag_cnt = op_info->num_lo_sges + 1; +- } else { +- frag_cnt = op_info->num_lo_sges; +- } +- +- addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; +- ret_code = sxe2_fragcnt_to_quanta_cnt(frag_cnt, &quanta); +- if (ret_code) { +- return ret_code; +- } +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- if (wr_info->imm_data_valid) { +- sxe2_set_immedata(wqe, 0, wr_info->imm_data); +- i = 0; +- } else { +- sxe2_set_sgelist_data(wqe, 0, op_info->lo_sg_list, +- qp->swqe_polarity); +- i = 1; +- } +- +- if (total_size == 0) { +- get_64bit_val(wqe, 8, &frag_info); +- frag_info = frag_info & (~SXE2_WQE_FRAG_VALID); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_VALID, !qp->swqe_polarity); +- set_64bit_val(wqe, 8, frag_info); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set frag_info [%#llx] field [%d]\n", +- frag_info, !qp->swqe_polarity); +- } +- } +- +- sxe2_set_remote_offset(wqe, 16, op_info->rem_addr.addr); +- +- for (byte_off = 32; i < op_info->num_lo_sges;) { +- sxe2_set_sgelist_data(wqe, byte_off, &op_info->lo_sg_list[i], +- qp->swqe_polarity); +- byte_off += 16; +- i++; +- } +- +- if (!(frag_cnt & 0x01) && frag_cnt) { +- sxe2_set_sgelist_data(wqe, byte_off, NULL, qp->swqe_polarity); +- } +- +- sxe2_set_write_hdr(wqe, addl_frag_cnt, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "write"); +- +- return 0; +-} +- +-int sxe2_hw_inline_rdma_write(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_rdma_write *op_info; +- __u32 wqe_idx; +- __u32 i, total_size = 0; +- __u16 quanta; +- +- op_info = &wr_info->op_info.rdma_write; +- +- if (unlikely(qp->max_sq_sge_cnt < op_info->num_lo_sges)) { +- return EINVAL; +- } +- +- for (i = 0; i < op_info->num_lo_sges; i++) { +- total_size += op_info->lo_sg_list[i].length; +- } +- +- if (unlikely(total_size > qp->max_inline_data)) { +- return EINVAL; +- } +- +- quanta = sxe2_inline_to_quanta_cnt(total_size); +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- if (wr_info->imm_data_valid) { +- sxe2_set_immedata(wqe, 0, wr_info->imm_data); +- } +- +- sxe2_set_remote_offset(wqe, 16, op_info->rem_addr.addr); +- +- sxe2_set_inline_data_seg_list((__u8 *)wqe, op_info->lo_sg_list, +- op_info->num_lo_sges, qp->swqe_polarity); +- +- sxe2_set_inlinewrite_hdr(wqe, total_size, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.write_cnt++; +- +- if (wr_info->signaled) { +- qp->statistics.total_signal_cnt++; +- } +- +- qp->statistics.total_sqe_cnt++; +- qp->statistics.last_send_sqwrid = wr_info->wr_id; +-#endif +- +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "inline_write"); +- +- return 0; +-} +- +-int sxe2_hw_rdma_read(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq) +-{ +- struct sxe2_rdma_read *op_info; +- int ret_code; +- __u32 i, byte_off, total_size = 0; +- __u32 addl_frag_cnt; +- __le64 *wqe; +- __u32 wqe_idx; +- __u16 quanta; +- __u64 frag_info = 0; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- bool ord_fence = false; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- +- op_info = &wr_info->op_info.rdma_read; +- if (qp->max_sq_sge_cnt < op_info->num_lo_sges) { +- return EINVAL; +- } +- +- for (i = 0; i < op_info->num_lo_sges; i++) { +- total_size += op_info->lo_sg_list[i].length; +- } +- +- ret_code = sxe2_fragcnt_to_quanta_cnt(op_info->num_lo_sges, &quanta); +- if (ret_code) { +- return ret_code; +- } +- if (qp->rd_fence_rate && (qp->ord_cnt++ == qp->rd_fence_rate)) { +- ord_fence = true; +- qp->ord_cnt = 0; +- } +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- addl_frag_cnt = +- op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0; +- +- sxe2_set_sgelist_data(wqe, 0, op_info->lo_sg_list, qp->swqe_polarity); +- +- if (total_size == 0) { +- get_64bit_val(wqe, 8, &frag_info); +- frag_info = frag_info & (~SXE2_WQE_FRAG_VALID); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_VALID, !qp->swqe_polarity); +- set_64bit_val(wqe, 8, frag_info); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set frag_info [%#llx] field [%d]\n", +- frag_info, !qp->swqe_polarity); +- } +- } +- +- sxe2_set_remote_offset(wqe, 16, op_info->rem_addr.addr); +- +- for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) { +- sxe2_set_sgelist_data(wqe, byte_off, &op_info->lo_sg_list[i], +- qp->swqe_polarity); +- byte_off += 16; +- } +- +- if (!(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) { +- sxe2_set_sgelist_data(wqe, byte_off, NULL, qp->swqe_polarity); +- } +- +- sxe2_set_read_hdr(wqe, addl_frag_cnt, wr_info, qp, ord_fence); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.read_cnt++; +- +- if (wr_info->signaled) { +- qp->statistics.total_signal_cnt++; +- } +- +- qp->statistics.total_sqe_cnt++; +- qp->statistics.last_send_sqwrid = wr_info->wr_id; +-#endif +- +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "rdma_read"); +- +- return 0; +-} +- +-int sxe2_hw_mw_bind(struct sxe2_qp_common *qp, struct sxe2_wr_info *wr_info, +- bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_bind_window *op_info; +- __u32 wqe_idx; +- __u16 quanta = SXE2_QP_WQE_MIN_QUANTA; +- +- op_info = &wr_info->op_info.bind_window; +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, +- 0, wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- sxe2_set_wqe_mw_bind(wqe, op_info, qp); +- +- sxe2_set_bindmw_hdr(wqe, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.bind_mw_cnt++; +- +- if (wr_info->signaled) { +- qp->statistics.total_signal_cnt++; +- } +- +- qp->statistics.total_sqe_cnt++; +- qp->statistics.last_send_sqwrid = wr_info->wr_id; +-#endif +- +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "mw_bind"); +- +- return 0; +-} +- +-int sxe2_hw_local_invalidate(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq) +-{ +- __le64 *wqe; +- struct sxe2_local_invalidate *op_info; +- __u32 wqe_idx; +- __u16 quanta = SXE2_QP_WQE_MIN_QUANTA; +- struct ibv_sge sge = {}; +- +- op_info = &wr_info->op_info.local_inval; +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, +- 0, wr_info->wr_id, wr_info->push_wqe); +- if (!wqe) { +- return ENOMEM; +- } +- +- sge.lkey = op_info->target_stag; +- sxe2_set_sgelist_data(wqe, 0, &sge, 0); +- +- set_64bit_val(wqe, 16, 0); +- +- sxe2_set_invalidate_hdr(wqe, wr_info, qp); +- +- if (wr_info->push_wqe) { +- sxe2_qp_push_wqe(qp, wqe, quanta, wqe_idx); +- } else if (post_sq) { +- sxe2_qp_ring_normal_db(qp); +- } +- +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.local_inv_cnt++; +- +- if (wr_info->signaled) { +- qp->statistics.total_signal_cnt++; +- } +- +- qp->statistics.total_sqe_cnt++; +- qp->statistics.last_send_sqwrid = wr_info->wr_id; +-#endif +- +- sxe2_dump_wqe(qp, wqe, quanta, wqe_idx, "local_invalidate"); +- +- return 0; +-} +- +-static int sxe2_hw_post_receive(struct sxe2_qp_common *qp, +- struct sxe2_rq_info *wr_info) +-{ +- __u32 wqe_idx, i, byte_off, sge_count_valid = 0; +- __u32 addl_frag_cnt; +- __le64 *wqe; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct ibv_sge *p_sge = NULL; +- +- uqp = container_of(qp, struct sxe2_uqp, qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (qp->max_rq_sge_cnt < wr_info->num_sges) { +- return EINVAL; +- } +- if (!qp->rq_size) { +- return EINVAL; +- } +- wqe = sxe2_qp_get_next_recv_wqe(qp, &wqe_idx); +- if (!wqe) { +- return ENOMEM; +- } +- for (i = 0; i < wr_info->num_sges; i++) { +- if (wr_info->sg_list[i].length != 0) { +- sge_count_valid++; +- if (p_sge == NULL) { +- p_sge = &wr_info->sg_list[i]; +- } +- } +- } +- qp->rq_wrid_array[wqe_idx] = wr_info->wr_id; +- +- addl_frag_cnt = sge_count_valid > 1 ? (sge_count_valid - 1) : 0; +- if (sge_count_valid) { +- sxe2_set_sgelist_data(wqe, 0, p_sge, qp->rwqe_polarity); +- } else { +- sxe2_set_sgelist_data(wqe, 0, NULL, qp->rwqe_polarity); +- } +- if (sge_count_valid) { +- for (i = 1, byte_off = SXE2_RQ_WQE_HEAD_OFFSET; i < wr_info->num_sges; i++) { +- if (wr_info->sg_list[i].length == 0) { +- continue; +- } +- sxe2_set_sgelist_data(wqe, byte_off, &wr_info->sg_list[i], +- qp->rwqe_polarity); +- byte_off += SXE2_RQ_WQE_FRAG_OFFSET; +- } +- if (!(sge_count_valid & SXE2_WQE_QUANTA_ODD_NUMBER) && sge_count_valid) { +- sxe2_set_sgelist_data(wqe, byte_off, NULL, qp->rwqe_polarity); +- } +- } +- set_64bit_val(wqe, 16, 0); +- +- sxe2_set_rcvq_hdr(wqe, addl_frag_cnt, qp); +- +- udma_to_device_barrier(); +- +- qp->doorbell_note[SXE2_QP_RQ_PI] = +- htole32(SXE2_RING_CURRENT_HEAD(qp->rq_ring) * qp->rq_wqe_size_multiplier); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("POST RCV(qpn = %u): wqe_idx %u wr_id %llu"\ +- " rq_pi %u ring_size %u\n", qp->qpn, wqe_idx, +- qp->rq_wrid_array[wqe_idx], +- SXE2_RING_CURRENT_HEAD(qp->rq_ring) * qp->rq_wqe_size_multiplier, +- SXE2_RING_SIZE(qp->rq_ring)); +- for (i = 0; i < wr_info->num_sges; i++) { +- SXE2_VERBS_LOG_DEBUG_BDF("sgelist[%d] addr %" PRIu64 " len [%u] lkey [%u]\n", +- i, wr_info->sg_list[i].addr, wr_info->sg_list[i].length, +- wr_info->sg_list[i].lkey); +- } +- } +- +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.total_rqe_cnt++; +- qp->statistics.last_send_rqwrid = wr_info->wr_id; +-#endif +- +- return 0; +-} +-static void sxe2_wrinfo_init_inv(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, +- struct sxe2_wr_info *wr_info) +-{ +- struct sxe2_uah *ah; +- +- if (ib_wr->opcode == IBV_WR_SEND || +- ib_wr->opcode == IBV_WR_SEND_WITH_IMM) { +- wr_info->op_type = SXE2_OP_TYPE_SEND; +- if (ib_wr->send_flags & IBV_SEND_SOLICITED) { +- wr_info->op_type = SXE2_OP_TYPE_SEND_SOL; +- } +- } else { +- wr_info->op_type = SXE2_OP_TYPE_SEND_INV; +- if (ib_wr->send_flags & IBV_SEND_SOLICITED) { +- wr_info->op_type = SXE2_OP_TYPE_SEND_SOL_INV; +- } +- wr_info->rkey_to_inv = ib_wr->invalidate_rkey; +- } +- wr_info->op_info.send.num_sges = (__u32)ib_wr->num_sge; +- wr_info->op_info.send.sg_list = (struct ibv_sge *)ib_wr->sg_list; +- if (ib_qp->qp_type == IBV_QPT_UD) { +- ah = container_of(ib_wr->wr.ud.ah, struct sxe2_uah, ibv_ah); +- wr_info->op_info.send.ah_id = ah->ah_id; +- wr_info->op_info.send.qkey = ib_wr->wr.ud.remote_qkey; +- wr_info->op_info.send.dest_qp = ib_wr->wr.ud.remote_qpn; +- } +- wr_info->funid = SXE2_RDMA_SEND; +- if (ib_wr->send_flags & IBV_SEND_INLINE) { +- wr_info->funid = SXE2_RDMA_SEND_INLINE; +- } +-} +-static int sxe2_wrinfo_init(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, +- struct sxe2_wr_info *wr_info) +-{ +- struct sxe2_common_attrs *uk_attrs; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- int err = 0; +- +- uqp = container_of(ib_qp, struct sxe2_uqp, verbs_qp.qp); +- sctx = container_of(ib_qp->context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- uk_attrs = &sctx->uk_attrs; +- +- memset(wr_info, 0, sizeof(*wr_info)); +- +- wr_info->wr_id = (__u64)(ib_wr->wr_id); +- wr_info->push_wqe = uqp->qp.push_db ? true : false; +- +- if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || uqp->sq_sig_all) { +- wr_info->signaled = true; +- } +- +- if (ib_wr->send_flags & IBV_SEND_FENCE) { +- wr_info->read_fence = true; +- } +- +- if (uqp->send_cq->report_rtt) { +- wr_info->report_rtt = true; +- } +- +- switch (ib_wr->opcode) { +- case IBV_WR_SEND_WITH_IMM: +- if (!(uqp->qp.qp_caps & SXE2_SEND_WITH_IMM)) { +- return EINVAL; +- } +- wr_info->imm_data_valid = true; +- wr_info->imm_data = ntohl(ib_wr->imm_data); +- SWITCH_FALLTHROUGH; +- case IBV_WR_SEND: +- case IBV_WR_SEND_WITH_INV: +- sxe2_wrinfo_init_inv(ib_qp, ib_wr, wr_info); +- break; +- case IBV_WR_RDMA_WRITE_WITH_IMM: +- if (!(uqp->qp.qp_caps & SXE2_WRITE_WITH_IMM)) { +- return EINVAL; +- } +- wr_info->imm_data_valid = true; +- wr_info->imm_data = ntohl(ib_wr->imm_data); +- SWITCH_FALLTHROUGH; +- case IBV_WR_RDMA_WRITE: +- wr_info->op_type = SXE2_OP_TYPE_RDMA_WRITE; +- if (ib_wr->send_flags & IBV_SEND_SOLICITED) { +- wr_info->op_type = SXE2_OP_TYPE_RDMA_WRITE_SOL; +- } +- +- wr_info->op_info.rdma_write.num_lo_sges = (__u32)ib_wr->num_sge; +- wr_info->op_info.rdma_write.lo_sg_list = ib_wr->sg_list; +- wr_info->op_info.rdma_write.rem_addr.addr = +- ib_wr->wr.rdma.remote_addr; +- wr_info->op_info.rdma_write.rem_addr.lkey = ib_wr->wr.rdma.rkey; +- wr_info->funid = SXE2_RDMA_WRITE; +- if (ib_wr->send_flags & IBV_SEND_INLINE) { +- wr_info->funid = SXE2_RDMA_WRITE_INLINE; +- } +- break; +- case IBV_WR_RDMA_READ: +- if (ib_wr->num_sge > (int)uk_attrs->max_hw_read_sges) { +- return EINVAL; +- } +- wr_info->op_type = SXE2_OP_TYPE_RDMA_READ; +- wr_info->op_info.rdma_read.rem_addr.addr = +- ib_wr->wr.rdma.remote_addr; +- wr_info->op_info.rdma_read.rem_addr.lkey = ib_wr->wr.rdma.rkey; +- +- wr_info->op_info.rdma_read.lo_sg_list = ib_wr->sg_list; +- wr_info->op_info.rdma_read.num_lo_sges = (__u32)ib_wr->num_sge; +- wr_info->funid = SXE2_RDMA_READ; +- break; +- case IBV_WR_LOCAL_INV: +- wr_info->op_type = SXE2_OP_TYPE_LOCAL_INV; +- wr_info->op_info.local_inval.target_stag = +- ib_wr->invalidate_rkey; +- wr_info->funid = SXE2_RDMA_LOCAL_INV; +- wr_info->post_wqe = true; +- break; +- default: +- err = EINVAL; +- break; +- } +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr_info qpn [0x%x] opcode [%d] "\ +- "wr_id %llu push_mode %d post_wqe %d signald %d\n", +- ib_qp->qp_num, ib_wr->opcode, wr_info->wr_id, +- wr_info->push_wqe, wr_info->post_wqe, wr_info->signaled); +- } +- return err; +-} +- +-int sxe2_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, +- struct ibv_send_wr **bad_wr) +-{ +- struct sxe2_uqp *uqp = NULL; +- bool reflush = false; +- struct sxe2_wr_info wr_info; +- struct sxe2_rdma_ucontext *sctx = NULL; +- int err; +- +- uqp = container_of(ib_qp, struct sxe2_uqp, verbs_qp.qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("upost send enter, qpn [%u]\n", +- uqp->qp.qpn); +- } +- +- err = pthread_spin_lock(&uqp->lock); +- if (err) { +- return err; +- } +- +- if (!SXE2_RING_MORE_WORK(uqp->qp.sq_ring) && +- ib_qp->state == IBV_QPS_ERR) { +- reflush = true; +- } +- +- while (ib_wr) { +- err = sxe2_wrinfo_init(ib_qp, ib_wr, &wr_info); +- if (err) { +- *bad_wr = ib_wr; +- break; +- } +- +-#if 0 +- wr_info.post_wqe = true; +-#endif +- err = g_rdma_op[wr_info.funid](&uqp->qp, &wr_info, +- wr_info.post_wqe); +- if (err) { +- *bad_wr = ib_wr; +- break; +- } +- ib_wr = ib_wr->next; +- } +- +- if (!uqp->qp.push_db) +- sxe2_qp_ring_normal_db(&uqp->qp); +- if (reflush) +- sxe2_qp_flush_wqe(ib_qp, true, false); +- +- pthread_spin_unlock(&uqp->lock); +- +- return err; +-} +- +-int sxe2_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, +- struct ibv_recv_wr **bad_wr) +-{ +- struct sxe2_rq_info rq_info = {}; +- struct sxe2_uqp *uqp = NULL; +- bool reflush = false; +- struct sxe2_rdma_ucontext *sctx = NULL; +- int err; +- +- uqp = container_of(ib_qp, struct sxe2_uqp, verbs_qp.qp); +- sctx = container_of(uqp->verbs_qp.qp.context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("upost receive enter, qpn [%u]\n", +- uqp->qp.qpn); +- } +- +- err = pthread_spin_lock(&uqp->lock); +- if (err) { +- return err; +- } +- +- if (!SXE2_RING_MORE_WORK(uqp->qp.rq_ring) && +- ib_qp->state == IBV_QPS_ERR) { +- reflush = true; +- } +- +- while (ib_wr) { +- if (ib_wr->num_sge > (int)uqp->qp.max_rq_sge_cnt) { +- *bad_wr = ib_wr; +- err = EINVAL; +- goto error; +- } +- rq_info.num_sges = (__u32)ib_wr->num_sge; +- rq_info.wr_id = ib_wr->wr_id; +- rq_info.sg_list = ib_wr->sg_list; +- err = sxe2_hw_post_receive(&uqp->qp, &rq_info); +- if (err) { +- *bad_wr = ib_wr; +- goto error; +- } +- +- if (reflush) { +- sxe2_qp_flush_wqe(ib_qp, false, true); +- } +- +- ib_wr = ib_wr->next; +- } +-error: +- pthread_spin_unlock(&uqp->lock); +- +- return err; +-} +- +-static int sxe2_get_next_cqe(struct sxe2_cq_uk *cq, struct sxe2_cqe_info *cqe_info) +-{ +- int i; +- __le64 *cqe; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- vendor_cq = container_of(cq, struct sxe2_ucq, cq); +- sctx = container_of(vendor_cq->verbs_cq.cq.context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- cqe = SXE2_GET_CURRENT_CQ_ELEM(cq); +- +- cqe_info->info.buf[SXE2_CQE_SIZE - 1] = le64toh(cqe[SXE2_CQE_SIZE - 1]); +- if (cqe_info->info.field.cqe_valid != cq->polarity) { +- return -ENOENT; +- } +- +- udma_from_device_barrier(); +- +- for (i = 0; i < SXE2_CQE_SIZE - 1; i++) { +- cqe_info->info.buf[i] = le64toh(cqe[i]); +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("CQ1 (%d) CI(0x%x) cqeinfo:\n"\ +- "%#llx(%#llx) %#llx(%#llx) %#llx(%#llx) %#llx(%#llx)\n"\ +- "%#llx(%#llx) %#llx(%#llx) %#llx(%#llx) %#llx(%#llx)\n", +- cq->cq_id, +- cq->cq_ring.head, +- cqe[0], cqe_info->info.buf[0], +- cqe[1], cqe_info->info.buf[1], +- cqe[2], cqe_info->info.buf[2], +- cqe[3], cqe_info->info.buf[3], +- cqe[4], cqe_info->info.buf[4], +- cqe[5], cqe_info->info.buf[5], +- cqe[6], cqe_info->info.buf[6], +- cqe[7], cqe_info->info.buf[7]); +- } +- +- return SXE2_CQ_OK; +-} +- +-static int sxe2_hw_flush_one_sq_wqe(struct sxe2_cq_uk *cq, +- struct sxe2_qp_common *qp, struct sxe2_cqe_info *cqe_info) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- __le64 *sw_wqe; +- __u64 wqe_hdr; +- __u32 tail; +- +- vendor_cq = container_of(cq, struct sxe2_ucq, cq); +- sctx = container_of(vendor_cq->verbs_cq.cq.context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (!SXE2_RING_MORE_WORK(qp->sq_ring) || !SXE2_RING_MORE_WORK_PAD(qp->sq_ring)) { +- return -ENOENT; +- } +- do { +- tail = qp->sq_ring.tail; +- sw_wqe = qp->sq_base[tail].elem; +- get_64bit_val(sw_wqe, 24, &wqe_hdr); +- cqe_info->info.field.op = (__u8)FIELD_GET(SXE2_WQE_OPCODE, wqe_hdr); +- SXE2_RING_SET_TAIL(qp->sq_ring, +- tail + qp->sq_wrtrk_array[tail].quanta); +- if (cqe_info->info.field.op != SXE2_OP_TYPE_NOP) { +- cqe_info->wr_id = qp->sq_wrtrk_array[tail].wrid; +- cqe_info->bytes = qp->sq_wrtrk_array[tail].wr_len; +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("flush qp %u sq_pi %u sq_ci %u wqe_idx %u wr_id %llu.\n", +- qp->qpn, SXE2_RING_CURRENT_HEAD(qp->sq_ring), +- SXE2_RING_CURRENT_TAIL(qp->sq_ring), tail, +- cqe_info->wr_id); +- } +-#ifdef SXE2_CFG_DEBUG +- qp->statistics.flushed_sq_cnt++; +- qp->statistics.last_rcvd_sqwrid = cqe_info->wr_id; +-#endif +- break; +- } +- } while (1); +- +- return SXE2_CQ_OK; +-} +- +-static void sxe2_move_srq_ring_tail(struct sxe2_srq_verbs *srq) +-{ +- __u32 tail; +- while (SXE2_RING_MORE_WORK(srq->srq_ring)) { +- tail = SXE2_RING_CURRENT_TAIL(srq->srq_ring); +- if (srq->srqe_array[tail] == SXE2_SRQE_BUSY) { +- break; +- } +- SXE2_RING_MOVE_TAIL(srq->srq_ring); +- } +-} +-static void sxe2_hw_deal_srq_cqe(struct sxe2_rdma_ucontext *sctx, struct sxe2_cq_uk *cq, +- struct sxe2_cqe_info *cqe_info, struct sxe2_qp_common *qp, __u32 qpn) +-{ +- struct sxe2_srq_verbs *srq; +- __u32 wqe_idx; +- __u32 array_idx; +- +- srq = qp->srq; +- wqe_idx = (__u32)cqe_info->info.field.wq_desc_idx; +- array_idx = wqe_idx / srq->wqe_size_multiplier; +- if (srq->srqe_array[array_idx] == SXE2_SRQE_BUSY) { +- srq->srqe_array[array_idx] = SXE2_SRQE_FREE; +- } else { +- SXE2_VERBS_LOG_ERROR_BDF("cq %d received invalid cqe"\ +- "(qpn %u srqn %u wqe_idx %u mul_size %d flag %d).\n", +- cq->cq_id, qpn, srq->srq_id, wqe_idx, +- srq->wqe_size_multiplier, srq->srqe_array[array_idx]); +- } +- cqe_info->wr_id = srq->srq_wrid_array[array_idx]; +- sxe2_move_srq_ring_tail(srq); +-} +-static int sxe2_hw_deal_rq_cqe(struct sxe2_rdma_ucontext *sctx, struct sxe2_cq_uk *cq, +- struct sxe2_cqe_info *cqe_info, struct sxe2_qp_common *qp, __u32 qpn) +-{ +- __u32 wqe_idx; +- __u32 array_idx; +- +- wqe_idx = (__u32)cqe_info->info.field.wq_desc_idx; +- array_idx = wqe_idx / qp->rq_wqe_size_multiplier; +- if (cqe_info->info.field.major_err == SXE2_WR_FLUSH_ERR) { +- if (!SXE2_RING_MORE_WORK(qp->rq_ring)) { +- return -ENOENT; +- } +- SXE2_VERBS_LOG_DEBUG_BDF("flush qp %u state rq_pi %u rq_ci %u .\n", +- qpn, SXE2_RING_CURRENT_HEAD(qp->rq_ring), +- SXE2_RING_CURRENT_TAIL(qp->rq_ring)); +- array_idx = qp->rq_ring.tail; +- } +- cqe_info->wr_id = qp->rq_wrid_array[array_idx]; +- SXE2_RING_SET_TAIL(qp->rq_ring, array_idx + 1); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("update qp %u rq_ci %u wqeidx %u arridx %u wrid %llu.\n", +- qpn, SXE2_RING_CURRENT_TAIL(qp->rq_ring), wqe_idx, +- array_idx, cqe_info->wr_id); +- } +- return 0; +-} +-static int sxe2_hw_deal_sq_cqe(struct sxe2_rdma_ucontext *sctx, struct sxe2_cq_uk *cq, +- struct sxe2_cqe_info *cqe_info, struct sxe2_qp_common *qp, __u32 qpn) +-{ +- __u32 wqe_idx; +- +- wqe_idx = (__u32)cqe_info->info.field.wq_desc_idx; +- +- if (cqe_info->info.field.push_dropped) { +- qp->llwqe_mode = false; +- qp->push_dropped = true; +- } +- if (cqe_info->info.field.major_err != SXE2_WR_FLUSH_ERR) { +- cqe_info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; +- cqe_info->bytes = qp->sq_wrtrk_array[wqe_idx].wr_len; +- SXE2_RING_SET_TAIL(qp->sq_ring, wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("update qp %u sq_ci %u wqe_idx %u wr_id %llu.\n", +- qpn, SXE2_RING_CURRENT_TAIL(qp->sq_ring), wqe_idx, +- cqe_info->wr_id); +- } +- } else { +- return sxe2_hw_flush_one_sq_wqe(cq, qp, cqe_info); +- } +- return 0; +-} +- +-static int sxe2_hw_cq_poll(struct sxe2_cq_uk *cq, +- struct sxe2_cqe_info *cqe_info) +-{ +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_ring *cur_ring = NULL; +- __u32 wqe_idx; +- int ret_code = SXE2_CQ_OK; +- __le64 *cqe; +- __u64 qword = 0; +- __u32 qpn = 0; +- bool move_cq_head = true; +- +- vendor_cq = container_of(cq, struct sxe2_ucq, cq); +- sctx = container_of(vendor_cq->verbs_cq.cq.context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- ret_code = sxe2_get_next_cqe(cq, cqe_info); +- if (ret_code != SXE2_CQ_OK) { +- return ret_code; +- } +- +- qp = (struct sxe2_qp_common *)(unsigned long)cqe_info->info.field.qpc; +- if (!qp || qp->destroy_pending) { +- ret_code = -EFAULT; +- goto exit; +- } +- +- qpn = qp->qpn; +- wqe_idx = (__u32)cqe_info->info.field.wq_desc_idx; +- cqe_info->bytes = cqe_info->info.field.payload_len; +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("cqe process cqn [%u] ci [%u] qp [%u] wqeidx %u.\n", +- cq->cq_id, SXE2_RING_CURRENT_HEAD(cq->cq_ring), qpn, wqe_idx); +- } +- if (cqe_info->info.field.is_srq) { +- sxe2_hw_deal_srq_cqe(sctx, cq, cqe_info, qp, qpn); +- } else if (cqe_info->info.field.qp_type == SXE2_CQE_QTYPE_RQ) { +- cur_ring = &qp->rq_ring; +- ret_code = sxe2_hw_deal_rq_cqe(sctx, cq, cqe_info, qp, qpn); +- if (ret_code) { +- goto exit; +- } +- } else { +- cur_ring = &qp->sq_ring; +- ret_code = sxe2_hw_deal_sq_cqe(sctx, cq, cqe_info, qp, qpn); +- if (ret_code) { +- goto exit; +- } +- } +- +-exit: +- if (!ret_code && cqe_info->info.field.major_err == SXE2_WR_FLUSH_ERR) { +- if (cur_ring && SXE2_RING_MORE_WORK(*cur_ring)) { +- move_cq_head = false; +- } +- } +- +- if (move_cq_head) { +- SXE2_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); +- if (!SXE2_RING_CURRENT_HEAD(cq->cq_ring)) { +- cq->polarity ^= 1; +- } +- +- SXE2_RING_MOVE_TAIL(cq->cq_ring); +- cq->doorbell_note[SXE2_CQ_SET_CI] = +- htole32(SXE2_RING_CURRENT_HEAD(cq->cq_ring)); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("update qp %u cq %d ci %u.\n", +- qpn, cq->cq_id, SXE2_RING_CURRENT_HEAD(cq->cq_ring)); +- } +- } else { +- cqe = SXE2_GET_CURRENT_CQ_ELEM(cq); +- get_64bit_val(cqe, 24, &qword); +- qword &= ~SXE2_CQE_WQEIDX; +- qword |= FIELD_PREP(SXE2_CQE_WQEIDX, cur_ring->tail); +- set_64bit_val(cqe, 24, qword); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("cqn [%u] idx [%u] flush wqeidx %d to %d qpn %u.\n", +- cq->cq_id, SXE2_RING_CURRENT_HEAD(cq->cq_ring), +- wqe_idx, cur_ring->tail, qpn); +- } +- } +- +- return ret_code; +-} +- +-static enum ibv_wc_status sxe2_flush_err_to_ib_wc_status(enum sxe2_major_opcode opcode) +-{ +- switch (opcode) { +- case SXE2_LOCAL_PROTECTION_ERR: +- return IBV_WC_LOC_PROT_ERR; +- case SXE2_BAD_RESPONSE_ERR: +- return IBV_WC_BAD_RESP_ERR; +- case SXE2_REMOTE_ACCESS_ERR: +- return IBV_WC_REM_ACCESS_ERR; +- case SXE2_LOCAL_QP_OP_ERR: +- return IBV_WC_LOC_QP_OP_ERR; +- case SXE2_REMOTE_OPERATION_ERR: +- return IBV_WC_REM_OP_ERR; +- case SXE2_LOCAL_LEN_ERR: +- return IBV_WC_LOC_LEN_ERR; +- case SXE2_LOCAL_ACCESS_ERR: +- return IBV_WC_LOC_ACCESS_ERR; +- case SXE2_WR_FLUSH_ERR: +- return IBV_WC_WR_FLUSH_ERR; +- case SXE2_TRANS_RETRY_CNT_EXCEED_ERR: +- return IBV_WC_RETRY_EXC_ERR; +- case SXE2_MW_BIND_ERR: +- return IBV_WC_MW_BIND_ERR; +- case SXE2_REMOTE_INVALID_REQUEST_ERR: +- return IBV_WC_REM_INV_REQ_ERR; +- case SXE2_RNR_RETRY_CNT_EXCEED_ERR: +- return IBV_WC_RNR_RETRY_EXC_ERR; +- default: +- return IBV_WC_GENERAL_ERR; +- } +-} +- +-static enum ibv_wc_opcode sxe2_get_ib_wc_op_sq(struct sxe2_cqe_info *cur_cqe) +-{ +- enum ibv_wc_opcode opcode; +- +- switch (cur_cqe->info.field.op) { +- case SXE2_OP_TYPE_RDMA_WRITE: +- case SXE2_OP_TYPE_RDMA_WRITE_SOL: +- opcode = IBV_WC_RDMA_WRITE; +- break; +- case SXE2_OP_TYPE_RDMA_READ: +- opcode = IBV_WC_RDMA_READ; +- break; +- case SXE2_OP_TYPE_SEND_SOL: +- case SXE2_OP_TYPE_SEND_SOL_INV: +- case SXE2_OP_TYPE_SEND_INV: +- case SXE2_OP_TYPE_SEND: +- opcode = IBV_WC_SEND; +- break; +- case SXE2_OP_TYPE_BIND_MW: +- opcode = IBV_WC_BIND_MW; +- break; +- case SXE2_OP_TYPE_LOCAL_INV: +- opcode = IBV_WC_LOCAL_INV; +- break; +- default: +- opcode = IBV_WC_DRIVER1; +- break; +- } +- return opcode; +-} +- +-static enum ibv_wc_opcode sxe2_get_ib_wc_op_rq(struct sxe2_cqe_info *cur_cqe) +-{ +- enum ibv_wc_opcode opcode; +- +- if (cur_cqe->info.field.imm_data_flag) { +- opcode = IBV_WC_RECV_RDMA_WITH_IMM; +- } else { +- opcode = IBV_WC_RECV; +- } +- +- return opcode; +-} +- +-static void sxe2_cq_cqe_ext(struct sxe2_cqe_info *cur_cqe) +-{ +- struct sxe2_ucq *ucq = container_of(cur_cqe, struct sxe2_ucq, cur_cqe); +- struct ibv_cq_ex *ibvcq_ex = &ucq->verbs_cq.cq_ex; +- ibvcq_ex->wr_id = cur_cqe->wr_id; +- if (cur_cqe->info.field.error) { +- ibvcq_ex->status = sxe2_flush_err_to_ib_wc_status(cur_cqe->info.field.major_err); +- } else { +- ibvcq_ex->status = IBV_WC_SUCCESS; +- } +-} +- +-static void sxe2_cq_pollinfo_to_wc(struct ibv_wc *entry, struct sxe2_cqe_info *cur_cqe) +-{ +- struct sxe2_qp_common *qp; +- struct ibv_qp *ib_qp; +- +- entry->wc_flags = 0; +- entry->wr_id = cur_cqe->wr_id; +- entry->qp_num = cur_cqe->info.field.qp_id; +- qp = (struct sxe2_qp_common *)(unsigned long)cur_cqe->info.field.qpc; +- ib_qp = qp->back_qp; +- +- if (cur_cqe->info.field.error) { +- entry->status = sxe2_flush_err_to_ib_wc_status(cur_cqe->info.field.major_err); +- entry->vendor_err = (uint32_t)(cur_cqe->info.field.major_err << 16 | +- cur_cqe->info.field.minor_err); +- } else { +- entry->status = IBV_WC_SUCCESS; +- } +- +- if (cur_cqe->info.field.imm_data_flag) { +- entry->imm_data = htonl(cur_cqe->info.field.imme_data); +- entry->wc_flags |= IBV_WC_WITH_IMM; +- } +- +- if (cur_cqe->info.field.qp_type == SXE2_CQE_QTYPE_SQ) { +- entry->opcode = sxe2_get_ib_wc_op_sq(cur_cqe); +- } else { +- entry->opcode = sxe2_get_ib_wc_op_rq(cur_cqe); +- if (ib_qp->qp_type != IBV_QPT_UD && +- cur_cqe->info.field.stag_or_lrkey) { +- entry->invalidated_rkey = cur_cqe->info.field.l_r_key; +- entry->wc_flags |= IBV_WC_WITH_INV; +- } +- } +- +- if (ib_qp->qp_type == IBV_QPT_UD) { +- entry->src_qp = cur_cqe->info.field.ud_src_qpn; +- entry->wc_flags |= IBV_WC_GRH; +- } else { +- entry->src_qp = cur_cqe->info.field.qp_id; +- } +- entry->byte_len = cur_cqe->bytes; +-} +- +-static int sxe2_poll_one(struct sxe2_cq_uk *ukcq, struct sxe2_cqe_info *cur_cqe, +- struct ibv_wc *entry) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- vendor_cq = container_of(ukcq, struct sxe2_ucq, cq); +- sctx = container_of(vendor_cq->verbs_cq.cq.context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- int ret = sxe2_hw_cq_poll(ukcq, cur_cqe); +- if (ret) { +- if (ret == -EFAULT) { +- SXE2_VERBS_LOG_WARN_BDF("CQ %d skip cqe(CI 0x%x) for qp destroy.\n", +- ukcq->cq_id, ukcq->cq_ring.head); +- } +- return ret; +- } +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("CQ (%d) CI(0x%x) wrid(%llu) cqeinfo:\n"\ +- "payload_len : %ul\n"\ +- "packet_seq : %d\n"\ +- "qpc : %#llx\n"\ +- "l_r_key : %ul\n"\ +- "qp_id : %d\n"\ +- "minor_err : %d\n"\ +- "major_err : %d\n"\ +- "wq_desc_idx : %d\n"\ +- "extended_cqe : %d\n"\ +- "push_dropped : %d\n"\ +- "ipv4 : %d\n"\ +- "stag_or_lrkey : %d\n"\ +- "solicited_evt : %d\n"\ +- "error : %d\n"\ +- "op : %d\n"\ +- "qp_type : %d\n"\ +- "imme_data : %ul\n"\ +- "srqn : %d\n"\ +- "is_srq : %d\n"\ +- "cqe_timestamp : %#llx\n"\ +- "ud_smac : %#llx\n"\ +- "ud_vlan_tag : %d\n"\ +- "ud_src_qpn : %d\n"\ +- "vsi_index : %d\n"\ +- "vlan_tag_flag : %d\n"\ +- "ud_smac_valid : %d\n"\ +- "imm_data_flag : %d\n"\ +- "cqe_valid : %d\n", +- ukcq->cq_id, +- ukcq->cq_ring.head, +- cur_cqe->wr_id, +- cur_cqe->info.field.payload_len, +- cur_cqe->info.field.packet_seq, +- cur_cqe->info.field.qpc, +- cur_cqe->info.field.l_r_key, +- cur_cqe->info.field.qp_id, +- cur_cqe->info.field.minor_err, +- cur_cqe->info.field.major_err, +- cur_cqe->info.field.wq_desc_idx, +- cur_cqe->info.field.extended_cqe, +- cur_cqe->info.field.push_dropped, +- cur_cqe->info.field.ipv4, +- cur_cqe->info.field.stag_or_lrkey, +- cur_cqe->info.field.solicited_evt, +- cur_cqe->info.field.error, +- cur_cqe->info.field.op, +- cur_cqe->info.field.qp_type, +- cur_cqe->info.field.imme_data, +- cur_cqe->info.field.srqn, +- cur_cqe->info.field.is_srq, +- cur_cqe->info.field.cqe_timestamp, +- (__u64)cur_cqe->info.field.ud_smac, +- cur_cqe->info.field.ud_vlan_tag, +- cur_cqe->info.field.ud_src_qpn, +- cur_cqe->info.field.vsi_index, +- cur_cqe->info.field.vlan_tag_flag, +- cur_cqe->info.field.ud_smac_valid, +- cur_cqe->info.field.imm_data_flag, +- cur_cqe->info.field.cqe_valid); +- } +- +- if (!entry) { +- sxe2_cq_cqe_ext(cur_cqe); +- } else { +- sxe2_cq_pollinfo_to_wc(entry, cur_cqe); +- } +- +- return 0; +-} +- +-int sxe2_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) +-{ +- struct sxe2_ucq *ucq = NULL; +- struct sxe2_cqe_info *cur_cqe; +- int ret; +- int npolled = 0; +- +- ucq = container_of(cq, struct sxe2_ucq, verbs_cq.cq); +- cur_cqe = &ucq->cur_cqe; +- +- ret = pthread_spin_lock(&ucq->lock); +- if (ret) { +- return -ret; +- } +- while (npolled < num_entries) { +- ret = sxe2_poll_one(&ucq->cq, cur_cqe, wc + npolled); +- if (ret == SXE2_CQ_OK) { +- ++npolled; +- continue; +- } +- if (ret == -ENOENT) { +- break; +- } +- } +- +- pthread_spin_unlock(&ucq->lock); +- return npolled; +-} +- +-static void sxe2_cq_arm_notify(struct sxe2_cq_uk *arm_cq, +- enum sxe2_arm_type arm_type) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- uint64_t doorbell; +- uint32_t sn; +- uint32_t ci; +- uint32_t cmd; +- +- vendor_cq = container_of(arm_cq, struct sxe2_ucq, cq); +- sctx = container_of(vendor_cq->verbs_cq.cq.context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- sn = arm_cq->arm_sn & 3; +- ci = SXE2_RING_CURRENT_HEAD(arm_cq->cq_ring); +- +- if (arm_type == SXE2_CQ_ARM_SOLICITED) { +- cmd = (uint32_t)SXE2_CQ_DB_REQ_SOLICITED; +- } else { +- cmd = SXE2_CQ_DB_REQ_NOSOLICITED; +- } +- +- doorbell = 0; +- doorbell = sn << 29 | cmd | ci; +- doorbell <<= 32; +- doorbell |= arm_cq->cq_id; +- +- arm_cq->doorbell_note[SXE2_CQ_ARM_DB] = htole32(sn << 29 | cmd | ci); +- +- udma_to_device_barrier(); +- +- set_64bit_val(arm_cq->cqe_alloc_db, 0, doorbell); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("CQ (%u -> %p) ARM NOTIFY DBNOTE(0x%x)"\ +- " DB(0x%lx) CI(%u) SN(%u) CMD(%u)\n", arm_cq->cq_id, +- arm_cq->cqe_alloc_db, htole32(sn << 29 | cmd | ci), +- htole64(doorbell), ci, sn, cmd); +- } +-} +- +-int sxe2_uarm_cq(struct ibv_cq *cq, int solicited) +-{ +- struct sxe2_ucq *vendor_cq = NULL; +- enum sxe2_arm_type arm_type; +- int ret; +- +- arm_type = solicited ? SXE2_CQ_ARM_SOLICITED : SXE2_CQ_ARM_NEXT; +- vendor_cq = container_of(cq, struct sxe2_ucq, verbs_cq.cq); +- ret = pthread_spin_lock(&vendor_cq->lock); +- if (ret) { +- return ret; +- } +- +- sxe2_cq_arm_notify(&vendor_cq->cq, arm_type); +- +- pthread_spin_unlock(&vendor_cq->lock); +- +- return 0; +-} +- +-void sxe2_ucq_event(struct ibv_cq *cq) +-{ +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_cq_uk *arm_cq; +- +- vendor_cq = container_of(cq, struct sxe2_ucq, verbs_cq.cq); +- arm_cq = &vendor_cq->cq; +- if (pthread_spin_lock(&vendor_cq->lock)) { +- return; +- } +- +- arm_cq->arm_sn += 1; +- +- pthread_spin_unlock(&vendor_cq->lock); +-} +- +-static void sxe2_clean_base_cq(struct sxe2_qp_common *qp, struct sxe2_cq_uk *cq, int cq_type) +-{ +- __le64 *cqe; +- __u64 qword, qpc; +- __u32 cq_head; +- __u8 polarity, cq_polarity; +- +- cq_head = cq->cq_ring.head; +- cq_polarity = cq->polarity; +- do { +- cqe = ((struct sxe2_cqe *)(cq->cq_base))[cq_head].buf; +- get_64bit_val(cqe, 56, &qword); +- polarity = (__u8)FIELD_GET(SXE2_CQE_VALID, qword); +- if (polarity != cq_polarity) { +- break; +- } +- +- get_64bit_val(cqe, 8, &qpc); +- if ((void *)(uintptr_t)qpc == (void *)qp) { +- set_64bit_val(cqe, 8, 0); +-#ifdef SXE2_CFG_DEBUG +- if (cq_type == SQ_CQ) { +- qp->statistics.cleaned_sq_cnt++; +- } else { +- qp->statistics.cleaned_rq_cnt++; +- } +-#endif +- } +- +- cq_head = (cq_head + 1) % cq->cq_ring.size; +- if (!cq_head) { +- cq_polarity ^= 1; +- } +- } while (true); +-} +- +-void sxe2_clean_cqes(struct sxe2_qp_common *qp, +- struct sxe2_ucq *vendor_cq, int cq_type) +-{ +- struct sxe2_cq_uk *cq = &vendor_cq->cq; +- int ret; +- +- ret = pthread_spin_lock(&vendor_cq->lock); +- if (ret) { +- return; +- } +- +- sxe2_clean_base_cq(qp, cq, cq_type); +- +- pthread_spin_unlock(&vendor_cq->lock); +-} +- +-int sxe2_start_poll(struct ibv_cq_ex *ibvcq_ex, struct ibv_poll_cq_attr *attr) +-{ +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_rdma_ucontext *sctx = NULL; +- int ret; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- ret = pthread_spin_lock(&vendor_cq->lock); +- if (ret) { +- return ret; +- } +- +- cur_cqe = &vendor_cq->cur_cqe; +- ret = sxe2_poll_one(&vendor_cq->cq, cur_cqe, NULL); +- if (ret == SXE2_CQ_OK) { +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("sxe2 start poll, cqn [%u] wr_id %"PRIu64" succeed.\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id); +- } +- return SXE2_CQ_OK; +- } +- +- if (ret == SXE2_CQ_NOENT) { +- ret = ENOENT; +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("sxe2 start poll, cqn [%u] wr_id %"PRIu64" failed[%d].\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, ret); +- } +- +- pthread_spin_unlock(&vendor_cq->lock); +- +- return ret; +-} +- +-int sxe2_next_poll(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_rdma_ucontext *sctx = NULL; +- int ret; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("sxe2 next poll, cqn [%u] wr_id %"PRIu64".\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id); +- } +- cur_cqe = &vendor_cq->cur_cqe; +- ret = sxe2_poll_one(&vendor_cq->cq, cur_cqe, NULL); +- if (ret == SXE2_CQ_NOENT) { +- return ENOENT; +- } +- +- return ret; +-} +- +-void sxe2_end_poll(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("sxe2 end poll, cqn [%u] wr_id %"PRIu64".\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id); +- } +- +- pthread_spin_unlock(&vendor_cq->lock); +-} +- +-uint64_t sxe2_wc_read_completion_ts(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- uint64_t timestamp; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- timestamp = vendor_cq->cur_cqe.info.field.cqe_timestamp / HCA_CORE_CLOCK_500_MHZ; +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read comp ts, cqn [%u] wr_id %"PRIu64" ret %"PRIu64"\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, timestamp); +- } +- return timestamp; +-} +- +-uint64_t sxe2_wc_read_completion_wallclock_ns(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- uint64_t timestamp_ns; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- timestamp_ns = vendor_cq->cur_cqe.info.field.cqe_timestamp * 1000; +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read timestamp ns, cqn [%u] wr_id %"PRIu64" ret %"PRIu64"\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, timestamp_ns); +- } +- +- return timestamp_ns; +-} +- +-enum ibv_wc_opcode sxe2_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_rdma_ucontext *sctx = NULL; +- enum ibv_wc_opcode opcode; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- cur_cqe = &vendor_cq->cur_cqe; +- +- if (cur_cqe->info.field.qp_type == SXE2_CQE_QTYPE_SQ) { +- opcode = sxe2_get_ib_wc_op_sq(cur_cqe); +- } else { +- opcode = sxe2_get_ib_wc_op_rq(cur_cqe); +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read opcode, cqn [%u] wr_id %"PRIu64" ret %d\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, opcode); +- } +- +- return opcode; +-} +- +-uint32_t sxe2_wc_read_vendor_err(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __u32 error = 0; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- cur_cqe = &vendor_cq->cur_cqe; +- if (cur_cqe->info.field.error) { +- error = (__u32)(cur_cqe->info.field.major_err << 16 | cur_cqe->info.field.minor_err); +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read vendor error, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, error); +- } +- +- return error; +-} +- +-unsigned int sxe2_wc_read_wc_flags(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_qp_common *qp; +- struct ibv_qp *ib_qp; +- unsigned int wc_flags = 0; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- cur_cqe = &vendor_cq->cur_cqe; +- qp = (struct sxe2_qp_common *)cur_cqe->info.field.qpc; +- ib_qp = qp->back_qp; +- +- if (cur_cqe->info.field.imm_data_flag) { +- wc_flags |= IBV_WC_WITH_IMM; +- } +- +- if (ib_qp->qp_type == IBV_QPT_UD) { +- wc_flags |= IBV_WC_GRH; +- } else { +- if ((cur_cqe->info.field.qp_type == SXE2_CQE_QTYPE_RQ) +- && (cur_cqe->info.field.stag_or_lrkey)) { +- wc_flags |= IBV_WC_WITH_INV; +- } +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read wc flags, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, wc_flags); +- } +- +- return wc_flags; +-} +- +-uint32_t sxe2_wc_read_byte_len(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- __u32 bytes; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, +- verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- bytes = vendor_cq->cur_cqe.bytes; +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read length, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, bytes); +- } +- +- return bytes; +-} +- +-__be32 sxe2_wc_read_imm_data(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_ucq *vendor_cq = NULL; +- __u32 imme_data = 0; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- cur_cqe = &vendor_cq->cur_cqe; +- +- if (cur_cqe->info.field.imm_data_flag) { +- imme_data = cur_cqe->info.field.imme_data; +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read imme data, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, imme_data); +- } +- return htonl(imme_data); +-} +- +-uint32_t sxe2_wc_read_qp_num(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- __u32 qpn; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, +- verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- qpn = vendor_cq->cur_cqe.info.field.qp_id; +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read qpn, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, qpn); +- } +- return qpn; +-} +- +-uint32_t sxe2_wc_read_src_qp(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_cqe_info *cur_cqe; +- struct sxe2_ucq *vendor_cq = NULL; +- struct sxe2_qp_common *qp; +- struct ibv_qp *ib_qp; +- __u32 src_qpn; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- cur_cqe = &vendor_cq->cur_cqe; +- qp = (struct sxe2_qp_common *)cur_cqe->info.field.qpc; +- ib_qp = qp->back_qp; +- if (ib_qp->qp_type == IBV_QPT_UD) { +- src_qpn = cur_cqe->info.field.ud_src_qpn; +- } else { +- src_qpn = cur_cqe->info.field.qp_id; +- } +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read src qpn, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, src_qpn); +- } +- return src_qpn; +-} +- +-uint32_t sxe2_wc_read_slid(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read src lid, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, 0); +- } +- +- return 0; +-} +- +-uint8_t sxe2_wc_read_sl(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read sl, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, 0); +- } +- +- return 0; +-} +- +-uint8_t sxe2_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcq_ex) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_ucq *vendor_cq = NULL; +- +- vendor_cq = container_of(ibvcq_ex, struct sxe2_ucq, verbs_cq.cq_ex); +- sctx = container_of(vendor_cq->verbs_cq.cq_ex.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wc read dlid path bits, cqn [%u] wr_id %"PRIu64" ret %u\n", +- vendor_cq->cq.cq_id, vendor_cq->verbs_cq.cq_ex.wr_id, 0); +- } +- +- return 0; +-} +- +-static __le64 *sxe2_srq_get_next_recv_wqe(struct sxe2_srq_verbs *srq, __u32 *wqe_idx) +-{ +- int ret_code; +- __le64 *wqe; +- +- if (SXE2_RING_FULL_ERR(srq->srq_ring)) +- return NULL; +- +- *wqe_idx = SXE2_RING_CURRENT_HEAD(srq->srq_ring); +- if (srq->srqe_array[*wqe_idx] == SXE2_SRQE_BUSY) +- return NULL; +- +- SXE2_RING_MOVE_HEAD(srq->srq_ring, ret_code); +- if (ret_code) +- return NULL; +- +- srq->srqe_array[*wqe_idx] = SXE2_SRQE_BUSY; +- +- if (!*wqe_idx) +- srq->srq_polarity = !srq->srq_polarity; +- +- wqe = srq->srq_base[*wqe_idx * srq->wqe_size_multiplier].elem; +- +- return wqe; +-} +- +-static int sxe2_hw_srq_post_receive(struct sxe2_srq_verbs *srq, +- struct sxe2_rq_info *info) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_usrq *usrq = NULL; +- __u32 wqe_idx, i, byte_off; +- __u32 addl_sge_cnt; +- __le64 *wqe; +- __u64 hdr; +- +- usrq = container_of(srq, struct sxe2_usrq, srq_verbs); +- sctx = to_sctx(usrq->vsrq.srq.context); +- +- if (srq->max_srq_frag_cnt < info->num_sges) +- return -EINVAL; +- +- wqe = sxe2_srq_get_next_recv_wqe(srq, &wqe_idx); +- if (!wqe) +- return -ENOMEM; +- +- srq->srq_wrid_array[wqe_idx] = info->wr_id; +- +- addl_sge_cnt = info->num_sges > 1 ? info->num_sges - 1 : 0; +- sxe2_set_sgelist_data(wqe, 0, info->sg_list, srq->srq_polarity); +- +- for (i = 1, byte_off = 32; i < info->num_sges; i++) { +- sxe2_set_sgelist_data(wqe, byte_off, &info->sg_list[i], srq->srq_polarity); +- byte_off += 16; +- } +- +- if (!(info->num_sges & 0x01) && info->num_sges) { +- sxe2_set_sgelist_data(wqe, byte_off, NULL, srq->srq_polarity); +- } +- +- hdr = FIELD_PREP(SXE2_WQE_ADDSGECNT, addl_sge_cnt) | +- FIELD_PREP(SXE2_WQE_VALID, srq->srq_polarity); +- +- udma_to_device_barrier(); +- +- set_64bit_val(wqe, 24, hdr); +- +- set_64bit_val(srq->db_note, 0, +- (__u64)SXE2_RING_CURRENT_HEAD(srq->srq_ring) * srq->wqe_size_multiplier); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("POST SRQ RCV(srqn = %u): wqe_idx %u "\ +- "wr_id %llu ring_size %u mul_size %d\n", +- srq->srq_id, wqe_idx, srq->srq_wrid_array[wqe_idx], +- SXE2_RING_SIZE(srq->srq_ring), srq->wqe_size_multiplier); +- for (i = 0; i < info->num_sges; i++) { +- SXE2_VERBS_LOG_DEBUG_BDF("sgelist[%d] addr %" PRIu64 " len [%u] lkey [%u]\n", +- i, info->sg_list[i].addr, info->sg_list[i].length, +- info->sg_list[i].lkey); +- } +- } +- +- return 0; +-} +- +-int sxe2_upost_srq_recv(struct ibv_srq *ibv_srq, +- struct ibv_recv_wr *ibv_wr, struct ibv_recv_wr **bad_wr) +-{ +- struct sxe2_usrq *usrq = to_usrq(ibv_srq); +- struct sxe2_srq_verbs *verbs_srq = &usrq->srq_verbs; +- struct sxe2_rq_info rq_info = {}; +- int err = 0; +- +- pthread_spin_lock(&usrq->lock); +- while (ibv_wr) { +- if (ibv_wr->num_sge > (int)verbs_srq->max_srq_frag_cnt) { +- err = -EINVAL; +- goto out; +- } +- +- rq_info.num_sges = (__u32)ibv_wr->num_sge; +- rq_info.wr_id = ibv_wr->wr_id; +- rq_info.sg_list = ibv_wr->sg_list; +- err = sxe2_hw_srq_post_receive(verbs_srq, &rq_info); +- if (err) +- goto out; +- +- ibv_wr = ibv_wr->next; +- } +- +-out: +- pthread_spin_unlock(&usrq->lock); +- +- if (err) +- *bad_wr = ibv_wr; +- +- return err; +-} +- +-static __le64 *sxe2_init_wqe(struct sxe2_uqp *uqp, __u64 wr_id, unsigned int opcode, bool ord_fence) +-{ +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_qp_common *qp; +- unsigned int send_flags = uqp->verbs_qp.qp_ex.wr_flags; +- __le64 *wqe; +- __u64 hdr = 0; +- +- qp = &uqp->qp; +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("init wqe (qpn = %u): wr_id %"PRIu64" "\ +- "opcode %u wr_flags %x ring_size %u \n", +- uqp->verbs_qp.qp_ex.qp_base.qp_num, +- uqp->verbs_qp.qp_ex.wr_id, opcode, +- send_flags, SXE2_RING_SIZE(qp->sq_ring)); +- } +- +- if (SXE2_RING_FULL_ERR(qp->sq_ring)) { +- uqp->wqe_hdr = NULL; +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("sq wqe buf full, qpn [%u]\n", +- uqp->verbs_qp.qp_ex.qp_base.qp_num); +- return NULL; +- } +- +- wqe = (__u64 *)uqp->wqebuf; +- memset(wqe, 0, SXE2_QP_WQE_MIN_SIZE); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_OPCODE, opcode); +- if ((send_flags & IBV_SEND_SIGNALED) || uqp->sq_sig_all) { +- SXE2_SET_FIELD(hdr, SXE2_WQE_SIGCOMPL, true); +- } +- +- if (send_flags & IBV_SEND_FENCE || ord_fence) { +- SXE2_SET_FIELD(hdr, SXE2_WQE_READFENCE, true); +- } +- +- if (uqp->send_cq->report_rtt) { +- SXE2_SET_FIELD(hdr, SXE2_WQE_REPORTRTT, true); +- } +- +- if(uqp->qp.push_db) { +- SXE2_SET_FIELD(hdr, SXE2_WQE_PUSHWQE, true); +- } +- +- set_64bit_val(wqe, 24, hdr); +- +- uqp->wqe_hdr = wqe; +- +- return wqe; +-} +- +-void sxe2_wr_start(struct ibv_qp_ex *qp_ex) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- enum ibv_qp_state state = qp_ex->qp_base.state; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr start enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- if (state < IBV_QPS_RTS) { +- uqp->err = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("qp state err [%d] qpn [%u]\n", +- state, qp_ex->qp_base.qp_num); +- return; +- } +- +- (void)pthread_spin_lock(&uqp->lock); +- +- uqp->rb_sq_head = SXE2_RING_CURRENT_HEAD(uqp->qp.sq_ring); +- uqp->err = 0; +-} +- +-int sxe2_wr_complete(struct ibv_qp_ex *qp_ex) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- bool push_wqe; +- int err; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- err = uqp->err; +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr complete enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- if (unlikely(err)) { +- SXE2_RING_CURRENT_HEAD(uqp->qp.sq_ring) = uqp->rb_sq_head; +- SXE2_VERBS_LOG_ERROR_BDF("qp err [%d] qpn [%u]\n", +- err, qp_ex->qp_base.qp_num); +- goto out; +- } +- +- qp = &uqp->qp; +- +- push_wqe = qp->push_db ? true : false; +- +- if (push_wqe) +- sxe2_qp_push_wqe(qp, uqp->cur_wqe, uqp->quanta, uqp->wqe_idx); +- else +- sxe2_qp_ring_normal_db(qp); +- +- sxe2_dump_wqe(qp, uqp->cur_wqe, uqp->quanta, uqp->wqe_idx, "new_send"); +- +-out: +- pthread_spin_unlock(&uqp->lock); +- +- return err; +-} +- +-void sxe2_wr_abort(struct ibv_qp_ex *qp_ex) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr abort enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- SXE2_RING_CURRENT_HEAD(uqp->qp.sq_ring) = uqp->rb_sq_head; +- +- pthread_spin_unlock(&uqp->lock); +-} +- +-void sxe2_wr_send_rc_ud(struct ibv_qp_ex *qp_ex) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- unsigned int send_flags = qp_ex->wr_flags; +- unsigned int opcode = SXE2_OP_TYPE_SEND; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr send enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- if (send_flags & IBV_SEND_SOLICITED) +- opcode = SXE2_OP_TYPE_SEND_SOL; +- +- (void)sxe2_init_wqe(uqp, qp_ex->wr_id, opcode, false); +- +- uqp->funid = SXE2_RDMA_SEND; +-} +- +-void sxe2_wr_send_imm_rc_ud(struct ibv_qp_ex *qp_ex, __be32 imm_data) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- unsigned int send_flags = qp_ex->wr_flags; +- unsigned int opcode = SXE2_OP_TYPE_SEND; +- __le64 *wqe; +- __u64 hdr; +- __u64 val = 0; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr send immediate enter, qpn [%u] imm [%u]\n", +- qp_ex->qp_base.qp_num, be32toh(imm_data)); +- } +- if (send_flags & IBV_SEND_SOLICITED) +- opcode = SXE2_OP_TYPE_SEND_SOL; +- +- wqe = sxe2_init_wqe(uqp, qp_ex->wr_id, opcode, false); +- if (!wqe) +- return; +- +- get_64bit_val(wqe, 24, &hdr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_IMMEDATAFLG, true); +- SXE2_SET_FIELD(val, SXE2_WQE_IMMEDATA, be32toh(imm_data)); +- +- set_64bit_val(wqe, 0, val); +- set_64bit_val(wqe, 24, hdr); +- +- uqp->funid = SXE2_RDMA_SEND; +-} +- +-void sxe2_wr_send_inv_rc(struct ibv_qp_ex *qp_ex, uint32_t invalidate_rkey) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- unsigned int send_flags = qp_ex->wr_flags; +- unsigned int opcode = SXE2_OP_TYPE_SEND_INV; +- __le64 *wqe; +- __u64 hdr; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr send inv enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- if (send_flags & IBV_SEND_SOLICITED) { +- opcode = SXE2_OP_TYPE_SEND_SOL_INV; +- } +- +- wqe = sxe2_init_wqe(uqp, qp_ex->wr_id, opcode, false); +- if (!wqe) +- return; +- +- get_64bit_val(wqe, 24, &hdr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_REMOTEINVTAG, invalidate_rkey); +- +- set_64bit_val(wqe, 24, hdr); +- +- uqp->funid = SXE2_RDMA_SEND; +-} +- +-void sxe2_wr_rdma_read_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *wqe; +- __u64 hdr; +- bool ord_fence = false; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr read enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- if (uqp->qp.rd_fence_rate && (uqp->qp.ord_cnt++ == uqp->qp.rd_fence_rate)) { +- ord_fence = true; +- uqp->qp.ord_cnt = 0; +- } +- +- wqe = sxe2_init_wqe(uqp, qp_ex->wr_id, SXE2_OP_TYPE_RDMA_READ, ord_fence); +- if (!wqe) +- return; +- +- get_64bit_val(wqe, 24, &hdr); +- +- sxe2_set_remote_offset(wqe, 16, remote_addr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_REMOTETAG, rkey); +- +- set_64bit_val(wqe, 24, hdr); +- +- uqp->funid = SXE2_RDMA_READ; +-} +- +-void sxe2_wr_rdma_write_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr) +-{ +- unsigned int send_flags = qp_ex->wr_flags; +- unsigned int opcode = SXE2_OP_TYPE_RDMA_WRITE; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *wqe; +- __u64 hdr; +- +- if (send_flags & IBV_SEND_SOLICITED) +- opcode = SXE2_OP_TYPE_RDMA_WRITE_SOL; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr write enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- wqe = sxe2_init_wqe(uqp, qp_ex->wr_id, opcode, false); +- if (!wqe) +- return; +- +- get_64bit_val(wqe, 24, &hdr); +- +- sxe2_set_remote_offset(wqe, 16, remote_addr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_REMOTETAG, rkey); +- +- set_64bit_val(wqe, 24, hdr); +- +- uqp->funid = SXE2_RDMA_WRITE; +-} +- +-void sxe2_wr_rdma_write_imm_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr, __be32 imm_data) +-{ +- unsigned int send_flags = qp_ex->wr_flags; +- unsigned int opcode = SXE2_OP_TYPE_RDMA_WRITE; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *wqe; +- __u64 hdr; +- __u64 val = 0; +- +- if (send_flags & IBV_SEND_SOLICITED) +- opcode = SXE2_OP_TYPE_RDMA_WRITE_SOL; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr write with immediate enter, "\ +- "qpn [%u] imm [%u]\n", qp_ex->qp_base.qp_num, +- be32toh(imm_data)); +- } +- +- wqe = sxe2_init_wqe(uqp, qp_ex->wr_id, opcode, false); +- if (!wqe) +- return; +- +- get_64bit_val(wqe, 24, &hdr); +- +- sxe2_set_remote_offset(wqe, 16, remote_addr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_REMOTETAG, rkey); +- SXE2_SET_FIELD(hdr, SXE2_WQE_IMMEDATAFLG, true); +- SXE2_SET_FIELD(val, SXE2_WQE_IMMEDATA, be32toh(imm_data)); +- +- set_64bit_val(wqe, 24, hdr); +- +- set_64bit_val(wqe, 0, val); +- uqp->funid = SXE2_RDMA_WRITE; +-} +- +-void sxe2_wr_set_ud_addr(struct ibv_qp_ex *qp_ex, struct ibv_ah *ah, +- uint32_t remote_qpn, uint32_t remote_qkey) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- struct sxe2_uah *uah; +- __le64 *wqe; +- __u64 hdr; +- __u64 val = 0; +- +- uah = container_of(ah, struct sxe2_uah, ibv_ah); +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set ud addr enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- wqe = uqp->wqe_hdr; +- if (!wqe) { +- return; +- } +- +- get_64bit_val(wqe, 24, &hdr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_AH_ID, uah->ah_id); +- +- SXE2_SET_FIELD(val, SXE2_WQE_DESTQKEY, remote_qkey); +- SXE2_SET_FIELD(val, SXE2_WQE_DESTQPN, remote_qpn); +- +- set_64bit_val(wqe, 16, val); +- set_64bit_val(wqe, 24, hdr); +-} +- +-static void sxe2_fill_inline_data_buflist(__u8 *wqe, size_t num_buf, +- const struct ibv_data_buf *buf_list, __u8 polarity) +-{ +- __u8 inline_valid = (__u8)(polarity << SXE2_INLINE_VALID_S); +- __u32 quanta_bytes_remaining = 8; +- __u32 i; +- bool first_quanta = true; +- +- wqe += 8; +- +- for (i = 0; i < num_buf; i++) { +- __u8 *cur_sge = (__u8 *)(uintptr_t)buf_list[i].addr; +- __u32 sge_len = (__u32)buf_list[i].length; +- +- while (sge_len) { +- __u32 bytes_copied; +- +- bytes_copied = min(sge_len, quanta_bytes_remaining); +- memcpy(wqe, cur_sge, bytes_copied); +- wqe += bytes_copied; +- cur_sge += bytes_copied; +- quanta_bytes_remaining -= bytes_copied; +- sge_len -= bytes_copied; +- +- if (!quanta_bytes_remaining) { +- quanta_bytes_remaining = 31; +- +- if (first_quanta) { +- first_quanta = false; +- wqe += 16; +- } else { +- *wqe = inline_valid; +- wqe++; +- } +- } +- } +- } +- if (!first_quanta && quanta_bytes_remaining < 31) { +- *(wqe + quanta_bytes_remaining) = inline_valid; +- } +-} +- +-static inline void sxe2_fill_sgelist_data(void *wqe, __u32 offset, +- const struct ibv_sge *sge, __u8 valid) +-{ +- __u64 frag_info = 0; +- uint32_t len; +- +- if (sge) { +- len = (sge->length & ((uint32_t)1 << 31)) ? 0 : sge->length; +- set_64bit_val(wqe, offset,sge->addr); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_VALID, valid); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_LEN, len); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_STAG, sge->lkey); +- set_64bit_val(wqe, offset + 8, frag_info); +- } else { +- SXE2_SET_FIELD(frag_info, SXE2_WQE_VALID, valid); +- set_64bit_val(wqe, offset, 0); +- set_64bit_val(wqe, offset + 8, frag_info); +- } +-} +- +-void sxe2_wr_set_inline_data_rc_ud(struct ibv_qp_ex *qp_ex, +- void *addr, size_t length) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- const struct ibv_data_buf buff = {.addr = addr, .length = length}; +- __le64 *wqe; +- __le64 *cache_hdr; +- __u32 wqe_idx; +- __u16 quanta; +- bool push_wqe; +- __u64 hdr; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set inline data enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- qp = &uqp->qp; +- push_wqe = qp->push_db ? true : false; +- cache_hdr = uqp->wqe_hdr; +- if (!cache_hdr) { +- SXE2_VERBS_LOG_WARN_BDF("wr cache hdr null, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- if (length > uqp->qp.max_inline_data) { +- SXE2_VERBS_LOG_ERROR_BDF("wr length %zu exceed %u , qpn [%u]\n", +- length, uqp->qp.max_inline_data, qp_ex->qp_base.qp_num); +- uqp->err = EINVAL; +- return; +- } +- +- quanta = sxe2_inline_to_quanta_cnt((__u32)length); +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, (__u32)length, +- qp_ex->wr_id, push_wqe); +- if (!wqe) { +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("wr no memory, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- uqp->wqe_idx = wqe_idx; +- uqp->cur_wqe = wqe; +- uqp->quanta = quanta; +- +- memcpy(wqe, cache_hdr, SXE2_QP_WQE_MIN_SIZE); +- +- sxe2_fill_inline_data_buflist((__u8 *)wqe, 1, &buff, qp->swqe_polarity); +- +- get_64bit_val(wqe, 24, &hdr); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_INLINEDATAFLG, true); +- SXE2_SET_FIELD(hdr, SXE2_WQE_INLINEDATALEN, length); +- SXE2_SET_FIELD(hdr, SXE2_WQE_VALID, qp->swqe_polarity); +- set_64bit_val(wqe, 24, hdr); +- if (uqp->funid == SXE2_RDMA_SEND) { +- sxe2_dump_wqe_hdr(uqp, &hdr, SXE2_RDMA_SEND_INLINE); +- } else if (uqp->funid == SXE2_RDMA_WRITE) { +- sxe2_dump_wqe_hdr(uqp, &hdr, SXE2_RDMA_WRITE_INLINE); +- } +-} +- +-void sxe2_wr_set_inline_data_list_rc_ud(struct ibv_qp_ex *qp_ex, +- size_t num_buf, const struct ibv_data_buf *buf_list) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- size_t length = 0; +- __le64 *wqe; +- __le64 *cache_hdr; +- __u32 wqe_idx; +- __u16 quanta; +- bool push_wqe; +- __u64 hdr; +- size_t i; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set inline data list enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- qp = &uqp->qp; +- push_wqe = qp->push_db ? true : false; +- cache_hdr = uqp->wqe_hdr; +- if (!cache_hdr) { +- SXE2_VERBS_LOG_WARN_BDF("wr cache hdr null, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- for (i = 0; i < num_buf; i++) +- length += buf_list[i].length; +- +- if (length > uqp->qp.max_inline_data) { +- uqp->err = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("wr length %zu exceed %u , qpn [%u]\n", +- length, uqp->qp.max_inline_data, qp_ex->qp_base.qp_num); +- return; +- } +- +- quanta = sxe2_inline_to_quanta_cnt((__u32)length); +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, (__u32)length, +- qp_ex->wr_id, push_wqe); +- if (!wqe) { +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("wr no memory, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- uqp->wqe_idx = wqe_idx; +- uqp->cur_wqe = wqe; +- uqp->quanta = quanta; +- +- memcpy(wqe, cache_hdr, SXE2_QP_WQE_MIN_SIZE); +- +- sxe2_fill_inline_data_buflist((__u8 *)wqe, num_buf, buf_list, qp->swqe_polarity); +- +- get_64bit_val(wqe, 24, &hdr); +- SXE2_SET_FIELD(hdr, SXE2_WQE_INLINEDATAFLG, true); +- SXE2_SET_FIELD(hdr, SXE2_WQE_INLINEDATALEN, length); +- SXE2_SET_FIELD(hdr, SXE2_WQE_VALID, qp->swqe_polarity); +- set_64bit_val(wqe, 24, hdr); +- +- if (uqp->funid == SXE2_RDMA_SEND) { +- sxe2_dump_wqe_hdr(uqp, &hdr, SXE2_RDMA_SEND_INLINE); +- } else if (uqp->funid == SXE2_RDMA_WRITE) { +- sxe2_dump_wqe_hdr(uqp, &hdr, SXE2_RDMA_WRITE_INLINE); +- } +-} +- +-void sxe2_wr_set_sge_rc_ud(struct ibv_qp_ex *qp_ex, uint32_t lkey, +- uint64_t addr, uint32_t length) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *cache_hdr; +- __le64 *wqe; +- bool push_wqe; +- bool has_imme_data = false; +- __u32 wqe_idx; +- __u32 addl_frag_cnt; +- __u16 quanta; +- __u64 hdr; +- int ret_code; +- __u32 frag_cnt = 1; +- __u64 frag_info = 0; +- const struct ibv_sge sg_list = {.addr = addr, .length = length, .lkey = lkey}; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set sge data enter, "\ +- "qpn [%u] addr %" PRIu64 " len [%u]\n", +- qp_ex->qp_base.qp_num, addr, length); +- } +- +- qp = &uqp->qp; +- push_wqe = qp->push_db ? true : false; +- cache_hdr = uqp->wqe_hdr; +- if (!cache_hdr) { +- SXE2_VERBS_LOG_WARN_BDF("wr cache hdr null, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- get_64bit_val(cache_hdr, 24, &hdr); +- has_imme_data = SXE2_GET_FIELD(SXE2_WQE_IMMEDATAFLG, hdr); +- if (has_imme_data) { +- frag_cnt = frag_cnt + 1; +- } +- +- ret_code = sxe2_fragcnt_to_quanta_cnt(frag_cnt, &quanta); +- if (ret_code) { +- uqp->err = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("wr sge num %u exceed, qpn [%u]\n", +- frag_cnt, qp_ex->qp_base.qp_num); +- return; +- } +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, length, +- qp_ex->wr_id, push_wqe); +- if (!wqe) { +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("wr no memory, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- uqp->wqe_idx = wqe_idx; +- uqp->cur_wqe = wqe; +- uqp->quanta = quanta; +- +- addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; +- +- memcpy(wqe, cache_hdr, SXE2_QP_WQE_MIN_SIZE); +- +- if (has_imme_data) { +- sxe2_fill_sgelist_data(wqe, 32, &sg_list, qp->swqe_polarity); +- sxe2_fill_sgelist_data(wqe, 48, NULL, qp->swqe_polarity); +- } else { +- sxe2_fill_sgelist_data(wqe, 0, &sg_list, qp->swqe_polarity); +- } +- +- if (length == 0) { +- get_64bit_val(wqe, 8, &frag_info); +- frag_info = frag_info & (~SXE2_WQE_FRAG_VALID); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_VALID, !qp->swqe_polarity); +- set_64bit_val(wqe, 8, frag_info); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set frag_info [%#llx] field [%d]\n", +- frag_info, !qp->swqe_polarity); +- } +- } +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_ADDSGECNT, addl_frag_cnt); +- SXE2_SET_FIELD(hdr, SXE2_WQE_VALID, qp->swqe_polarity); +- set_64bit_val(wqe, 24, hdr); +- sxe2_dump_wqe_hdr(uqp, &hdr, uqp->funid); +-} +- +-void sxe2_wr_set_sge_list_rc_ud(struct ibv_qp_ex *qp_ex, +- size_t num_sge, const struct ibv_sge *sg_list) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_qp_common *qp; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *cache_hdr; +- bool push_wqe; +- bool has_imme_data = false; +- __le64 *wqe; +- __u32 i, wqe_idx, total_size = 0, byte_off; +- int ret_code; +- __u32 frag_cnt, addl_frag_cnt; +- __u64 hdr; +- __u16 quanta; +- __u64 frag_info = 0; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- qp = &uqp->qp; +- push_wqe = qp->push_db ? true : false; +- cache_hdr = uqp->wqe_hdr; +- if (!cache_hdr) { +- SXE2_VERBS_LOG_WARN_BDF("wr cache hdr null, qpn [%u]\n", qp_ex->qp_base.qp_num); +- return; +- } +- if (qp->max_sq_sge_cnt < num_sge) { +- uqp->err = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("wr sge num %zu exceed %u, qpn [%u]\n", +- num_sge, qp->max_sq_sge_cnt, qp_ex->qp_base.qp_num); +- return; +- } +- for (i = 0; i < num_sge; i++) { +- total_size += sg_list[i].length; +- } +- get_64bit_val(cache_hdr, 24, &hdr); +- has_imme_data = SXE2_GET_FIELD(SXE2_WQE_IMMEDATAFLG, hdr); +- if (has_imme_data) { +- frag_cnt = (__u32)(num_sge + 1); +- } else { +- frag_cnt = (__u32)num_sge; +- } +- ret_code = sxe2_fragcnt_to_quanta_cnt(frag_cnt, &quanta); +- if (ret_code) { +- uqp->err = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("wr sge num %u exceed, qpn [%u]\n", +- frag_cnt, qp_ex->qp_base.qp_num); +- return; +- } +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, +- qp_ex->wr_id, push_wqe); +- if (!wqe) { +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("wr no memory, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- uqp->wqe_idx = wqe_idx; +- uqp->cur_wqe = wqe; +- uqp->quanta = quanta; +- +- memcpy(wqe, cache_hdr, SXE2_QP_WQE_MIN_SIZE); +- +- addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; +- if (has_imme_data) { +- i = 0; +- } else { +- sxe2_fill_sgelist_data(wqe, 0, sg_list, qp->swqe_polarity); +- i = 1; +- } +- if (total_size == 0) { +- get_64bit_val(wqe, 8, &frag_info); +- frag_info = frag_info & (~SXE2_WQE_FRAG_VALID); +- SXE2_SET_FIELD(frag_info, SXE2_WQE_FRAG_VALID, !qp->swqe_polarity); +- set_64bit_val(wqe, 8, frag_info); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr set frag_info [%#llx] field [%d]\n", +- frag_info, !qp->swqe_polarity); +- } +- } +- +- for (byte_off = 32; i < num_sge;) { +- sxe2_fill_sgelist_data(wqe, byte_off, &sg_list[i], qp->swqe_polarity); +- byte_off += 16; +- i++; +- } +- +- if (!(frag_cnt & 0x01) && frag_cnt) { +- sxe2_fill_sgelist_data(wqe, byte_off, NULL, qp->swqe_polarity); +- } +- SXE2_SET_FIELD(hdr, SXE2_WQE_ADDSGECNT, addl_frag_cnt); +- SXE2_SET_FIELD(hdr, SXE2_WQE_VALID, qp->swqe_polarity); +- set_64bit_val(wqe, 24, hdr); +- sxe2_dump_wqe_hdr(uqp, &hdr, uqp->funid); +-} +- +-void sxe2_wr_bind_mw_rc(struct ibv_qp_ex *qp_ex, struct ibv_mw *mw, +- uint32_t rkey, const struct ibv_mw_bind_info *bind_info) +-{ +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __u64 hdr; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr bind mw unsupport, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- uqp->funid = SXE2_RDMA_BIND_MW; +- +- uqp->err = EOPNOTSUPP; +- +- sxe2_dump_wqe_hdr(uqp, &hdr, uqp->funid); +- return; +-} +- +-void sxe2_wr_local_inv_rc(struct ibv_qp_ex *qp_ex, uint32_t invalidate_rkey) +-{ +- struct sxe2_qp_common *qp; +- struct sxe2_uqp *uqp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- __le64 *cache_hdr; +- bool push_wqe; +- __u32 wqe_idx; +- __le64 *wqe; +- __u64 hdr; +- __u16 quanta = SXE2_QP_WQE_MIN_QUANTA; +- __u64 val = 0; +- +- uqp = container_of(qp_ex, struct sxe2_uqp, verbs_qp.qp_ex); +- sctx = container_of(uqp->verbs_qp.qp_ex.qp_base.context, +- struct sxe2_rdma_ucontext, ibv_ctx.context); +- if (sctx->enable_io_log) { +- SXE2_VERBS_LOG_DEBUG_BDF("wr local inv enter, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- } +- +- qp = &uqp->qp; +- push_wqe = qp->push_db ? true : false; +- cache_hdr = sxe2_init_wqe(uqp, qp_ex->wr_id, SXE2_OP_TYPE_LOCAL_INV, false); +- if (!cache_hdr) { +- return; +- } +- +- wqe = sxe2_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, +- 0, qp_ex->wr_id, push_wqe); +- if (!wqe) { +- uqp->err = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("wr no memory, qpn [%u]\n", +- qp_ex->qp_base.qp_num); +- return; +- } +- +- get_64bit_val(cache_hdr, 24, &hdr); +- +- uqp->wqe_idx = wqe_idx; +- uqp->cur_wqe = wqe; +- uqp->quanta = quanta; +- +- SXE2_SET_FIELD(val, SXE2_WQE_REMOTEINVTAG, invalidate_rkey); +- set_64bit_val(cache_hdr, 8, val); +- +- memcpy(wqe, cache_hdr, SXE2_QP_WQE_MIN_SIZE); +- +- SXE2_SET_FIELD(hdr, SXE2_WQE_VALID, qp->swqe_polarity); +- set_64bit_val(wqe, 24, hdr); +- uqp->funid = SXE2_RDMA_LOCAL_INV; +- sxe2_dump_wqe_hdr(uqp, &hdr, uqp->funid); +-} +- +-void sxe2_push_nop_wqe(struct sxe2_qp_common *qp) { +- __le64 *wqe; +- __u32 wqe_idx; +- +- wqe_idx = SXE2_RING_CURRENT_HEAD(qp->sq_ring); +- if (!wqe_idx) { +- qp->swqe_polarity = !qp->swqe_polarity; +- } +- wqe = qp->sq_base[wqe_idx].elem; +- +- qp->sq_wrtrk_array[wqe_idx].quanta = SXE2_QP_WQE_MIN_QUANTA; +- +- set_64bit_val(wqe, 0, 0); +- set_64bit_val(wqe, 8, 0); +- set_64bit_val(wqe, 16, 0); +- +- sxe2_set_nop_hdr(wqe, 24, qp); +- SXE2_RING_MOVE_HEAD_NOCHECK(qp->sq_ring); +- SXE2_RING_MOVE_TAIL(qp->sq_ring); +-} +- +diff -Naur rdma-core-48.0/providers/sxe2/io.h rdma-core-48.0.bak/providers/sxe2/io.h +--- rdma-core-48.0/providers/sxe2/io.h 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/io.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,477 +0,0 @@ +- +- +-#ifndef __IO_H__ +-#define __IO_H__ +- +-#define SXE2_OP_TYPE_RDMA_WRITE 0x00 +-#define SXE2_OP_TYPE_RDMA_READ 0x01 +-#define SXE2_OP_TYPE_RSVD1 0x02 +-#define SXE2_OP_TYPE_SEND 0x03 +-#define SXE2_OP_TYPE_SEND_INV 0x04 +-#define SXE2_OP_TYPE_SEND_SOL 0x05 +-#define SXE2_OP_TYPE_SEND_SOL_INV 0x06 +-#define SXE2_OP_TYPE_RSVD2 0x07 +-#define SXE2_OP_TYPE_BIND_MW 0x08 +-#define SXE2_OP_TYPE_FAST_REG_MR 0x09 +-#define SXE2_OP_TYPE_LOCAL_INV 0x0a +-#define SXE2_OP_TYPE_RDMA_READ_INV 0x0b +-#define SXE2_OP_TYPE_NOP 0x0c +-#define SXE2_OP_TYPE_RDMA_WRITE_SOL 0x0d +- +-#define SXE2_CQE_QTYPE_RQ 0 +-#define SXE2_CQE_QTYPE_SQ 1 +- +-#define SXE2_MAX_FRAGCNT 16 +-#define SXE2_QP_WQE_MIN_SIZE 32 +-#define SXE2_QP_WQE_MAX_SIZE 256 +-#define SXE2_QP_MAX_INLINE_PER_QUANTA 31 +- +-#define SXE2_SQ_RSVD 8 +-#define SXE2_RQ_RSVD 8 +- +-#define SXE2_INLINE_VALID_S 7 +-#define HCA_CORE_CLOCK_500_MHZ 500 +- +-#define SXE2_WQE_IMMEDATA GENMASK_ULL(31, 0) +- +-#define SXE2_WQE_FRAG_LEN GENMASK_ULL(62, 32) +-#define SXE2_WQE_FRAG_STAG GENMASK_ULL(31, 0) +-#define SXE2_WQE_FRAG_VALID BIT_ULL(63) +- +-#define SXE2_WQE_DESTQKEY GENMASK_ULL(31, 0) +-#define SXE2_WQE_DESTQPN GENMASK_ULL(55, 32) +- +-#define SXE2_WQE_REMOTEINVTAG GENMASK_ULL(31, 0) +-#define SXE2_WQE_REMOTETAG GENMASK_ULL(31, 0) +-#define SXE2_WQE_AH_ID GENMASK_ULL(16, 0) +-#define SXE2_WQE_OPCODE GENMASK_ULL(37, 32) +-#define SXE2_WQE_ADDSGECNT GENMASK_ULL(41, 38) +-#define SXE2_WQE_REPORTRTT BIT_ULL(46) +-#define SXE2_WQE_IMMEDATAFLG BIT_ULL(47) +-#define SXE2_WQE_INLINEDATALEN GENMASK_ULL(55, 48) +-#define SXE2_WQE_PUSHWQE BIT_ULL(56) +-#define SXE2_WQE_INLINEDATAFLG BIT_ULL(57) +-#define SXE2_WQE_READFENCE BIT_ULL(60) +-#define SXE2_WQE_LOCALFENCE BIT_ULL(61) +-#define SXE2_WQE_SIGCOMPL BIT_ULL(62) +-#define SXE2_WQE_VALID BIT_ULL(63) +- +-#define SXE2_WQE_MR_STAG GENMASK_ULL(63, 32) +-#define SXE2_WQE_MW_STAG GENMASK_ULL(31, 0) +-#define SXE2_WQE_MW_LEN GENMASK_ULL(45, 0) +-#define SXE2_WQE_ACCESSRIGHT GENMASK_ULL(52, 48) +-#define SXE2_WQE_VA_BASE_FLAG BIT_ULL(53) +-#define SXE2_WQE_MW_TYPE BIT_ULL(54) +- +-#define SXE2_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20) +-#define SXE2_CQE_VALID BIT_ULL(63) +-#define SXE2_CQE_WQEIDX GENMASK_ULL(46, 32) +- +-#define SXE2_SET_FIELD(origin, mask, val) \ +- ((origin) |= FIELD_PREP(mask, val)) +- +-#define SXE2_GET_FIELD(mask, val) \ +- FIELD_GET(mask, val) +- +-enum sxe2_addressing_type { +- SXE2_ADDR_TYPE_ZERO_BASED = 0, +- SXE2_ADDR_TYPE_VA_BASED = 1, +-}; +- +-enum sxe2_cq_err{ +- SXE2_CQ_OK = 0, +- SXE2_CQ_NOENT = -2, +-}; +- +-enum { +- SXE2_CQ_DB_REQ_SOLICITED = 1 << 31, +- SXE2_CQ_DB_REQ_NOSOLICITED = 0 << 31 +-}; +- +-enum { +- SXE2_CQ_SET_CI = 0, +- SXE2_CQ_ARM_DB = 1, +-}; +- +-enum { +- SXE2_QP_RQ_PI = 0, +- SXE2_QP_SQ_CI = 3, +-}; +- +-enum sxe2_arm_type { +- SXE2_CQ_ARM_NEXT = 0, +- SXE2_CQ_ARM_SOLICITED = 1, +-}; +- +-enum sxe2_major_opcode { +- SXE2_SUCCESS = 0, +- SXE2_LOCAL_LEN_ERR = 0x1, +- SXE2_LOCAL_QP_OP_ERR = 0x2, +- SXE2_RSV1_ERR, +- SXE2_LOCAL_PROTECTION_ERR = 0x4, +- SXE2_WR_FLUSH_ERR = 0x5, +- SXE2_MW_BIND_ERR = 0x6, +- SXE2_FAST_REGISTER_ERR = 0x7, +- SXE2_INVALID_KEY_ERR = 0x8, +- SXE2_BAD_RESPONSE_ERR = 0x10, +- SXE2_LOCAL_ACCESS_ERR = 0x11, +- SXE2_REMOTE_INVALID_REQUEST_ERR = 0x12, +- SXE2_REMOTE_ACCESS_ERR = 0x13, +- SXE2_REMOTE_OPERATION_ERR = 0x14, +- SXE2_TRANS_RETRY_CNT_EXCEED_ERR = 0x15, +- SXE2_RNR_RETRY_CNT_EXCEED_ERR = 0x16, +- SXE2_ABORT_ERR = 0x22, +- SXE2_DIF_CHECK_ERR = 0x23, +- SXE2_SQ_FAKE_ERR = 0x24, +-}; +- +-struct sxe2_post_send { +- struct ibv_sge *sg_list; +- __u32 num_sges; +- __u32 qkey; +- __u32 dest_qp; +- __u32 ah_id; +-}; +- +-struct sxe2_rdma_write { +- struct ibv_sge *lo_sg_list; +- struct ibv_sge rem_addr; +- __u32 num_lo_sges; +-}; +- +-struct sxe2_rdma_read { +- struct ibv_sge *lo_sg_list; +- struct ibv_sge rem_addr; +- __u32 num_lo_sges; +-}; +- +-struct sxe2_bind_window { +- __u32 mr_stag; +- __u32 mw_stag; +- __u64 bind_len; +- void *va; +- enum sxe2_addressing_type addressing_type; +- bool ena_reads : 1; +- bool ena_writes : 1; +- bool mem_window_type_1 : 1; +-}; +- +-struct sxe2_local_invalidate { +- __u32 target_stag; +-}; +- +-struct sxe2_wr_info { +- __u64 wr_id; +- __u8 op_type; +- __u8 l4len; +- bool signaled : 1; +- bool read_fence : 1; +- bool local_fence : 1; +- bool inline_data : 1; +- bool imm_data_valid : 1; +- bool push_wqe : 1; +- bool report_rtt : 1; +- bool udp_hdr : 1; +- bool defer_flag : 1; +- bool post_wqe : 1; +- __u32 imm_data; +- __u32 rkey_to_inv; +- union { +- struct sxe2_post_send send; +- struct sxe2_rdma_write rdma_write; +- struct sxe2_rdma_read rdma_read; +- struct sxe2_bind_window bind_window; +- struct sxe2_local_invalidate local_inval; +- } op_info; +- enum sxe2_disp_id funid; +-}; +- +-struct sxe2_rq_info { +- __u64 wr_id; +- struct ibv_sge *sg_list; +- __u32 num_sges; +-}; +- +-struct sxe2_imme_data { +- __le64 imme_data; +-}; +- +-struct sxe2_frag_data { +- __le64 tag_offset; +- union { +- struct { +- __le64 stag : 32; +- __le64 frag_len : 31; +- __le64 frag_valid : 1; +- } field; +- __le64 val; +- } offset8; +-}; +- +-union sxe2_dqpn_data { +- struct { +- __le64 dest_qkey : 32; +- __le64 dest_qpn : 24; +- __le64 rsv0 : 8; +- } field; +- __le64 val; +-}; +- +-struct sxe2_bindmw_info { +- __le64 mw_va_base; +- union { +- struct { +- __le64 mw_key : 32; +- __le64 mr_key : 32; +- } field; +- __le64 val; +- } offset8; +- union { +- struct { +- __le64 mw_len : 46; +- __le64 rsv0 : 18; +- } field; +- __le64 val; +- } offset16; +-}; +- +-union sxe2_send_hdr { +- struct { +- __u64 remote_inv_rkey : 32; +- __u64 op : 6; +- __u64 addfragcnt : 4; +- __u64 rsvd1 : 4; +- __u64 report_rtt : 1; +- __u64 imme_data_flag : 1; +- __u64 rsvd2 : 8; +- __u64 push_wqe : 1; +- __u64 inline_data_flag : 1; +- __u64 rsvd3 : 1; +- __u64 rsvd4 : 1; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_send_inline_hdr { +- struct { +- __u64 remote_inv_rkey : 32; +- __u64 op : 6; +- __u64 rsvd1 : 8; +- __u64 report_rtt : 1; +- __u64 imme_data_flag : 1; +- __u64 inline_data_len : 8; +- __u64 push_wqe : 1; +- __u64 inline_data_flag : 1; +- __u64 rsvd2 : 2; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_write_hdr { +- struct { +- __u64 remote_key : 32; +- __u64 op : 6; +- __u64 addfragcnt : 4; +- __u64 rsvd1 : 4; +- __u64 report_rtt : 1; +- __u64 imme_data_flag : 1; +- __u64 rsvd2 : 8; +- __u64 push_wqe : 1; +- __u64 inline_data_flag : 1; +- __u64 rsvd3 : 2; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_write_inline_hdr { +- struct { +- __u64 remote_key : 32; +- __u64 op : 6; +- __u64 rsvd1 : 8; +- __u64 report_rtt : 1; +- __u64 imme_data_flag : 1; +- __u64 inline_data_len : 8; +- __u64 push_wqe : 1; +- __u64 inline_data_flag : 1; +- __u64 rsvd2 : 2; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_read_hdr { +- struct { +- __u64 remote_key : 32; +- __u64 op : 6; +- __u64 addfragcnt : 4; +- __u64 rsvd1 : 4; +- __u64 report_rtt : 1; +- __u64 rsvd2 : 1; +- __u64 rsvd3 : 8; +- __u64 push_wqe : 1; +- __u64 rsvd4 : 3; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_bindmw_hdr { +- struct { +- __u64 rsvd0 : 32; +- __u64 op : 6; +- __u64 rsvd1 : 10; +- __u64 access : 5; +- __u64 va_base_flag : 1; +- __u64 mw_type : 1; +- __u64 rsvd2 : 1; +- __u64 push_wqe : 1; +- __u64 rsvd3 : 3; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_inval_hdr { +- struct { +- __u64 rsvd0 : 32; +- __u64 op : 6; +- __u64 rsvd1 : 18; +- __u64 push_wqe : 1; +- __u64 rsvd3 : 3; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_fregmr_hdr { +- struct { +- __u64 consumer_key : 8; +- __u64 mr_index : 24; +- __u64 op : 6; +- __u64 log_entity_size : 5; +- __u64 rsvd1 : 5; +- __u64 access : 5; +- __u64 va_base_flag : 1; +- __u64 pbl_mode : 2; +- __u64 push_wqe : 1; +- __u64 rsvd3 : 3; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_rq_hdr { +- struct { +- __u64 rsv0 : 38; +- __u64 addfragcnt : 4; +- __u64 rsv1 : 21; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-union sxe2_nop_hdr { +- struct { +- __u64 rsv0 : 32; +- __u64 op : 6; +- __u64 addfragcnt : 4; +- __u64 rsv1 : 14; +- __u64 push_wqe : 1; +- __u64 rsv2 : 3; +- __u64 read_fence : 1; +- __u64 local_fence : 1; +- __u64 signaled_completion : 1; +- __u64 wqe_valid : 1; +- } field; +- __u64 val; +-}; +- +-enum sxe2_qp_caps { +- SXE2_WRITE_WITH_IMM = 1, +- SXE2_SEND_WITH_IMM = 2, +- SXE2_LLWQE_MODE = 8, +-}; +- +-enum sxe2_qp_wqe_size { +- SXE2_WQE_SIZE_32 = 32, +- SXE2_WQE_SIZE_64 = 64, +- SXE2_WQE_SIZE_96 = 96, +- SXE2_WQE_SIZE_128 = 128, +- SXE2_WQE_SIZE_256 = 256, +-}; +- +-static inline void set_64bit_val(__le64 *wqe_words, __u32 byte_index, __u64 val) +-{ +- wqe_words[byte_index >> 3] = htole64(val); +-} +- +-static inline void set_32bit_val(__le32 *wqe_words, __u32 byte_index, __u32 val) +-{ +- wqe_words[byte_index >> 2] = htole32(val); +-} +- +-static inline void get_64bit_val(__le64 *wqe_words, __u32 byte_index, +- __u64 *val) +-{ +- *val = le64toh(wqe_words[byte_index >> 3]); +-} +- +-static inline void get_32bit_val(__le32 *wqe_words, __u32 byte_index, +- __u32 *val) +-{ +- *val = le32toh(wqe_words[byte_index >> 2]); +-} +- +-static inline void db_wr32(__u32 val, __u32 *wqe_word) +-{ +- *wqe_word = val; +-} +- +-typedef int (*rdma_disp_func)(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-int sxe2_hw_send(struct sxe2_qp_common *qp, struct sxe2_wr_info *wr_info, +- bool post_sq); +-int sxe2_hw_inline_send(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-int sxe2_hw_rdma_write(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-int sxe2_hw_inline_rdma_write(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-int sxe2_hw_rdma_read(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-int sxe2_hw_mw_bind(struct sxe2_qp_common *qp, struct sxe2_wr_info *wr_info, +- bool post_sq); +-int sxe2_hw_local_invalidate(struct sxe2_qp_common *qp, +- struct sxe2_wr_info *wr_info, bool post_sq); +-void sxe2_clean_cqes(struct sxe2_qp_common *qp, +- struct sxe2_ucq *vendor_cq, int cq_type); +-void sxe2_push_nop_wqe(struct sxe2_qp_common *qp); +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/log.c rdma-core-48.0.bak/providers/sxe2/log.c +--- rdma-core-48.0/providers/sxe2/log.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/log.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,468 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "log.h" +- +-#define SXE2_LOG_EXTEND_INFO "SXE2_LOG_EXTEND_INFO" +-#define SXE2_LOG_FILE_SIZE "SXE2_LOG_FILE_SIZE" +-#define SXE2_LOG_SERIAL "SXE2_LOG_SERIAL" +-#define SXE2_LOG_COVER_WRITE "SXE2_LOG_COVER_WRITE" +- +-#define SXE2_LOG_PATH "/var/log/" +-#define SXE2_LOG_FILE_NAME "sxe2_rdma_file_current.log" +-#define SXE2_LOG_FLOCK_NAME "sxe2_rdma_file_log.flock" +-#define CONCAT(a, b) a##b +- +-#define SXE2_LOG_DEFAULT_DEBUG_FILE_SIZE \ +- (800) +-#define SXE2_LOG_DEFAULT_RELEASE_FILE_SIZE \ +- (300) +- +-log_mgr_s g_log_mgr = {false, NULL, 200, false, false, -1, -1, false}; +-pthread_mutex_t g_log_mutex; +- +-log_mgr_s *log_mgr_point_get(void) +-{ +- return &g_log_mgr; +-} +- +-void log_set_serial_switch(bool on) +-{ +- log_mgr_s *p_log_mgr = NULL; +- +- p_log_mgr = log_mgr_point_get( ); +- p_log_mgr->serial_on = on; +- return; +-} +- +-int log_init0(bool is_ut) +-{ +- char *env = NULL; +- int file_size; +- int rc; +- +- env = getenv(SXE2_LOG_EXTEND_INFO); +- if (env) { +- if (0 == strncmp(env, "true", 4)) { +- g_log_mgr.extend = true; +- } else { +- g_log_mgr.extend = false; +- } +- } +- +- env = getenv(SXE2_LOG_SERIAL); +- if (env) { +- if (0 == strncmp(env, "true", 4)) { +- g_log_mgr.serial_on = true; +- } else { +- g_log_mgr.serial_on = false; +- } +- } +- +- env = getenv(SXE2_LOG_COVER_WRITE); +- if (env) { +- if (0 == strncmp(env, "true", 4)) { +- g_log_mgr.file_cover_write_flag = true; +- } else { +- g_log_mgr.file_cover_write_flag = false; +- } +- } +- +-#if defined(SXE2_CFG_DEBUG) +- g_log_mgr.file_size = SXE2_LOG_DEFAULT_DEBUG_FILE_SIZE; +-#else +- g_log_mgr.file_size = SXE2_LOG_DEFAULT_RELEASE_FILE_SIZE; +-#endif +- +- env = getenv(SXE2_LOG_FILE_SIZE); +- if (env) { +- file_size = atoi(env); +- if (file_size > 0) { +- g_log_mgr.file_size = file_size; +- } +- } +- +- if (g_log_mgr.fd == -1) { +- if (is_ut) { +- g_log_mgr.is_ut = true; +- g_log_mgr.fd = +- open(SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } else { +- g_log_mgr.is_ut = false; +- g_log_mgr.fd = +- open(SXE2_LOG_PATH SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } +- if (g_log_mgr.fd == -1) { +- perror("open"); +- } +- } +- +- if (g_log_mgr.lockfd == -1) { +- if (is_ut) { +- g_log_mgr.lockfd = +- open(SXE2_LOG_FLOCK_NAME, O_WRONLY | O_CREAT, 0666); +- } else { +- g_log_mgr.lockfd = open(SXE2_LOG_PATH SXE2_LOG_FLOCK_NAME, +- O_WRONLY | O_CREAT, 0666); +- } +- if (g_log_mgr.fd == -1) { +- perror("open"); +- } +- } +- +- rc = pthread_mutex_init(&g_log_mutex, NULL); +- if (rc != 0) { +- perror("mutex_init"); +- } +- +- return 0; +-} +- +-int log_uninit0(void) +-{ +- if (g_log_mgr.fd != -1) { +- close(g_log_mgr.fd); +- g_log_mgr.fd = -1; +- } +- +- if (g_log_mgr.lockfd != -1) { +- close(g_log_mgr.lockfd); +- g_log_mgr.lockfd = -1; +- } +- +- pthread_mutex_destroy(&g_log_mutex); +- +- return 0; +-} +- +-static inline bool log_file_lock(int fd) +-{ +- int rc = flock(fd, LOCK_EX); +- return (bool)(rc == 0); +-} +- +-static inline bool log_file_unlock(int fd) +-{ +- int rc = flock(fd, LOCK_UN); +- return (bool)(rc == 0); +-} +- +-void log_file_name_set(char *p_name) +-{ +- struct timeval tv; +- time_t time_stamp_now; +- struct tm *p_time_now; +- long usec; +- +- gettimeofday(&tv, NULL); +- usec = tv.tv_usec; +- time_stamp_now = tv.tv_sec; +- p_time_now = localtime(&time_stamp_now); +- +- if (g_log_mgr.is_ut) { +- sprintf(p_name, +- "sxe2_rdma_file_%04d_%02d_%02d-%02d_%02d_%02d.%06ld.log", +- p_time_now->tm_year + 1900, p_time_now->tm_mon + 1, +- p_time_now->tm_mday, p_time_now->tm_hour, p_time_now->tm_min, +- p_time_now->tm_sec, usec); +- } else { +- sprintf(p_name, +- SXE2_LOG_PATH +- "sxe2_rdma_file_%04d_%02d_%02d-%02d_%02d_%02d.%06ld.log", +- p_time_now->tm_year + 1900, p_time_now->tm_mon + 1, +- p_time_now->tm_mday, p_time_now->tm_hour, p_time_now->tm_min, +- p_time_now->tm_sec, usec); +- } +- +- return; +-} +- +-void log_file_store(const char *p_name) +-{ +- char new_name[35] = {0}; +- log_file_name_set(new_name); +- (void)rename(p_name, new_name); +- return; +-} +- +-void log_file_not_cover_write(void) +-{ +- int new_fd = -1; +- int old_fd = g_log_mgr.fd; +- struct stat st; +- if (fstat(old_fd, &st) < 0 || (st.st_size >> 20) < g_log_mgr.file_size) { +- goto end; +- } +- +- if (g_log_mgr.is_ut) { +- new_fd = open(SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } else { +- new_fd = open(SXE2_LOG_PATH SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } +- if (new_fd < 0) { +- goto end; +- } +- +- if (fstat(new_fd, &st) < 0 || (st.st_size >> 20) < g_log_mgr.file_size) { +- g_log_mgr.fd = new_fd; +- close(old_fd); +- goto end; +- } +- close(new_fd); +- +- if (g_log_mgr.is_ut) { +- log_file_store(SXE2_LOG_FILE_NAME); +- } else { +- log_file_store(SXE2_LOG_PATH SXE2_LOG_FILE_NAME); +- } +- +- if (g_log_mgr.is_ut) { +- new_fd = open(SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } else { +- new_fd = open(SXE2_LOG_PATH SXE2_LOG_FILE_NAME, +- O_APPEND | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } +- if (new_fd < 0) { +- goto end; +- } +- g_log_mgr.fd = new_fd; +- +- close(old_fd); +-end: +- return; +-} +- +-void log_file_cover_write(void) +-{ +- int old_fd = -1, new_fd = -1; +- old_fd = g_log_mgr.fd; +- struct stat st; +- if (fstat(g_log_mgr.fd, &st) < 0 || +- (st.st_size >> 20) < g_log_mgr.file_size) { +- goto end; +- } +- +- if (g_log_mgr.is_ut) { +- new_fd = +- open(SXE2_LOG_FILE_NAME, +- O_APPEND | O_TRUNC | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } else { +- new_fd = +- open(SXE2_LOG_PATH SXE2_LOG_FILE_NAME, +- O_APPEND | O_TRUNC | O_WRONLY | O_CREAT | O_NONBLOCK, 0666); +- } +- if (new_fd < 0) { +- goto end; +- } +- g_log_mgr.fd = new_fd; +- close(old_fd); +-end: +- return; +-} +- +-int log_file_rotate(void) +-{ +- int ret = 0; +- __u64 fileSize; +- struct stat st; +- sigset_t mask, oldmask; +- if (fstat(g_log_mgr.fd, &st) == -1) { +- perror("fstat"); +- ret = -1; +- goto end; +- } +- +- fileSize = st.st_size >> 20; +- if (fileSize >= (__u64)g_log_mgr.file_size) { +- if (sigfillset(&mask) == -1) { +- perror("sigfillset"); +- ret = -1; +- goto end; +- } +- +- if (pthread_sigmask(SIG_BLOCK, &mask, &oldmask) != 0) { +- perror("pthread_sigmask"); +- ret = -1; +- goto end; +- } +- +- log_file_lock(g_log_mgr.lockfd); +- +- if (g_log_mgr.file_cover_write_flag) { +- log_file_cover_write( ); +- } else { +- log_file_not_cover_write( ); +- } +- +- if (g_log_mgr.fd < 0) { +- ret = -1; +- } +- +- log_file_unlock(g_log_mgr.lockfd); +- +- if (pthread_sigmask(SIG_SETMASK, &oldmask, NULL) != 0) { +- perror("pthread_sigmask"); +- ret = -1; +- } +- } +-end: +- return ret; +-} +- +-__u64 sxe2_current_time_us_get(void) +-{ +- struct timeval tv = {0}; +- gettimeofday(&tv, NULL); +- return (SXE2_SEC2US(tv.tv_sec) + tv.tv_usec); +-} +- +-__u32 log_time_to_str(char *p_buf, __u32 buf_size, struct timeval *p_time) +-{ +- struct tm time; +- __u32 use_len; +- +- (void)localtime_r(&p_time->tv_sec, &time); +- +- use_len = (__u32)strftime(p_buf, buf_size, "[%Y/%m/%d.%H:%M:%S", &time); +- use_len += snprintf(p_buf + use_len, buf_size - use_len, ".%06lu]", +- p_time->tv_usec); +- +- return use_len; +-} +- +-__u32 log_format_prefix(FILE *p_stream, char *p_buf, __u32 buf_size, +- struct timeval *p_time_stamp, const char *bdf, +- __u32 level, const char *p_func_name, __u32 line) +-{ +- static const char *log_level_str[LOG_LEVEL_MAX] = { +- "INVALID", "FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE", +- }; +- __u32 use_len = 0; +- int fd; +- if (p_stream == NULL) { +- fd = g_log_mgr.fd; +- } else { +- fd = fileno(p_stream); +- } +- +- use_len = log_time_to_str(p_buf, buf_size, p_time_stamp); +- if (g_log_mgr.extend) { +- use_len += snprintf(p_buf + use_len, buf_size - use_len, "[%s][%d][%d]", +- log_level_str[level], (int)getpid( ), fd); +- } else { +- use_len += snprintf(p_buf + use_len, buf_size - use_len, "[%s]", +- log_level_str[level]); +- } +- +- if (NULL != bdf) { +- use_len += snprintf(p_buf + use_len, buf_size - use_len, "[%s][%s:%u]", +- bdf, p_func_name, line); +- } else { +- use_len += snprintf(p_buf + use_len, buf_size - use_len, "[%s:%u]", +- p_func_name, line); +- } +- +- return use_len; +-} +- +-__u32 log_buff_fill(FILE *p_stream, char *p_buff, __u32 buff_size, +- struct timeval *p_time_stamp, const char *bdf, __u32 level, +- const char *p_func_name, __u32 line, const char *p_format, +- va_list va_list) +-{ +- log_hdr_s *p_log_hdr = NULL; +- char *p_data_buf; +- __u32 log_data_cap; +- __u32 use_len; +- +- p_log_hdr = (log_hdr_s *)p_buff; +- p_data_buf = p_buff + sizeof(log_hdr_s); +- log_data_cap = +- buff_size - sizeof(log_hdr_s) - 1; +- +- p_log_hdr->ts = SXE2_SEC2US(p_time_stamp->tv_sec) + p_time_stamp->tv_usec; +- p_log_hdr->magic = LOG_MAGIC; +- +- use_len = log_format_prefix(p_stream, p_data_buf, log_data_cap, +- p_time_stamp, bdf, level, p_func_name, line); +- use_len += vsnprintf(p_data_buf + use_len, log_data_cap - use_len, p_format, +- va_list); +- use_len = use_len > log_data_cap - 1 ? log_data_cap - 1 : use_len; +- if (p_data_buf[use_len - 1] != '\n') { +- use_len += sprintf(p_data_buf + use_len, "\n"); +- } +- p_log_hdr->len = (__u16)use_len; +- +- return use_len; +-} +- +-void log_store(FILE *p_stream, const char *bdf, log_level_e level, +- const char *p_file, const char *p_func_name, __u32 line, +- const char *p_format, ...) +-{ +- struct timeval time_stamp; +- __u64 us_now; +- char *local_log_buf = malloc(LOG_LINE_MAX); +- __u32 use_len; +- va_list va_list; +- ssize_t ret = 0; +- char *p_data_buf = local_log_buf + sizeof(log_hdr_s); +- int fd; +- +- if ((p_func_name == NULL) || (p_format == NULL)) { +- goto end; +- } +- +- us_now = sxe2_current_time_us_get( ); +- time_stamp.tv_sec = (__time_t)SXE2_US2SEC(us_now); +- time_stamp.tv_usec = (__suseconds_t)(us_now - SXE2_SEC2US(time_stamp.tv_sec)); +- va_start(va_list, p_format); +- use_len = log_buff_fill(p_stream, local_log_buf, LOG_LINE_MAX, &time_stamp, +- bdf, level, p_func_name, line, p_format, va_list); +- va_end(va_list); +- +- if (NULL == p_stream) { +- if (g_log_mgr.fd == -1) { +- goto err; +- } +- +- if (log_file_rotate( ) < 0) { +- goto err; +- } +- +- ret = write(g_log_mgr.fd, p_data_buf, use_len); +- if (ret < 1) { +- perror("write"); +- } +- } else { +- fd = fileno(p_stream); +- ret = write(fd, p_data_buf, use_len); +- if (ret < 1) { +- perror("write"); +- } +- } +- +-err: +- if (g_log_mgr.serial_on) { +- printf("%s", p_data_buf); +- } +-end: +- (void)p_file; +- free(local_log_buf); +- return; +-} +diff -Naur rdma-core-48.0/providers/sxe2/log.h rdma-core-48.0.bak/providers/sxe2/log.h +--- rdma-core-48.0/providers/sxe2/log.h 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/log.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,245 +0,0 @@ +- +-#ifndef _SXE2_LOG_H_ +-#define _SXE2_LOG_H_ +- +-#include +-#include +-#include +-#include +- +-#define SXE2_LOG_FD_NULL NULL +-#define SXE2_LOG_FILE_NULL NULL +-#define LOG_LINE_MAX (2048) +-#define LOG_MAGIC (0xBAAB) +- +-#define SXE2_SEC2US(_sec) ((_sec)*1000000ULL) +-#define SXE2_US2SEC(_us) ((_us)/1000000ULL) +- +-#define SXE2_LOG_IOLOG_ON "SXE2_LOG_IOLOG_SWITCH" +-#define SXE2_VERBS_LOG_LEVEL "SXE2_RDMA_LOG_LEVEL" +-#define SXE2_INVALID_BDF "ff:ff.f" +- +-typedef struct log_hdr{ +- __u64 ts; +- __u32 pad; +- __u16 len; +- __u16 magic; +-}log_hdr_s; +- +-typedef enum log_level { +- LOG_LEVEL_INVALID, +- LOG_LEVEL_FATAL, +- LOG_LEVEL_ERROR, +- LOG_LEVEL_WARN, +- LOG_LEVEL_INFO, +- LOG_LEVEL_DEBUG, +- LOG_LEVEL_TRACE, +- LOG_LEVEL_MAX, +-} log_level_e; +- +-typedef struct log_persist_ops { +- void (*pre_persist)(void *persist_mgr); +- bool (*checkPersistSpace)(void *persist_mgr, __u32 persist_threshold); +- int (*flushInBlock)(void *persist_mgr, __u8 *buf, __u32 len); +- int (*flushOne)(void *persist_mgr, __u8 *buf, __u32 len); +- void (*postPersist)(void *persist_mgr); +- void (*printPersistInfo)(void *persist_mgr); +- int (*forcedExport)(void *persist_mgr); +- int (*forcedSync)(void *persist_mgr); +-}log_persist_ops_s; +- +-typedef struct log_mgr { +- bool serial_on; +- log_persist_ops_s *persist_ops; +- int file_size; +- bool file_cover_write_flag; +- bool extend; +- int fd; +- int lockfd; +- bool is_ut; +-} log_mgr_s; +- +-log_mgr_s *log_mgr_point_get(void); +- +-int log_init0(bool is_ut); +- +-int log_uninit0(void); +- +-void log_set_serial_switch(bool on); +- +-void log_file_name_set(char *p_name); +- +-void log_file_store(const char *p_name); +- +-void log_file_not_cover_write(void); +- +-void log_file_cover_write(void); +- +-int log_file_rotate(void); +- +-__u64 sxe2_current_time_us_get(void); +- +-__u32 log_time_to_str(char *p_buf, __u32 buf_size, struct timeval *p_time); +- +-__u32 log_format_prefix(FILE *p_stream, char *p_buf, __u32 buf_size, +- struct timeval *p_time_stamp, const char *bdf, +- __u32 level, const char *p_func_name, __u32 line); +- +-__u32 log_buff_fill(FILE *p_stream, char *p_buff, __u32 buff_size, +- struct timeval *p_time_stamp, const char *bdf, __u32 level, +- const char *p_func_name, __u32 line, const char *p_format, +- va_list va_list); +- +-void log_store(FILE *p_stream, const char *bdf, +- log_level_e level, const char *p_file, const char *p_func_name, __u32 line, +- const char *p_format, ...) __attribute__((format(printf, 7, 8))); +- +-#define SXE2_LOG_STORE(p_stream, bdf, level, fmt, ...) \ +- do { \ +- log_store(p_stream, bdf, level, \ +- (strrchr(__FILE__, '/') ? (strrchr(__FILE__, '/') + 1) \ +- : __FILE__), \ +- __func__, __LINE__, fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_LOG_DEV_INVALID(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_INVALID, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_FATAL(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_FATAL, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_ERROR(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_ERROR, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_WARN(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_WARN, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_INFO(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_INFO, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_DEBUG(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) +- +-#define SXE2_LOG_DEV_TRACE(p_stream, bdf, fmt, ...) \ +- SXE2_LOG_STORE(p_stream, bdf, LOG_LEVEL_TRACE, fmt, ##__VA_ARGS__) +- +-#define SXE2_VERBS_LOG_FATAL(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_ERROR(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_WARN(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_INFO(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_DEBUG(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +-#define SXE2_VERBS_LOG_TRACE(fmt, ...) \ +- do { \ +- (void)printf("%s: " fmt, __func__, ##__VA_ARGS__); \ +- (void)printf("\n"); \ +- } while (0) +- +-#if defined SXE2_CFG_DEBUG +- +-#define SXE2_VERBS_LOG_FATAL_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_FATAL) { \ +- SXE2_LOG_DEV_FATAL(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#define SXE2_VERBS_LOG_ERROR_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_ERROR) { \ +- SXE2_LOG_DEV_ERROR(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#define SXE2_VERBS_LOG_WARN_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_WARN) { \ +- SXE2_LOG_DEV_WARN(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#define SXE2_VERBS_LOG_INFO_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_INFO) { \ +- SXE2_LOG_DEV_INFO(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#define SXE2_VERBS_LOG_DEBUG_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_DEBUG) { \ +- SXE2_LOG_DEV_DEBUG(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#define SXE2_VERBS_LOG_TRACE_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_TRACE) { \ +- SXE2_LOG_DEV_TRACE(sctx->dbg_fp, sctx->bdf, fmt, ##__VA_ARGS__); \ +- } \ +- } while (0) +- +-#else +- +-#define SXE2_VERBS_LOG_FATAL_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_FATAL) \ +- SXE2_VERBS_LOG_FATAL(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_ERROR_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_ERROR) \ +- SXE2_VERBS_LOG_ERROR(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_WARN_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_WARN) \ +- SXE2_VERBS_LOG_WARN(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_INFO_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_INFO) \ +- SXE2_VERBS_LOG_INFO(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_DEBUG_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_DEBUG) \ +- SXE2_VERBS_LOG_DEBUG(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#define SXE2_VERBS_LOG_TRACE_BDF(fmt, ...) \ +- do { \ +- if (sctx && sctx->log_level >= LOG_LEVEL_TRACE) \ +- SXE2_VERBS_LOG_TRACE(fmt, ##__VA_ARGS__); \ +- } while (0) +- +-#endif +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/Makefile rdma-core-48.0.bak/providers/sxe2/Makefile +--- rdma-core-48.0/providers/sxe2/Makefile 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/Makefile 1970-01-01 08:00:00.000000000 +0800 +@@ -1,80 +0,0 @@ +-################################################################################ +-# Copyright (C) +-################################################################################ +-#(1) 固定模式. 定义路径信息 +-################################################################################ +-#注: MD_TOPDIR在Makefile.define将被使用, 因此需要预先定义 +-#考虑到内核模块编译, Makefile的调用路径与存储路径不一致, 因此不可以$(shell pwd)方式获取路径. +-TOPDIR_MP :=. +-MAKEPATH := $(abspath $(lastword $(MAKEFILE_LIST))) +-CURDIR := $(shell dirname $(MAKEPATH)) +-TOPDIR := $(abspath $(CURDIR)/$(TOPDIR_MP)) +-MD_TOPDIR := $(TOPDIR) +-MT_TOPDIR := $(TOPDIR) +- +-################################################################################# +-#(2) 自定义模式 +-################################################################################# +- +-###################################### +-#(2.1) 定义模块基本信息 +-####################################### +- +-SRCS := $(wildcard *.c) +-OBJS := $(patsubst %.c,%.o,$(SRCS)) +-OBJ_S := libsxe2-rdmav34.a +-#CMR_INCS := -I$(INCS_ARCH) +-#CMR_INCS += -I$(INCS_ARCH)/cmr/ +-CMR_INCS += -I/usr/include/libnl3 +-CMR_INCS += -I/usr/include/drm +-CMR_INCS += -I$(TOPDIR)/../rdma-core-46.0/build/include/ +-LIBS_DIR := $(TOPDIR)/../rdma-core-46.0/build/lib/libibverbs.so +-CCAN_DIR := $(TOPDIR)/../rdma-core-46.0/build/ccan/libccan_pic.a +-UTIL_DIR := $(TOPDIR)/../rdma-core-46.0/build/util/librdma_util_pic.a +-CMR_LIBS := -lpthread +-#LIBS_DIR += $(TOPDIR_MP)/lib/lib_shca_libbase_x86_64.a +- +-CFLAGS := -g -ggdb #支持调试信息 +-CFLAGS += -Wall +-#CFLAGS += -Werror #打开gcc所有告警选项，并将告警当作错误信息 +-CFLAGS += -Wextra #打开gcc额外告警选项 +-CFLAGS += -frecord-gcc-switches #后续可通过命令“readelf -p .GCC.command.line”查看编译选项参数 +-CFLAGS += -fprofile-arcs -ftest-coverage #启用gcover统计代码覆盖率 +- +-VERBS_CFLAGS := $(CFLAGS) #verbs使用的cflags +-VERBS_CFLAGS += -std=gnu11 #使用gnu11编译 +-VERBS_CFLAGS += -Wno-sign-compare #临时关闭 +-VERBS_CFLAGS += -Wno-unused-parameter #临时关闭 +-VERBS_CFLAGS += -Wno-missing-field-initializers #临时关闭 +-####################################### +-#(2.2) 定义执行目标 +-####################################### +- +-all:$(OBJS) +- ar -cr -o $(OBJ_S) $^ +- ranlib $(OBJ_S) +- +-$(OBJS):%.o:%.c +- gcc $(VERBS_CFLAGS) $(CMR_INCS) -c -o $@ $< $(CMR_LIBS) +- +-clean: +- @rm -rf *.o ut *.gcno *.gcda *.a +- @rm -rf $(TOPDIR)/rdma-core-46.0/build/* +- @rm -rf $(TOPDIR)/bin/verbs/* +-# @-$(MAKE) clean -C $(TOPDIR)/lib +- @rm -rf logfile.txt result.txt +- @echo all files made removed +- +-#makefile帮助文档 +-help: +- @echo Usage: +- @echo "The following are some of the valid targets for this Makefile:" +- @echo " all: Compile the whole project" +- @echo " clean: Clean Compile file" +- @echo "" +-.PHONY:all clean help liba +-%: +- @-$(warning MAKE target needed, please get help from command 'make help') +- @$(MAKE) help +- +-.DEFAULT_GOAL=all +diff -Naur rdma-core-48.0/providers/sxe2/mc.c rdma-core-48.0.bak/providers/sxe2/mc.c +--- rdma-core-48.0/providers/sxe2/mc.c 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/mc.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,13 +0,0 @@ +- +-#include "verbs.h" +-#include "sxe2_common.h" +- +-int sxe2_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, __u16 lid) +-{ +- return ibv_cmd_attach_mcast(qp, gid, lid); +-} +- +-int sxe2_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, __u16 lid) +-{ +- return ibv_cmd_detach_mcast(qp, gid, lid); +-} +diff -Naur rdma-core-48.0/providers/sxe2/mr.c rdma-core-48.0.bak/providers/sxe2/mr.c +--- rdma-core-48.0/providers/sxe2/mr.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/mr.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,98 +0,0 @@ +- +-#include +-#include +- +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "log.h" +- +-struct ibv_mr *sxe2_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, +- uint64_t hca_va, int acc) +-{ +- struct sxe2_umr *mr; +- struct ibv_reg_mr cmd; +- int ret; +- struct ib_uverbs_reg_mr_resp resp; +- struct sxe2_rdma_ucontext *sctx = to_sctx(pd->context); +- +- mr = calloc(1, sizeof(*mr)); +- if (!mr) { +- SXE2_VERBS_LOG_ERROR_BDF("user alloc mr mem failed\n"); +- return NULL; +- } +- +- ret = ibv_cmd_reg_mr(pd, addr, length, hca_va, acc, &mr->vmr, &cmd, +- sizeof(cmd), &resp, sizeof(resp)); +- if (ret) { +- free(mr); +- errno = ret; +- SXE2_VERBS_LOG_ERROR_BDF("ibv cmd reg mr err(%d)\n", ret); +- return NULL; +- } +- mr->acc_flags = acc; +- SXE2_VERBS_LOG_DEBUG_BDF("addr:%#lx, length:%zu, hca_va:%#lx,acc:%#x,handle:%#x, lkey:%#x, rkey:%#x", +- (uint64_t)addr, length, hca_va, acc, mr->vmr.ibv_mr.handle, +- mr->vmr.ibv_mr.lkey, mr->vmr.ibv_mr.rkey); +- +- return &mr->vmr.ibv_mr; +-} +- +-int sxe2_udereg_mr(struct verbs_mr *vmr) +-{ +- int ret; +- +- ret = ibv_cmd_dereg_mr(vmr); +- if (ret) { +- SXE2_VERBS_LOG_ERROR("ibv cmd dereg mr err(%d)\n", ret); +- return ret; +- } +- +- free(vmr); +- return 0; +-} +- +-int sxe2_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, +- void *addr, size_t length, int access) +-{ +- int ret; +- struct ibv_rereg_mr cmd; +- struct ib_uverbs_rereg_mr_resp resp; +- +- ret = ibv_cmd_rereg_mr(vmr, flags, addr, length, (uintptr_t)addr, +- access, pd, &cmd, sizeof(cmd), &resp, +- sizeof(resp)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR("ibv cmd rereg mr err(%d)\n", ret); +- } +- SXE2_VERBS_LOG_DEBUG("addr:%#lx, length:%zu, acc:%#x,handle:%#x, lkey:%#x, rkey:%#x, flags:%#x", +- (uint64_t)addr, length, access,vmr->ibv_mr.handle, vmr->ibv_mr.lkey, +- vmr->ibv_mr.rkey, flags); +- return ret; +-} +- +-struct ibv_mr *sxe2_ureg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, +- size_t length, uint64_t iova, int fd, +- int acc) +-{ +- struct sxe2_umr *mr; +- int ret; +- struct sxe2_rdma_ucontext *sctx = to_sctx(pd->context); +- +- mr = calloc(1, sizeof(*mr)); +- if (!mr) { +- SXE2_VERBS_LOG_ERROR_BDF("user alloc dma buf mr failed\n"); +- return NULL; +- } +- +- ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, acc, +- &mr->vmr); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv cmd reg dma buf mr err(%d)\n", ret); +- free(mr); +- errno = ret; +- return NULL; +- } +- mr->acc_flags = acc; +- +- return &mr->vmr.ibv_mr; +-} +diff -Naur rdma-core-48.0/providers/sxe2/pd.c rdma-core-48.0.bak/providers/sxe2/pd.c +--- rdma-core-48.0/providers/sxe2/pd.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/pd.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,87 +0,0 @@ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include "ccan/array_size.h" +-#include "util/compiler.h" +-#include "util/mmio.h" +-#include "util/util.h" +-#include +-#include +-#include "sxe2_common.h" +-#include "log.h" +- +-struct ibv_pd *sxe2_ualloc_pd(struct ibv_context *context) +-{ +- struct ibv_alloc_pd cmd; +- struct sxe2_ualloc_pd_resp resp; +- struct sxe2_upd *pd; +- struct ibv_pd *ibv_pd = NULL; +- struct sxe2_rdma_ucontext *sctx; +- int ret; +- +- memset(&resp, 0, sizeof(resp)); +- if (context == NULL) { +- SXE2_VERBS_LOG_ERROR("user alloc pd context err\n"); +- errno = EINVAL; +- goto end; +- } +- +- sctx = container_of(context, struct sxe2_rdma_ucontext, ibv_ctx.context); +- +- pd = calloc(1, sizeof(*pd)); +- if (!pd) { +- SXE2_VERBS_LOG_ERROR_BDF("user pd mem alloc failed\n"); +- goto end; +- } +- +- ret = ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv cmd alloc pd err(%d)\n", ret); +- free(pd); +- errno = ret; +- goto end; +- } +- +- pd->pd_id = resp.pd_id; +- ibv_pd = &pd->ibv_pd; +- +- SXE2_VERBS_LOG_DEBUG_BDF("user pd alloc resp.pdn:%u\n", pd->pd_id); +- +-end: +- return ibv_pd; +-} +- +-int sxe2_ufree_pd(struct ibv_pd *pd) +-{ +- int ret; +- struct sxe2_upd *upd; +- +- if (pd == NULL) { +- SXE2_VERBS_LOG_ERROR("user free pd is null\n"); +- ret = EINVAL; +- goto end; +- } +- +- upd = container_of(pd, struct sxe2_upd, ibv_pd); +- ret = ibv_cmd_dealloc_pd(pd); +- if (ret) { +- SXE2_VERBS_LOG_ERROR("ibv cmd dealloc pd err(%d)\n", ret); +- goto end; +- } +- free(upd); +-end: +- return ret; +-} +- +diff -Naur rdma-core-48.0/providers/sxe2/qp.c rdma-core-48.0.bak/providers/sxe2/qp.c +--- rdma-core-48.0/providers/sxe2/qp.c 2026-05-26 10:42:01.855075215 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/qp.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,782 +0,0 @@ +-#include +-#include +- +-#include "log.h" +-#include "verbs.h" +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "io.h" +-#include "ring.h" +-#include "buf.h" +- +-#define MIN_SQ_SIZE (8u) +-#define MIN_RQ_SIZE (8u) +-#define SQ_RSV_SIZE (8) +-#define RQ_RSV_SIZE (8) +-#define QP_QUANTA_SIZE (32) +-#define QP_DB_NOTE_SIZE (16) +-#define QP_MEMALIGN_1K (1024) +- +-enum { +- SXE2_SUPPORTED_SEND_OPS_FLAGS_RC = +- IBV_QP_EX_WITH_SEND | +- IBV_QP_EX_WITH_SEND_WITH_INV | +- IBV_QP_EX_WITH_SEND_WITH_IMM | +- IBV_QP_EX_WITH_RDMA_WRITE | +- IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM | +- IBV_QP_EX_WITH_RDMA_READ | +- IBV_QP_EX_WITH_LOCAL_INV | +- IBV_QP_EX_WITH_BIND_MW, +- SXE2_SUPPORTED_SEND_OPS_FLAGS_UD = +- IBV_QP_EX_WITH_SEND | +- IBV_QP_EX_WITH_SEND_WITH_IMM, +-}; +- +-static void sxe2_calc_sq_wqe_shift(__u32 max_sq_sge, __u32 max_inline_data, +- __u8 *wqe_shift) +-{ +- *wqe_shift = WQE_SIZE_32BYTE; +- +- if (max_sq_sge > MAX_SGE_SIZE_1 || max_inline_data > MAX_INLINE_DATA_8) { +- if (max_sq_sge < MAX_SGE_SIZE_4 && max_inline_data <= MAX_INLINE_DATA_39) { +- *wqe_shift = WQE_SIZE_64BYTE; +- } else if (max_sq_sge < MAX_SGE_SIZE_8 && max_inline_data <= MAX_INLINE_DATA_101) { +- *wqe_shift = WQE_SIZE_128BYTE; +- } else { +- *wqe_shift = WQE_SIZE_256BYTE; +- } +- } +-} +-static void sxe2_calc_rq_wqe_shift(__u32 max_rq_sge, __u8 *wqe_shift) +-{ +- *wqe_shift = WQE_SIZE_32BYTE; +- +- if (max_rq_sge > MAX_SGE_SIZE_1) { +- if (max_rq_sge < MAX_SGE_SIZE_4) { +- *wqe_shift = WQE_SIZE_64BYTE; +- } else if (max_rq_sge < MAX_SGE_SIZE_8) { +- *wqe_shift = WQE_SIZE_128BYTE; +- } else { +- *wqe_shift = WQE_SIZE_256BYTE; +- } +- } +-} +-static int sxe2_calc_sqdepth(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_common_attrs *common_attrs, +- __u32 max_wr, __u8 wqe_shift, +- __u32 *sq_depth) +-{ +- int ret = 0; +- __u32 depth; +- +- depth = sxe2_round_up_pow_2((max_wr << wqe_shift) + SQ_RSV_SIZE); +- +- depth = max(depth, MIN_SQ_SIZE); +- *sq_depth = depth; +- if (depth > common_attrs->max_hw_wq_quanta) { +- SXE2_VERBS_LOG_ERROR_BDF("sq depth illegal, sq depth(%u)\n", depth); +- ret = EINVAL; +- } +- +- return ret; +-} +-static int sxe2_calc_rqdepth(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_common_attrs *common_attrs, +- __u32 max_wr, __u8 wqe_shift, +- __u32 *rq_depth) +-{ +- int ret = 0; +- __u32 depth; +- depth = sxe2_round_up_pow_2((max_wr << wqe_shift) + RQ_RSV_SIZE); +- +- depth = max(depth, MIN_RQ_SIZE); +- *rq_depth = depth; +- if (depth > common_attrs->max_hw_rq_quanta) { +- SXE2_VERBS_LOG_ERROR_BDF("rq depth illegal, rq depth(%u)\n", depth); +- ret = EINVAL; +- } +- return ret; +-} +-static int sxe2_calc_sq_depth_shift(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_qp_common_init_info *init_info, +- __u32 *sq_depth, __u8 *sq_shift) +-{ +- int ret; +- +- sxe2_calc_sq_wqe_shift(init_info->max_sq_sge_cnt + 1, init_info->max_inline_data, +- sq_shift); +- ret = sxe2_calc_sqdepth(sctx, init_info->common_attrs, init_info->sq_size, +- *sq_shift, sq_depth); +- return ret; +-} +- +-static int sxe2_calc_rq_depth_shift(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_qp_common_init_info *init_info, +- __u32 *rq_depth, __u8 *rq_shift) +-{ +- int ret; +- +- if ((init_info->srq) || (init_info->rq_size == 0)) { +- *rq_depth = 0; +- *rq_shift = 0; +- ret = 0; +- goto end; +- } +- +- sxe2_calc_rq_wqe_shift(init_info->max_rq_sge_cnt, rq_shift); +- ret = sxe2_calc_rqdepth(sctx, init_info->common_attrs, init_info->rq_size, +- *rq_shift, rq_depth); +-end: +- return ret; +-} +- +-static void sxe2_fill_qp(struct sxe2_uqp *uqp, +- struct sxe2_qp_common_init_info *init_info, +- struct sxe2_ucreate_qp_resp resp, +- struct ibv_qp_init_attr_ex *attr_ex) +-{ +- __u32 sq_ring_size; +- uqp->sq_sig_all = (__u32)attr_ex->sq_sig_all; +- uqp->qp_type = attr_ex->qp_type; +- uqp->qp.back_qp = uqp; +- uqp->qp.lock = &uqp->lock; +- uqp->verbs_qp.qp.qp_num = resp.qpn; +- uqp->send_cq = container_of(attr_ex->send_cq, struct sxe2_ucq, verbs_cq.cq); +- uqp->recv_cq = container_of(attr_ex->recv_cq, struct sxe2_ucq, verbs_cq.cq); +- uqp->send_cq->uqp = uqp; +- uqp->recv_cq->uqp = uqp; +- +- uqp->qp.common_attrs = init_info->common_attrs; +- uqp->qp.sq_base = init_info->sq; +- uqp->qp.rq_base = init_info->rq; +- uqp->qp.srq = init_info->srq ? init_info->srq : NULL; +- uqp->qp.doorbell_note = init_info->doorbell_note; +- uqp->qp.sq_wrtrk_array = init_info->sq_wrtrk_array; +- uqp->qp.rq_wrid_array = init_info->rq_wrid_array; +- uqp->qp.qp_db_no_llwqe = init_info->qp_db_no_llwqe; +- uqp->qp.qp_caps = resp.qp_caps; +- uqp->qp.qpn = resp.qpn; +- uqp->qp.llwqe_mode = false; +- uqp->qp.max_sq_sge_cnt = init_info->max_sq_sge_cnt; +- uqp->qp.max_rq_sge_cnt = init_info->max_rq_sge_cnt; +- uqp->qp.sq_size = init_info->sq_size; +- uqp->qp.rq_size = init_info->rq_size; +- sq_ring_size = uqp->qp.sq_size << init_info->sq_shift; +- SXE2_RING_INIT(uqp->qp.sq_ring, sq_ring_size); +- SXE2_RING_INIT(uqp->qp.initial_ring, sq_ring_size); +- uqp->qp.swqe_polarity = 0; +- uqp->qp.rwqe_polarity = 0; +- uqp->qp.max_inline_data = init_info->max_inline_data; +- uqp->qp.rq_wqe_size = init_info->rq_shift; +- SXE2_RING_INIT(uqp->qp.rq_ring, uqp->qp.rq_size); +- uqp->qp.rq_wqe_size_multiplier = init_info->srq ? 0 : 1 << init_info->rq_shift; +-} +- +-static void sxe2_fill_wr_builders_rc_xrc_dc(struct ibv_qp_ex *ibqp) +-{ +- ibqp->wr_send = sxe2_wr_send_rc_ud; +- ibqp->wr_send_imm = sxe2_wr_send_imm_rc_ud; +- ibqp->wr_send_inv = sxe2_wr_send_inv_rc; +- ibqp->wr_rdma_write = sxe2_wr_rdma_write_rc; +- ibqp->wr_rdma_write_imm = sxe2_wr_rdma_write_imm_rc; +- ibqp->wr_rdma_read = sxe2_wr_rdma_read_rc; +- ibqp->wr_bind_mw = sxe2_wr_bind_mw_rc; +- ibqp->wr_local_inv = sxe2_wr_local_inv_rc; +-} +- +-static void sxe2_fill_wr_setters_rc_ud(struct ibv_qp_ex *ibqp) +-{ +- ibqp->wr_set_sge = sxe2_wr_set_sge_rc_ud; +- ibqp->wr_set_sge_list = sxe2_wr_set_sge_list_rc_ud; +- ibqp->wr_set_inline_data = sxe2_wr_set_inline_data_rc_ud; +- ibqp->wr_set_inline_data_list = sxe2_wr_set_inline_data_list_rc_ud; +-} +- +-static void sxe2_fill_wr_builders_ud(struct ibv_qp_ex *ibqp) +-{ +- ibqp->wr_send = sxe2_wr_send_rc_ud; +- ibqp->wr_send_imm = sxe2_wr_send_imm_rc_ud; +-} +- +-static int sxe2_qp_fill_wr_func(struct sxe2_rdma_ucontext *sctx, struct sxe2_uqp *uqp, +- const struct ibv_qp_init_attr_ex *attr) +-{ +- struct ibv_qp_ex *ibqp = &uqp->verbs_qp.qp_ex; +- uint64_t ib_ops = attr->send_ops_flags; +- int ret = 0; +- +- SXE2_VERBS_LOG_DEBUG_BDF("qp_type:%u, ops:%#llx\n", attr->qp_type, (__u64)ib_ops); +- +- ibqp->wr_start = sxe2_wr_start; +- ibqp->wr_complete = sxe2_wr_complete; +- ibqp->wr_abort = sxe2_wr_abort; +- +- if (ib_ops & IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP || +- ib_ops & IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD) { +- ret = EOPNOTSUPP; +- SXE2_VERBS_LOG_ERROR_BDF("atomics not support\n"); +- goto end; +- } +- +- switch (attr->qp_type) { +- case IBV_QPT_RC: +- if (ib_ops & ~SXE2_SUPPORTED_SEND_OPS_FLAGS_RC) { +- SXE2_VERBS_LOG_ERROR_BDF("not support ops:%#llx", (__u64)ib_ops); +- ret = EOPNOTSUPP; +- goto end; +- } +- +- sxe2_fill_wr_builders_rc_xrc_dc(ibqp); +- sxe2_fill_wr_setters_rc_ud(ibqp); +- +- break; +- +- case IBV_QPT_UD: +- if (ib_ops & ~SXE2_SUPPORTED_SEND_OPS_FLAGS_UD) { +- SXE2_VERBS_LOG_ERROR_BDF("not support ops:%#llx", (__u64)ib_ops); +- ret = EOPNOTSUPP; +- goto end; +- } +- +- sxe2_fill_wr_builders_ud(ibqp); +- sxe2_fill_wr_setters_rc_ud(ibqp); +- ibqp->wr_set_ud_addr = sxe2_wr_set_ud_addr; +- break; +- +- default: +- ret = EOPNOTSUPP; +- break; +- } +- +-end: +- return ret; +-} +- +-static const char *sxe2_qptype2key(enum ibv_qp_type type) +-{ +- const char *key = "HUGE_NA"; +- +- switch (type) { +- case IBV_QPT_RC: +- key = "HUGE_RC"; +- break; +- case IBV_QPT_UD: +- key = "HUGE_UD"; +- break; +- default: +- key = "HUGE_NA"; +- } +- +- return key; +-} +- +-static bool sxe2_is_huge_buf(const char *key) +-{ +- bool ret = false; +- char *env = getenv(key); +- +- if (env && (!strncmp(env, "y", strlen(env)))) { +- ret = true; +- } +- +- return ret; +-} +-static int sxe2_alloc_qp_buf(struct sxe2_rdma_ucontext *sctx, sxe2_buf_s *buf, size_t size, +- enum ibv_qp_type qp_type) +-{ +- int ret; +- sxe2_alloc_type_e type; +- sxe2_alloc_type_e default_type = SXE2_ALLOC_TYPE_ANON; +- const char *qp_huge_key; +- long page_size = 0; +- +- qp_huge_key = sxe2_qptype2key(qp_type); +- SXE2_VERBS_LOG_DEBUG_BDF("in sxe2_alloc_qp_buf qp_huge_key(%s)", qp_huge_key); +- if (sxe2_is_huge_buf(qp_huge_key)) { +- default_type = SXE2_ALLOC_TYPE_HUGE; +- } +- +- sxe2_alloc_type_get(SXE2_QP_PREFIX, &type, default_type); +- +- page_size = sysconf(_SC_PAGE_SIZE); +- if (page_size < 0) { +- SXE2_VERBS_LOG_ERROR_BDF("get system page size failed."); +- ret = EPERM; +- goto end; +- } +- ret = sxe2_prefered_buf_alloc( +- sctx, buf, size, (size_t)page_size, type); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("qp buf alloc err ret (%d)", ret); +- goto end; +- } +- +- memset(buf->buf, 0, buf->length); +- +-end: +- return ret; +-} +- +-static void sxe2_free_qp_buf(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf) +-{ +- sxe2_actual_buf_free(ctx, buf); +-} +- +-static struct ibv_qp *ucreate_qp(struct ibv_context *context, struct ibv_qp_init_attr_ex *attr_ex) +-{ +- struct sxe2_qp_common_init_info init_info = {}; +- struct sxe2_ucreate_qp cmd = {}; +- struct sxe2_ucreate_qp_resp resp = {}; +- struct sxe2_uqp *uqp; +- int err; +- struct ibv_qp *ibqp = NULL; +- struct sxe2_rdma_ucontext *sctx = to_sctx(context); +- struct sxe2_common_attrs *common_attrs = &sctx->uk_attrs; +- +- if (attr_ex->qp_type != IBV_QPT_RC && attr_ex->qp_type != IBV_QPT_UD) { +- errno = EOPNOTSUPP; +- SXE2_VERBS_LOG_ERROR_BDF("qp type illegal!, qp_type:%d\n", attr_ex->qp_type); +- goto end; +- } +- if (attr_ex->cap.max_send_sge > common_attrs->max_hw_wq_frags || +- attr_ex->cap.max_recv_sge > common_attrs->max_hw_wq_frags || +- attr_ex->cap.max_send_wr > common_attrs->max_hw_wq_quanta || +- attr_ex->cap.max_recv_wr > common_attrs->max_hw_rq_quanta || +- attr_ex->cap.max_inline_data > common_attrs->max_hw_inline) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("qp request caps illegal!, max_send_sge(%u) max_recv_sge(%u)" +- "max_send_wr(%u) max_recv_wr(%u) max_inline_data(%u)" +- "dev hw_attrs: max_hw_inline(%d), max_hw_wq_frags(%d)," +- "max_hw_wq_quanta(%d), max_hw_rq_quanta(%d),\n", attr_ex->cap.max_send_sge, +- attr_ex->cap.max_recv_sge, attr_ex->cap.max_send_wr, attr_ex->cap.max_recv_wr, +- attr_ex->cap.max_inline_data, common_attrs->max_hw_inline, +- common_attrs->max_hw_wq_frags, common_attrs->max_hw_wq_quanta, +- common_attrs->max_hw_rq_quanta); +- goto end; +- } +- +- SXE2_VERBS_LOG_INFO_BDF("qp caps, max_send_sge(%u) max_recv_sge(%u)" +- "max_send_wr(%u) max_recv_wr(%u) max_inline_data(%u)\n", attr_ex->cap.max_send_sge, +- attr_ex->cap.max_recv_sge, attr_ex->cap.max_send_wr, attr_ex->cap.max_recv_wr, +- attr_ex->cap.max_inline_data); +- init_info.sq_size = attr_ex->cap.max_send_wr; +- init_info.rq_size = attr_ex->cap.max_recv_wr; +- +- init_info.max_sq_sge_cnt = attr_ex->cap.max_send_sge; +- init_info.max_rq_sge_cnt = attr_ex->cap.max_recv_sge; +- init_info.max_inline_data = attr_ex->cap.max_inline_data; +- init_info.common_attrs = common_attrs; +- if (attr_ex->srq) { +- init_info.srq = &((to_usrq(attr_ex->srq))->srq_verbs); +- } +- +- err = sxe2_calc_sq_depth_shift(sctx, &init_info, &init_info.sq_depth, &init_info.sq_shift); +- if (err) { +- errno = err; +- SXE2_VERBS_LOG_ERROR_BDF("sxe2_calc_sq_depth_shift fail, err(%d)\n", err); +- goto end; +- } +- err = sxe2_calc_rq_depth_shift(sctx, &init_info, &init_info.rq_depth, &init_info.rq_shift); +- if (err) { +- errno = err; +- SXE2_VERBS_LOG_ERROR_BDF("sxe2_calc_rq_depth_shift fail, err(%d)\n", err); +- goto end; +- } +- uqp = memalign(QP_MEMALIGN_1K, sizeof(*uqp)); +- if (!uqp) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("memalign uqp fail!\n"); +- goto end; +- } +- memset(uqp, 0, sizeof(*uqp)); +- if (pthread_spin_init(&uqp->lock, PTHREAD_PROCESS_PRIVATE)) { +- errno = EBUSY; +- SXE2_VERBS_LOG_ERROR_BDF("pthread_spin_init fail!\n"); +- goto err_free_qp; +- } +- uqp->qp.common_attrs = common_attrs; +- init_info.sq_size = init_info.sq_depth >> init_info.sq_shift; +- init_info.rq_size = init_info.rq_depth >> init_info.rq_shift; +- +- init_info.qp_db_no_llwqe = (__u32 *)sctx->qp_db_no_llwqe; +- init_info.sq_wrtrk_array = calloc(init_info.sq_depth, sizeof(*init_info.sq_wrtrk_array)); +- if (!init_info.sq_wrtrk_array) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("calloc sq_wrtrk_array fail!\n"); +- goto err_destroy_lock; +- } +- +- if (init_info.rq_depth != 0) { +- init_info.rq_wrid_array = calloc(init_info.rq_depth, sizeof(*init_info.rq_wrid_array)); +- if (!init_info.rq_wrid_array) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("calloc rq_wrid_array fail!\n"); +- goto err_free_sq_wrtrk; +- } +- } else { +- init_info.rq_wrid_array = NULL; +- } +- +- uqp->buf_size = (init_info.sq_depth + init_info.rq_depth) * QP_QUANTA_SIZE; +- +- if (sxe2_alloc_qp_buf(sctx, &uqp->buf, uqp->buf_size, attr_ex->qp_type)) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("total_size %zu", uqp->buf_size); +- goto err_free_rq_wrid; +- } +- init_info.rq = uqp->buf.buf; +- if (!init_info.rq) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("alloc qp buffer fail!\n"); +- goto err_free_rq_wrid; +- } +- memset(init_info.rq, 0, uqp->buf_size); +- +- init_info.doorbell_note = sxe2_alloc_hw_buf(QP_DB_NOTE_SIZE); +- if (!init_info.doorbell_note) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("alloc shadow area buffer fail!\n"); +- goto err_free_vmap_qp; +- } +- memset(init_info.doorbell_note, 0, QP_DB_NOTE_SIZE); +- +- init_info.sq = &init_info.rq[init_info.rq_depth]; +- +- if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) { +- err = sxe2_qp_fill_wr_func(sctx, uqp, attr_ex); +- if (err) { +- errno = err; +- SXE2_VERBS_LOG_ERROR_BDF("fill_wr_func fail, err:%d", err); +- goto err_free_vmap_doorbell_note; +- } +- } +- +- SXE2_VERBS_LOG_INFO_BDF("rq depth %u shift %u", init_info.rq_depth, init_info.rq_shift); +- cmd.sq_depth = init_info.sq_depth; +- cmd.rq_depth = init_info.rq_depth; +- cmd.sq_shift = init_info.sq_shift; +- cmd.rq_shift = init_info.rq_shift; +- cmd.user_wqe_bufs = (__u64)(init_info.rq); +- cmd.doorbell_note = (__u64)(init_info.doorbell_note); +- cmd.user_compl_ctx = (__u64)(&uqp->qp); +- err = ibv_cmd_create_qp_ex(context, &uqp->verbs_qp, attr_ex, &(cmd.ibv_cmd), +- sizeof(cmd), &(resp.ibv_resp), +- sizeof(struct sxe2_ucreate_qp_resp)); +- if (err) { +- errno = err; +- SXE2_VERBS_LOG_ERROR_BDF("ibv_cmd_create_qp_ex fail!, err(%d)\n", err); +- goto err_free_vmap_doorbell_note; +- } +- +- sxe2_fill_qp(uqp, &init_info, resp, attr_ex); +- +- attr_ex->cap.max_send_wr = (init_info.sq_depth - SQ_RSV_SIZE) >> init_info.sq_shift; +- if ((init_info.srq) || (init_info.rq_depth == 0)) { +- attr_ex->cap.max_recv_wr = 0; +- } else { +- attr_ex->cap.max_recv_wr = (init_info.rq_depth - RQ_RSV_SIZE) >> init_info.rq_shift; +- } +- +- if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS) { +- uqp->verbs_qp.comp_mask |= VERBS_QP_EX; +- } +- +- ibqp = &uqp->verbs_qp.qp; +- goto end; +-err_free_vmap_doorbell_note: +- sxe2_free_hw_buf(init_info.doorbell_note, QP_DB_NOTE_SIZE); +-err_free_vmap_qp: +- sxe2_free_qp_buf(sctx, &uqp->buf); +-err_free_rq_wrid: +- if (init_info.rq_wrid_array) { +- free(init_info.rq_wrid_array); +- init_info.rq_wrid_array = NULL; +- } +-err_free_sq_wrtrk: +- free(init_info.sq_wrtrk_array); +- init_info.sq_wrtrk_array = NULL; +-err_destroy_lock: +- pthread_spin_destroy(&uqp->lock); +-err_free_qp: +- if (uqp) { +- free(uqp); +- uqp = NULL; +- } +-end: +- return ibqp; +-} +- +-struct ibv_qp *sxe2_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) +-{ +- struct ibv_qp *qp = NULL; +- struct ibv_qp_init_attr_ex attr_ex; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- if (!pd || !attr) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("inval param, pd(%p), attr(%p)", pd, attr); +- goto out; +- } +- +- sctx = to_sctx(pd->context); +- +- SXE2_VERBS_LOG_DEBUG_BDF( +- "type:%u, max_inline_data:%u, max_send_wr:%u" +- "max_send_sge:%u, max_send_recv_wr:%u, max_recv_sge:%u, pd_handle:%u", +- attr->qp_type, attr->cap.max_inline_data, attr->cap.max_send_wr, +- attr->cap.max_send_sge, attr->cap.max_recv_wr, attr->cap.max_recv_sge, +- pd->handle); +- +- memset(&attr_ex, 0, sizeof(attr_ex)); +- memcpy(&attr_ex, attr, sizeof(*attr)); +- +- attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD; +- attr_ex.pd = pd; +- qp = ucreate_qp(pd->context, &attr_ex); +- if (qp) { +- memcpy(attr, &attr_ex, sizeof(*attr)); +- } +- +-out: +- return qp; +-} +- +-struct ibv_qp *sxe2_ucreate_qp_ex(struct ibv_context *context, +- struct ibv_qp_init_attr_ex *attr_ex) +-{ +- struct ibv_qp *qp = NULL; +- struct sxe2_rdma_ucontext *sctx = NULL; +- +- if (!context || !attr_ex) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF("inval param, context(%p), attr_ex(%p)", context, attr_ex); +- goto out; +- } +- +- sctx = to_sctx(context); +- +- SXE2_VERBS_LOG_DEBUG_BDF( +- "type:%u, max_inline_data:%u, max_send_wr:%u, " +- "max_send_sge:%u, max_send_recv_wr:%u, " +- "max_recv_sge:%u, pd_handle:%u, comp_mask:%#x," +- "create_flags:%#x, send_flags:%#lx", +- attr_ex->qp_type, attr_ex->cap.max_inline_data, +- attr_ex->cap.max_send_wr, attr_ex->cap.max_send_sge, +- attr_ex->cap.max_recv_wr, attr_ex->cap.max_recv_sge, +- attr_ex->pd->handle, attr_ex->comp_mask, attr_ex->create_flags, +- attr_ex->send_ops_flags); +- qp = ucreate_qp(context, attr_ex); +- +-out: +- return qp; +-} +- +-int sxe2_udestroy_qp(struct ibv_qp *ibqp) +-{ +- struct sxe2_uqp *uqp; +-#ifdef SXE2_CFG_DEBUG +- struct sxe2_qp_common *qp; +-#endif +- struct sxe2_rdma_ucontext *sctx = container_of(ibqp->context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- int ret = 0; +- +- uqp = container_of(ibqp, struct sxe2_uqp, verbs_qp.qp); +-#ifdef SXE2_CFG_DEBUG +- qp = &uqp->qp; +-#endif +- ret = pthread_spin_destroy(&uqp->lock); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("pthread_spin_destroy fail!\n"); +- } +- +- ret = ibv_cmd_destroy_qp(ibqp); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv_cmd_destroy_qp fail!, ret(%d)\n", ret); +- } +- +- if (uqp->qp.verbs_llwqe) { +- db_uput_qp_llwqe(sctx, uqp->qp.verbs_llwqe); +- } +- if (uqp->send_cq) { +- sxe2_clean_cqes(&uqp->qp, uqp->send_cq, SQ_CQ); +- } +- if (uqp->recv_cq && uqp->recv_cq != uqp->send_cq) { +- sxe2_clean_cqes(&uqp->qp, uqp->recv_cq, RQ_CQ); +- } +- if (uqp->qp.sq_wrtrk_array) { +- free(uqp->qp.sq_wrtrk_array); +- uqp->qp.sq_wrtrk_array = NULL; +- } +- if (uqp->qp.rq_wrid_array) { +- free(uqp->qp.rq_wrid_array); +- uqp->qp.rq_wrid_array = NULL; +- } +- sxe2_free_qp_buf(sctx, &uqp->buf); +- sxe2_free_hw_buf(uqp->qp.doorbell_note, QP_DB_NOTE_SIZE); +- uqp->qp.doorbell_note = NULL; +-#ifdef SXE2_CFG_DEBUG +- SXE2_VERBS_LOG_DEBUG_BDF("qp [%d] io statistics:\n"\ +- "total_sqe_cnt : %u\n"\ +- "total_rqe_cnt : %u\n"\ +- "finished_sqe_cnt : %u\n"\ +- "finished_rqe_cnt : %u\n"\ +- "finished_rqe_insrq_cnt : %u\n"\ +- "flushed_sq_cnt : %u\n"\ +- "flushed_rq_cnt : %u\n"\ +- "cleaned_sq_cnt : %u\n"\ +- "cleaned_rq_cnt : %u\n"\ +- "cleaned_flushsq_cnt : %u\n"\ +- "cleaned_flushrq_cnt : %u\n"\ +- "total_signal_cnt : %u\n"\ +- "send_cnt : %u\n"\ +- "send_inv_cnt : %u\n"\ +- "read_cnt : %u\n"\ +- "write_cnt : %u\n"\ +- "local_inv_cnt : %u\n"\ +- "bind_mw_cnt : %u\n"\ +- "fast_regmr_cnt : %u\n"\ +- "last_send_sqwrid : %llu\n"\ +- "last_rcvd_sqwrid : %llu\n"\ +- "last_send_rqwrid : %llu\n"\ +- "last_rcvd_rqwrid : %llu\n", +- qp->qpn, +- qp->statistics.total_sqe_cnt, +- qp->statistics.total_rqe_cnt, +- qp->statistics.finished_sqe_cnt, +- qp->statistics.finished_rqe_cnt, +- qp->statistics.finished_rqe_insrq_cnt, +- qp->statistics.flushed_sq_cnt, +- qp->statistics.flushed_rq_cnt, +- qp->statistics.cleaned_sq_cnt, +- qp->statistics.cleaned_rq_cnt, +- qp->statistics.cleaned_flushsq_cnt, +- qp->statistics.cleaned_flushrq_cnt, +- qp->statistics.total_signal_cnt, +- qp->statistics.send_cnt, +- qp->statistics.send_inv_cnt, +- qp->statistics.read_cnt, +- qp->statistics.write_cnt, +- qp->statistics.local_inv_cnt, +- qp->statistics.bind_mw_cnt, +- qp->statistics.fast_regmr_cnt, +- qp->statistics.last_send_sqwrid, +- qp->statistics.last_rcvd_sqwrid, +- qp->statistics.last_send_rqwrid, +- qp->statistics.last_rcvd_rqwrid); +-#endif +- if (uqp) { +- free(uqp); +- uqp = NULL; +- } +- return ret; +-} +- +-int sxe2_umodify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask) +-{ +- struct sxe2_umodify_qp cmd = {}; +- struct sxe2_umodify_qp_resp resp = {}; +- struct sxe2_uqp *uqp; +- struct sxe2_verbs_llwqe *ll_wqe = NULL; +- struct sxe2_rdma_ucontext *sctx = container_of(ibqp->context, struct sxe2_rdma_ucontext, +- ibv_ctx.context); +- bool need_alloc_page = false; +- __u8 alloc_page_type = 0; +- __u32 db_mmap_size; +- __u32 db_page_id; +- __u64 db_mmap_offset; +- int ret; +- SXE2_VERBS_LOG_DEBUG_BDF("uqp:umodify qp %u start\n", ibqp->qp_num); +- if (attr_mask & IBV_QP_STATE) { +- SXE2_VERBS_LOG_DEBUG_BDF("uqp %u :umodify qp cur start %u next state %u\n", +- ibqp->qp_num, ibqp->state, attr->qp_state); +- } +- uqp = container_of(ibqp, struct sxe2_uqp, verbs_qp.qp); +- if (sctx->ll_mode && +- attr_mask & IBV_QP_STATE && attr->qp_state == IBV_QPS_RTS) { +- ll_wqe = db_uget_qp_llwqe(ibqp->context, &need_alloc_page, &alloc_page_type); +- if (ll_wqe) { +- uqp->qp.verbs_llwqe = ll_wqe; +- uqp->qp.push_wqe = ll_wqe->wqe_addr; +- uqp->qp.push_db = ll_wqe->db_addr; +- uqp->qp.llwqe_enable = true; +- uqp->qp.llwqe_mode = true; +- cmd.llwqe_enable = true; +- cmd.llwqe_page_index = ll_wqe->db_page_id; +- } +- if (need_alloc_page) { +- cmd.new_page_alloc = true; +- } +- } else if (attr_mask & IBV_QP_STATE && attr->qp_state == IBV_QPS_RESET) { +- if (uqp->send_cq) { +- sxe2_clean_cqes(&uqp->qp, uqp->send_cq, SQ_CQ); +- } +- if (uqp->recv_cq && uqp->recv_cq != uqp->send_cq) { +- sxe2_clean_cqes(&uqp->qp, uqp->recv_cq, RQ_CQ); +- } +- +- uqp->qp.sq_ring.head = 0; +- uqp->qp.sq_ring.tail = 0; +- uqp->qp.rq_ring.head = 0; +- uqp->qp.rq_ring.tail = 0; +- uqp->qp.initial_ring.head = 0; +- uqp->qp.initial_ring.tail = 0; +- if (uqp->qp.verbs_llwqe) { +- db_uput_qp_llwqe(sctx, uqp->qp.verbs_llwqe); +- uqp->qp.verbs_llwqe = NULL; +- } +- uqp->qp.swqe_polarity = 0; +- uqp->qp.rwqe_polarity = 0; +- uqp->qp.llwqe_mode = false; +- memset(uqp->qp.rq_base, 0, uqp->buf_size); +- } +- if ((attr_mask & IBV_QP_STATE) && +- (ibqp->state == IBV_QPS_SQD) +- &&(attr->qp_state == IBV_QPS_RTS)) { +- if (uqp->qp.sq_ring.head == uqp->qp.sq_ring.tail) { +- sxe2_push_nop_wqe(&uqp->qp); +- SXE2_VERBS_LOG_DEBUG_BDF("uqp %u:before modify sqd to rts push nop head=%u tail=%u\n", +- ibqp->qp_num, uqp->qp.sq_ring.head, uqp->qp.sq_ring.tail); +- } +- } +- +- ret = ibv_cmd_modify_qp_ex(ibqp, attr, attr_mask, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("ibv_modify_qp ret(%d)\n", ret); +- } +- if (sctx->ll_mode && need_alloc_page && +- attr_mask & IBV_QP_STATE && attr->qp_state == IBV_QPS_RTS) { +- db_mmap_size = resp.db_mmap_size; +- db_page_id = resp.db_page_id; +- db_mmap_offset = resp.db_mmap_offset; +- if (!ll_wqe) { +- ll_wqe = alloc_db_page_and_get_qp_llwqe(ibqp->context, db_mmap_size, db_page_id, db_mmap_offset, alloc_page_type); +- if (ll_wqe) { +- uqp->qp.verbs_llwqe = ll_wqe; +- uqp->qp.push_wqe = ll_wqe->wqe_addr; +- uqp->qp.push_db = ll_wqe->db_addr; +- uqp->qp.llwqe_enable = true; +- uqp->qp.llwqe_mode = true; +- } else { +- uqp->qp.llwqe_enable = false; +- uqp->qp.llwqe_mode = false; +- } +- } else { +- if (!db_ualloc_page_and_llwqes(ibqp->context, false, db_mmap_size, db_page_id, db_mmap_offset)) { +- SXE2_VERBS_LOG_WARN_BDF("LLWQE:Alloc new shared db page failed"); +- } +- } +- } +- +- uqp->qp.rd_fence_rate = resp.rd_fence_rate; +- SXE2_VERBS_LOG_DEBUG_BDF("uqp %u:modify qp finish\n", ibqp->qp_num); +- return ret; +-} +- +-int sxe2_uquery_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask, +- struct ibv_qp_init_attr *init_attr) +-{ +- struct ibv_query_qp cmd; +- +- return ibv_cmd_query_qp(ibqp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); +-} +diff -Naur rdma-core-48.0/providers/sxe2/readme.txt rdma-core-48.0.bak/providers/sxe2/readme.txt +--- rdma-core-48.0/providers/sxe2/readme.txt 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/readme.txt 1970-01-01 08:00:00.000000000 +0800 +@@ -1,15 +0,0 @@ +-ah.c：ah用户态接口实现 +-ah.h：ah用户态接口声明 +-cq.c：cq用户态创建、删除、查询、修改接口实现 +-db.c：doorbell资源管理 +-sxe2_abi.h：基于ibv标准req、resp结构体扩展sxe2厂商字段 +-sxe2_private_verbs.h：厂商私有接口声明 +-qp.c ：qp用户态创建、删除、查询、修改接口实现 +-srq.c ：srq用户态创建、删除、查询、修改接口、post_srq_recv实现 +-verbs.c ：用户态厂商驱动注册到libibverbs的接口函数，qp.c、srq.c是各个接口的具体实现 +-io.h：数据面相关结构体声明，如wqe结构体 +-io.c：post_send实现、post_recv实现、post_srq_recv实现、poll_cq实现、arm_cq实现、cq_event实现 +-mr.c：mr用户态接口实现 +-mw.c：mw用户态接口实现 +-pd.c：pd用户态接口实现 +-mc.c：组播用户态接口实现 +diff -Naur rdma-core-48.0/providers/sxe2/ring.h rdma-core-48.0.bak/providers/sxe2/ring.h +--- rdma-core-48.0/providers/sxe2/ring.h 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/ring.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,127 +0,0 @@ +-#ifndef __RING_H_ +-#define __RING_H_ +- +-#define SXE2_RING_INIT(_ring, _size) \ +- { \ +- (_ring).head = 0; \ +- (_ring).tail = 0; \ +- (_ring).size = (_size); \ +- } +-#define SXE2_RING_SIZE(_ring) ((_ring).size) +-#define SXE2_RING_CURRENT_HEAD(_ring) ((_ring).head) +-#define SXE2_RING_CURRENT_TAIL(_ring) ((_ring).tail) +- +-#define SXE2_RING_USED_QUANTA(_ring) \ +- ( \ +- (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \ +- ) +-#define SXE2_RING_USED_QUANTA_PAD(_ring) \ +- ( \ +- (((_ring).head + (_ring).size - (_ring).tail - 1) % (_ring).size) \ +- ) +- +-#define SXE2_RING_FREE_QUANTA(_ring) \ +- ( \ +- ((_ring).size - SXE2_RING_USED_QUANTA(_ring) - 1) \ +- ) +- +-#define SXE2_SQ_RING_FREE_QUANTA(_ring) \ +- ( \ +- ((_ring).size - SXE2_RING_USED_QUANTA(_ring) - 1) \ +- ) +- +-#define SXE2_RING_FULL_ERR(_ring) \ +- ( \ +- (SXE2_RING_USED_QUANTA(_ring) == ((_ring).size - 1)) \ +- ) +- +-#define SXE2_SQ_RING_FULL_ERR(_ring) \ +- ( \ +- (SXE2_RING_USED_QUANTA(_ring) == ((_ring).size - 1)) \ +- ) +- +-#define SXE2_RING_MORE_WORK(_ring) \ +- ( \ +- (SXE2_RING_USED_QUANTA(_ring) != 0) \ +- ) +-#define SXE2_RING_MORE_WORK_PAD(_ring) \ +- ( \ +- (SXE2_RING_USED_QUANTA_PAD(_ring) != 0) \ +- ) +- +-#define SXE2_RING_MOVE_HEAD(_ring, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if (!SXE2_RING_FULL_ERR(_ring)) { \ +- (_ring).head = ((_ring).head + 1) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ENOMEM; \ +- } \ +- } +- +-#define SXE2_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if ((SXE2_RING_USED_QUANTA(_ring) + (_count)) < size) { \ +- (_ring).head = ((_ring).head + (_count)) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ENOMEM; \ +- } \ +- } +- +-#define SXE2_SQ_RING_MOVE_HEAD(_ring, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if (!SXE2_SQ_RING_FULL_ERR(_ring)) { \ +- (_ring).head = ((_ring).head + 1) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ENOMEM; \ +- } \ +- } +- +-#define SXE2_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ +- { \ +- register __u32 size; \ +- size = (_ring).size; \ +- if ((SXE2_RING_USED_QUANTA(_ring) + (_count)) < (size - 1)) { \ +- (_ring).head = ((_ring).head + (_count)) % size; \ +- (_retcode) = 0; \ +- } else { \ +- (_retcode) = ENOMEM; \ +- } \ +- } +- +-#define SXE2_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \ +- (_ring).head = ((_ring).head + (_count)) % (_ring).size +- +-#define SXE2_RING_MOVE_TAIL(_ring) \ +- (_ring).tail = ((_ring).tail + 1) % (_ring).size +- +-#define SXE2_RING_MOVE_HEAD_NOCHECK(_ring) \ +- (_ring).head = ((_ring).head + 1) % (_ring).size +- +-#define SXE2_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \ +- (_ring).tail = ((_ring).tail + (_count)) % (_ring).size +- +-#define SXE2_RING_SET_TAIL(_ring, _pos) \ +- (_ring).tail = (_pos) % (_ring).size +- +-#define SXE2_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \ +- { \ +- index = SXE2_RING_CURRENT_HEAD(_ring); \ +- SXE2_RING_MOVE_HEAD(_ring, _retcode); \ +- } +- +-#define SXE2_CQP_INIT_WQE(wqe) memset(wqe, 0, 64) +- +-#define SXE2_GET_CURRENT_CQ_ELEM(_cq) \ +- ( \ +- (_cq)->cq_base[SXE2_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \ +- ) +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/srq.c rdma-core-48.0.bak/providers/sxe2/srq.c +--- rdma-core-48.0/providers/sxe2/srq.c 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/srq.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,451 +0,0 @@ +- +-#include +-#include +-#include "log.h" +-#include "verbs.h" +-#include "sxe2_common.h" +-#include "sxe2_abi.h" +-#include "ring.h" +-#include "io.h" +-#include "buf.h" +- +-#define SXE2_VERBS_SRQE_MIN_SIZE (32) +-#define SXE2_SRQE_MAX_FRAG_CNT_SHIFT \ +- (2) +-#define SXE2_SRQE_MAX_FRAG_CNT_SUB (1) +-#define SXE2_SRQ_SIZE_MUL_SHIFT (1) +-#define SXE2_SRQ_POLARITY_INIT (0) +-#define SXE2_SRQ_DB_NOTE_SIZE (16) +-#define SXE2_SRQ_MAX_BUF_SIZE (2 * 1024 * 1024) +- +-static void srq_ucal_wqe_shift(__u32 max_srq_sge, __u8 *srqe_shift) +-{ +- *srqe_shift = WQE_SIZE_32BYTE; +- +- if (max_srq_sge > MAX_SGE_SIZE_1) { +- if (max_srq_sge < MAX_SGE_SIZE_4) { +- *srqe_shift = WQE_SIZE_64BYTE; +- } else if (max_srq_sge < MAX_SGE_SIZE_8) { +- *srqe_shift = WQE_SIZE_128BYTE; +- } else { +- *srqe_shift = WQE_SIZE_256BYTE; +- } +- } +- return; +-} +- +-static int srq_uget_depth(struct sxe2_common_attrs *common_attrs, +- __u32 srq_size, __u8 shift, __u32 *srq_depth) +-{ +- int ret = 0; +- +- *srq_depth = sxe2_round_up_pow_2((srq_size << shift) + SXE2_RQ_RSVD); +- +- if (*srq_depth > common_attrs->max_hw_srq_quanta) { +- ret = EINVAL; +- } +- +- return ret; +-} +- +-static void srq_uinit_verbs_common(struct sxe2_srq_verbs *srq, +- struct sxe2_srq_verbs_init_info *info) +-{ +- __u8 srqshift; +- +- srq->common_attrs = info->common_attrs; +- +- srq_ucal_wqe_shift(info->max_srq_frag_cnt, &srqshift); +- +- srq->srq_base = info->srq; +- srq->db_note = info->db_note; +- srq->srq_wrid_array = info->srq_wrid_array; +- srq->srqe_array = info->srqe_array; +- +- srq->srq_id = info->srq_id; +- +- srq->srq_polarity = SXE2_SRQ_POLARITY_INIT; +- +- srq->srq_size = info->srq_size; +- +- srq->wqe_size = srqshift; +- +- srq->max_srq_frag_cnt = info->max_srq_frag_cnt; +- +- SXE2_RING_INIT(srq->srq_ring, srq->srq_size); +- +- srq->wqe_size_multiplier = SXE2_SRQ_SIZE_MUL_SHIFT << srqshift; +- +- srq->srq_buf_size = info->srq_buf_size; +- +- return; +-} +- +-int sxe2_uget_srq_num(struct ibv_srq *ibv_srq, uint32_t *srqn) +-{ +- int ret = 0; +- struct sxe2_usrq *usrq; +- struct sxe2_rdma_ucontext *sctx; +- +- if (!ibv_srq || !srqn) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR("inv param, ibv_srq %p, srqn addr %p", +- ibv_srq, srqn); +- goto end; +- } +- +- usrq = container_of(ibv_srq, struct sxe2_usrq, vsrq.srq); +- sctx = to_sctx(ibv_srq->context); +- if (!usrq || !sctx) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF( +- "SRQ:destroy srq attr illegal:usrq:%p, sctx:%p, ret:%d\n", usrq, +- sctx, ret); +- goto end; +- } +- +- *srqn = usrq->srq_verbs.srq_id; +- +-end: +- return ret; +-} +- +-static int sxe2_alloc_srq_buf(struct sxe2_rdma_ucontext *sctx, sxe2_buf_s *buf, size_t size) +-{ +- int ret; +- sxe2_alloc_type_e type; +- sxe2_alloc_type_e default_type = SXE2_ALLOC_TYPE_ANON; +- long page_size; +- +- sxe2_alloc_type_get(SXE2_SRQ_PREFIX, &type, default_type); +- +- page_size = sysconf(_SC_PAGE_SIZE); +- if (page_size < 0) { +- SXE2_VERBS_LOG_ERROR_BDF("get system page size failed."); +- ret = EPERM; +- goto end; +- } +- ret = sxe2_prefered_buf_alloc( +- sctx, buf, size, page_size, type); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("cq buf alloc err ret (%d)", ret); +- goto end; +- } +- +- memset(buf->buf, 0, buf->length); +- +-end: +- return ret; +-} +- +-static void sxe2_free_srq_buf(struct sxe2_rdma_ucontext *ctx, sxe2_buf_s *buf) +-{ +- sxe2_actual_buf_free(ctx, buf); +-} +- +-struct ibv_srq *sxe2_ucreate_srq(struct ibv_pd *pd, +- struct ibv_srq_init_attr *attr) +-{ +- struct sxe2_usrq *usrq; +- struct ibv_srq *ibvsrq = NULL; +- struct sxe2_rdma_ucontext *sctx; +- struct sxe2_ucreate_srq cmd; +- struct sxe2_ucreate_srq_resp resp; +- struct sxe2_srq_verbs_init_info info = { }; +- __u32 depth = 0; +- __u8 shift = 0; +- int ret = 0; +- +- if (!pd || !attr) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR( +- "SRQ:inv create srq input param, pd:%p, attr:%p, ret:%d", pd, attr, +- errno); +- goto end; +- } +- +- sctx = to_sctx(pd->context); +- if (attr->attr.max_wr > sctx->uk_attrs.max_hw_srq_wr || +- attr->attr.max_sge > sctx->uk_attrs.max_hw_wq_frags) { +- errno = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF( +- "SRQ:create srq attr illegal:attr max_wr:%#x, max_sge:%#x, " +- "srq_limit:%#x, ret:%d\n", +- attr->attr.max_wr, attr->attr.max_sge, attr->attr.srq_limit, errno); +- goto end; +- } +- SXE2_VERBS_LOG_INFO_BDF( +- "SRQ:create srq start, attr max_wr:%#x, max_sge:%#x, srq_limit:%#x", +- attr->attr.max_wr, attr->attr.max_sge, attr->attr.srq_limit); +- +- usrq = calloc(1, sizeof(*usrq)); +- if (!usrq) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF( +- "SRQ:user mem alloc failed, mem size:%zu, ret:%d\n", sizeof(*usrq), +- errno); +- goto end; +- } +- +- if (pthread_spin_init(&usrq->lock, PTHREAD_PROCESS_SHARED)) { +- errno = EBUSY; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:lock init failed, ret:%d\n", errno); +- goto free_usrq; +- } +- +- memset(&cmd, 0, sizeof(cmd)); +- memset(&resp, 0, sizeof(resp)); +- memset(&info, 0, sizeof(info)); +- ibvsrq = &usrq->vsrq.srq; +- +- info.common_attrs = &sctx->uk_attrs; +- info.max_srq_frag_cnt = attr->attr.max_sge; +- +- srq_ucal_wqe_shift(info.max_srq_frag_cnt, &shift); +- ret = srq_uget_depth(info.common_attrs, attr->attr.max_wr, shift, &depth); +- if (ret) { +- errno = ret; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:calculate srq depth failed, ret:%d\n", +- ret); +- goto free_mutex; +- } +- +- info.srq_buf_size = depth * SXE2_VERBS_SRQE_MIN_SIZE; +- +- info.srq_size = depth >> shift; +- +- info.srq_wrid_array = calloc(info.srq_size, sizeof(*info.srq_wrid_array)); +- if (!info.srq_wrid_array) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:calloc srq_wrid_array failed, ret: %d\n", +- errno); +- goto free_mutex; +- } +- +- info.srqe_array = calloc(info.srq_size, sizeof(*info.srqe_array)); +- if (!info.srqe_array) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:calloc srqe_array failed, ret: %d\n", +- errno); +- goto free_wrid_array; +- } +- +- if (sxe2_alloc_srq_buf(sctx, &usrq->buf, info.srq_buf_size)) { +- errno = ENOMEM; +- goto free_srqe_array; +- } +- +- info.srq = usrq->buf.buf; +- if (!info.srq) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:buf alloc failed, size:%#x, ret:%d\n", +- info.srq_buf_size, errno); +- goto free_srqe_array; +- } +- memset(info.srq, 0, info.srq_buf_size); +- +- info.db_note = (__u64 *)sxe2_alloc_hw_buf(SXE2_SRQ_DB_NOTE_SIZE); +- if (!info.db_note) { +- errno = ENOMEM; +- SXE2_VERBS_LOG_ERROR_BDF( +- "SRQ:DB Note buf alloc failed, size:%#x, ret %d\n", +- SXE2_SRQ_DB_NOTE_SIZE, errno); +- goto free_srq_buf; +- } +- memset(info.db_note, 0, SXE2_SRQ_DB_NOTE_SIZE); +- +- pthread_spin_lock(&usrq->lock); +- cmd.user_srq_buf = (__u64)((uintptr_t)info.srq); +- cmd.user_srq_db_note = (__u64)((uintptr_t)info.db_note); +- cmd.srq_cmpl_ctx = (__u64)((uintptr_t)(&usrq->srq_verbs)); +- cmd.srq_buf_size = info.srq_buf_size; +- cmd.srq_size = info.srq_size; +- cmd.max_wr_cal = ((depth - SXE2_RQ_RSVD) >> shift); +- ret = ibv_cmd_create_srq(pd, ibvsrq, attr, &cmd.ibv_cmd, sizeof(cmd), +- &resp.ibv_resp, sizeof(resp)); +- pthread_spin_unlock(&usrq->lock); +- if (ret) { +- errno = ret; +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:ibv cmd create srq failed, ret:%d\n", +- ret); +- goto free_srq_db_note; +- } +- +- info.srq_id = resp.srq_id; +- +- srq_uinit_verbs_common(&usrq->srq_verbs, &info); +- +- SXE2_VERBS_LOG_INFO_BDF( +- "SRQ(%#x):Create over, srq_buf_base addr:%p, srq_db_note addr:%p" +- "srq_size:%#x, max_srq_frag_cnt:%#x, srq_buf_size:%#x\n", +- usrq->srq_verbs.srq_id, usrq->srq_verbs.srq_base, +- usrq->srq_verbs.db_note, usrq->srq_verbs.srq_size, +- usrq->srq_verbs.max_srq_frag_cnt, usrq->srq_verbs.srq_buf_size); +- +- usrq->max_wr = (depth - SXE2_RQ_RSVD) >> shift; +- usrq->max_sge = usrq->srq_verbs.max_srq_frag_cnt; +- attr->attr.max_wr = usrq->max_wr; +- attr->attr.max_sge = usrq->max_sge; +- +- goto end; +- +-free_srq_db_note: +- sxe2_free_hw_buf(info.db_note, SXE2_SRQ_DB_NOTE_SIZE); +- info.db_note = NULL; +-free_srq_buf: +- sxe2_free_srq_buf(sctx, &usrq->buf); +- info.srq = NULL; +-free_srqe_array: +- free(info.srqe_array); +- info.srqe_array = NULL; +-free_wrid_array: +- free(info.srq_wrid_array); +- info.srq_wrid_array = NULL; +-free_mutex: +- pthread_spin_destroy(&usrq->lock); +-free_usrq: +- free(usrq); +- usrq = NULL; +- ibvsrq = NULL; +-end: +- return ibvsrq; +-} +- +-struct ibv_srq *sxe2_ucreate_srq_ex(struct ibv_context *context, +- struct ibv_srq_init_attr_ex *attr) +-{ +- struct ibv_srq *srq = NULL; +- struct sxe2_rdma_ucontext *sctx; +- +- if (!context || !attr) { +- SXE2_VERBS_LOG_ERROR("SRQ:Create inv param, context(%p) attr(%p)", +- context, attr); +- goto end; +- } +- +- sctx = to_sctx(context); +- SXE2_VERBS_LOG_INFO_BDF("SRQ:Create ex in, mask(%#x) srq_type(%#x) " +- "max_sge(%#x) max_wr(%#x) srq_limit(%#x)", +- attr->comp_mask, (__u32)attr->srq_type, +- attr->attr.max_sge, attr->attr.max_wr, +- attr->attr.srq_limit); +- +- if ((attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) && +- (attr->srq_type == IBV_SRQT_BASIC)) { +- srq = sxe2_ucreate_srq(attr->pd, (struct ibv_srq_init_attr *)attr); +- } else { +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:Create inv mask, comp_mask(%d) srq_type(%d)", +- attr->comp_mask, attr->srq_type); +- } +- +-end: +- return srq; +-} +- +-int sxe2_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, +- int attr_mask) +-{ +- int ret = 0; +- struct sxe2_rdma_ucontext *sctx; +- struct ibv_modify_srq cmd; +- +- if (!srq || !attr) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR( +- "SRQ:inv modify srq input param, srq:%p, attr:%p, ret:%d", srq, +- attr, ret); +- goto end; +- } +- sctx = to_sctx(srq->context); +- +- ret = ibv_cmd_modify_srq(srq, attr, attr_mask, &cmd, sizeof(cmd)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:ibv modify srq kernel failed, ret:%d", ret); +- goto end; +- } +- +- SXE2_VERBS_LOG_INFO_BDF("SRQ:Modify attr_mask:%#x, srq_limit:%#x\n", +- (__u32)attr_mask, attr->srq_limit); +- +-end: +- return ret; +-} +- +-int sxe2_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr) +-{ +- int ret = 0; +- struct sxe2_rdma_ucontext *sctx; +- struct ibv_query_srq cmd; +- +- if (!srq || !attr) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR( +- "SRQ:inv modify srq input param, srq:%p, attr:%p, ret:%d", srq, +- attr, ret); +- goto end; +- } +- sctx = to_sctx(srq->context); +- +- ret = ibv_cmd_query_srq(srq, attr, &cmd, sizeof(cmd)); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("SRQ:ibv query srq kernel failed, ret:%d", +- ret); +- goto end; +- } +- +- SXE2_VERBS_LOG_INFO_BDF("SRQ:Query max_wr:%#x, max_sge:%#x, limit:%#x\n", +- attr->max_wr, attr->max_sge, attr->srq_limit); +- +-end: +- return ret; +-} +- +-int sxe2_udestroy_srq(struct ibv_srq *ibv_srq) +-{ +- struct sxe2_usrq *usrq; +- struct sxe2_rdma_ucontext *sctx; +- int ret = 0; +- +- if (!ibv_srq) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR( +- "SRQ:inv destroy srq input param, ibv_srq:%p, ret:%d", ibv_srq, +- ret); +- goto end; +- } +- +- usrq = container_of(ibv_srq, struct sxe2_usrq, vsrq.srq); +- sctx = to_sctx(ibv_srq->context); +- if (!usrq || !sctx) { +- ret = EINVAL; +- SXE2_VERBS_LOG_ERROR_BDF( +- "SRQ:destroy srq attr illegal:usrq:%p, sctx:%p, ret:%d\n", usrq, +- sctx, ret); +- goto end; +- } +- +- ret = ibv_cmd_destroy_srq(ibv_srq); +- if (ret) { +- SXE2_VERBS_LOG_ERROR_BDF("SRQ(%#x):destroy srq kernel failed: " +- "srq_size:%#x, ret:%d\n", +- usrq->srq_verbs.srq_id, +- usrq->srq_verbs.srq_size, ret); +- goto end; +- } +- +- sxe2_free_srq_buf(sctx, &usrq->buf); +- sxe2_free_hw_buf(usrq->srq_verbs.db_note, SXE2_DB_NOTE_SIZE); +- usrq->srq_verbs.db_note = NULL; +- free(usrq->srq_verbs.srqe_array); +- usrq->srq_verbs.srqe_array = NULL; +- free(usrq->srq_verbs.srq_wrid_array); +- usrq->srq_verbs.srq_wrid_array = NULL; +- +- pthread_spin_destroy(&usrq->lock); +- +- free(usrq); +- usrq = NULL; +- +-end: +- return ret; +-} +diff -Naur rdma-core-48.0/providers/sxe2/sxe2-abi.h rdma-core-48.0.bak/providers/sxe2/sxe2-abi.h +--- rdma-core-48.0/providers/sxe2/sxe2-abi.h 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/sxe2-abi.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,108 +0,0 @@ +-#ifndef __SXE2_ABI_STRUCT_H__ +-#define __SXE2_ABI_STRUCT_H__ +- +-#include +- +-#define SXE2_RDMA_ABI_VER (5) +- +-struct sxe2_create_ah_resp { +- __u32 ah_id; +- __u8 rsvd[4]; +-}; +- +-struct sxe2_modify_qp_req { +- __u8 sq_flush; +- __u8 rq_flush; +- __u8 llwqe_enable; +- __u8 new_page_alloc; +- __u32 llwqe_page_index; +-}; +- +-struct sxe2_modify_qp_resp { +- __u8 rd_fence_rate; +- __u8 rsvd[3]; +- __u32 db_mmap_size; +- __u32 db_page_id; +- __u32 rsvd1; +- __u64 db_mmap_offset; +-}; +- +-struct sxe2_create_qp_req { +- __u32 sq_depth; +- __u32 rq_depth; +- __u8 sq_shift; +- __u8 rq_shift; +- __u8 rsvd[6]; +- __u64 user_wqe_bufs; +- __u64 doorbell_note; +- __u64 user_compl_ctx; +-}; +- +-struct sxe2_create_cq_req { +- __aligned_u64 user_cq_buf; +- __aligned_u64 user_cq_db_note; +-}; +- +-struct sxe2_create_cq_resp { +- __u32 cq_id; +- __u32 ncqe; +-}; +- +-struct sxe2_alloc_pd_resp { +- __u32 pd_id; +- __u8 rsvd[4]; +-}; +- +-struct sxe2_create_qp_resp { +- __u32 qpn; +- __u32 qp_caps; +-}; +- +-struct sxe2_alloc_ucontext_req { +- __u32 rsvd32; +- __u8 userspace_ver; +- __u8 rsvd8[3]; +- __aligned_u64 comp_mask; +-}; +- +-struct sxe2_alloc_ucontext_resp { +- __u32 max_pds; +- __u32 max_qps; +- __u32 wq_size; +- __u8 kernel_ver; +- __u8 rsvd[3]; +- __aligned_u64 feature_flags; +- __aligned_u64 db_mmap_key; +- __u32 max_hw_wq_frags; +- __u32 max_hw_read_sges; +- __u32 max_hw_inline; +- __u32 max_hw_rq_quanta; +- __u32 max_hw_wq_quanta; +- __u32 min_hw_cq_size; +- __u32 max_hw_cq_size; +- __u16 max_hw_sq_chunk; +- __u8 hw_rev; +- __u8 is_pf; +- __aligned_u64 comp_mask; +- __u16 min_hw_wq_size; +- __u32 max_db; +- __u8 rsvd3[2]; +- __u8 bdf[16]; +- __u32 max_hw_srq_quanta; +- __u32 max_hw_srq_wr; +-}; +- +-struct sxe2_create_srq_req { +- __aligned_u64 user_srq_buf; +- __aligned_u64 user_srq_db_note; +- __aligned_u64 srq_cmpl_ctx; +- __u32 srq_buf_size; +- __u32 srq_size; +- __u32 max_wr_cal; +-}; +- +-struct sxe2_create_srq_resp { +- __u32 srq_id; +-}; +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/sxe2_abi.h rdma-core-48.0.bak/providers/sxe2/sxe2_abi.h +--- rdma-core-48.0/providers/sxe2/sxe2_abi.h 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/sxe2_abi.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,29 +0,0 @@ +-#ifndef __SXE2_ABI_H__ +-#define __SXE2_ABI_H__ +- +-#include +-#include +-#include +-#include "sxe2_common.h" +- +-#define SXE2_IB_UVERBS_ABI_VERSION 1 +-DECLARE_DRV_CMD(sxe2_ucreate_ah, IB_USER_VERBS_CMD_CREATE_AH, +- empty, sxe2_create_ah_resp); +-DECLARE_DRV_CMD(sxe2_umodify_qp, IB_USER_VERBS_EX_CMD_MODIFY_QP, +- sxe2_modify_qp_req, sxe2_modify_qp_resp); +-DECLARE_DRV_CMD(sxe2_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ, +- sxe2_create_cq_req, sxe2_create_cq_resp); +-DECLARE_DRV_CMD(sxe2_ucreate_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ, +- sxe2_create_cq_req, sxe2_create_cq_resp); +-DECLARE_DRV_CMD(sxe2_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, +- empty, sxe2_alloc_pd_resp); +-DECLARE_DRV_CMD(sxe2_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP, +- sxe2_create_qp_req, sxe2_create_qp_resp); +-DECLARE_DRV_CMD(sxe2_get_context, IB_USER_VERBS_CMD_GET_CONTEXT, +- sxe2_alloc_ucontext_req, sxe2_alloc_ucontext_resp); +-DECLARE_DRV_CMD(sxe2_ucreate_srq, IB_USER_VERBS_CMD_CREATE_SRQ, +- sxe2_create_srq_req, sxe2_create_srq_resp); +-DECLARE_DRV_CMD(sxe2_ucreate_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ, +- sxe2_create_srq_req, sxe2_create_srq_resp); +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/sxe2_common.c rdma-core-48.0.bak/providers/sxe2/sxe2_common.c +--- rdma-core-48.0/providers/sxe2/sxe2_common.c 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/sxe2_common.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1,37 +0,0 @@ +-#include "sxe2_common.h" +- +-void *sxe2_alloc_hw_buf(size_t size) +-{ +- void *buf; +- long page_size = sysconf(_SC_PAGE_SIZE); +- +- buf = memalign(page_size, size); +- if (!buf) { +- goto end; +- } +- +- if (ibv_dontfork_range(buf, size)) { +- free(buf); +- buf = NULL; +- goto end; +- } +-end: +- return buf; +-} +- +-void sxe2_free_hw_buf(void *buf, size_t size) +-{ +- ibv_dofork_range(buf, size); +- free(buf); +-} +- +-__u32 sxe2_round_up_pow_2(__u32 value) +-{ +- int count = 1; +- +- for (value--; count <= 16; count *= 2) { +- value |= value >> count; +- } +- return ++value; +-} +- +diff -Naur rdma-core-48.0/providers/sxe2/sxe2_common.h rdma-core-48.0.bak/providers/sxe2/sxe2_common.h +--- rdma-core-48.0/providers/sxe2/sxe2_common.h 2026-05-26 10:42:01.855075215 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/sxe2_common.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,717 +0,0 @@ +-#ifndef __SXE2_COMMON_H__ +-#define __SXE2_COMMON_H__ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include "util/compiler.h" +-#include +-#include "util/udma_barrier.h" +-#include "util/util.h" +-#include "ccan/minmax.h" +-#include +-#include "sxe2_abi.h" +- +-#define SXE2_RDMA_BDF_BUFF_LEN 16 +-#define SXE2_WQE_SIZE 4 +-#define SXE2_QP_WQE_MAX_QUANTA 8 +-#define SXE2_QP_WQE_MIN_QUANTA 1 +-#define SXE2_CQE_SIZE 8 +-#define SXE2_U_MINCQ_SIZE (4) +-#define SXE2_DB_NOTE_SIZE (64) +-#define SXE2_DB_PAGE_SIZE (4096) +-#define SXE2_FEATURE_CQ_RESIZE (2ULL) +-#define SXE2_SRQE_BUSY 1 +-#define SXE2_SRQE_FREE 0 +- +-#define SQ_CQ 1 +-#define RQ_CQ 2 +- +-#define SXE2_WQE_QUANTA_ODD_NUMBER 0x1 +- +-#define SXE2_RQ_WQE_HEAD_OFFSET 32 +-#define SXE2_RQ_WQE_FRAG_OFFSET 16 +- +-#define SXE2_SINGLE_THREADED "SXE2_SINGLE_THREADED" +-#define SXE2_LL_MODE "SXE2_LL_MODE" +-#define SXE2_TOTAL_LL_WQE "SXE2_TOTAL_LL_WQE" +-#define SXE2_DEDICATED_LL_WQE "SXE2_DEDICATED_LL_WQE" +- +-#define sxe2_handle void* +-#define sxe2_qp_handle sxe2_handle +- +-#define SXE2_CQ_PREFIX "SXE2_CQ" +-#define SXE2_QP_PREFIX "SXE2_QP" +-#define SXE2_SRQ_PREFIX "SXE2_SRQ" +- +-#if defined(SXE2_CFG_DEBUG) && defined(SXE2_SUPPORT_INJECT) +-#define SXE2_INJECT_LLWQE_ERR "SXE2_INJECT_LLWQE_ERR" +-#endif +- +-enum sxe2_supported_wc_flags { +- SXE2_CQ_SUPPORTED_WC_FLAGS = IBV_WC_EX_WITH_BYTE_LEN +- | IBV_WC_EX_WITH_IMM +- | IBV_WC_EX_WITH_QP_NUM +- | IBV_WC_EX_WITH_SRC_QP +- | IBV_WC_EX_WITH_SLID +- | IBV_WC_EX_WITH_SL +- | IBV_WC_EX_WITH_DLID_PATH_BITS +- | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK +- | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP, +-}; +- +-enum inline_data_size { +- MAX_INLINE_DATA_8 = 8, +- MAX_INLINE_DATA_39 = 39, +- MAX_INLINE_DATA_101 = 101, +-}; +- +-enum sge_size { +- MAX_SGE_SIZE_0 = 0, +- MAX_SGE_SIZE_1, +- MAX_SGE_SIZE_2, +- MAX_SGE_SIZE_3, +- MAX_SGE_SIZE_4, +- MAX_SGE_SIZE_5, +- MAX_SGE_SIZE_6, +- MAX_SGE_SIZE_7, +- MAX_SGE_SIZE_8, +- MAX_SGE_SIZE_9, +- MAX_SGE_SIZE_10, +- MAX_SGE_SIZE_11, +- MAX_SGE_SIZE_12, +- MAX_SGE_SIZE_13, +- MAX_SGE_SIZE_14, +-}; +- +-enum wqe_size_shift { +- WQE_SIZE_32BYTE = 0, +- WQE_SIZE_64BYTE, +- WQE_SIZE_128BYTE, +- WQE_SIZE_256BYTE, +-}; +- +-enum sxe2_verbs_db_page_type { +- SXE2_VERBS_DB_PAGE_TYPE_LLWQE = 0x0, +- SXE2_VERBS_DB_PAGE_TYPE_NO_LLWQE = 0x1, +-}; +- +-enum sxe2_verbs_db_llwqe_page_type { +- SXE2_VERBS_DB_PAGE_TYPE_DEDICATED = 0x0, +- SXE2_VERBS_DB_PAGE_TYPE_SHARED = 0x1, +-}; +- +-enum sxe2_disp_id { +- SXE2_RDMA_SEND = 0, +- SXE2_RDMA_SEND_INLINE = 1, +- SXE2_RDMA_WRITE = 2, +- SXE2_RDMA_WRITE_INLINE = 3, +- SXE2_RDMA_READ = 4, +- SXE2_RDMA_BIND_MW = 5, +- SXE2_RDMA_LOCAL_INV = 6, +- SXE2_RDMA_FAST_REG_MR = 7, +- SXE2_RDMA_MAX_ID +-}; +- +-typedef enum sxe2_alloc_type { +- SXE2_ALLOC_TYPE_ANON, +- SXE2_ALLOC_TYPE_HUGE, +- SXE2_ALLOC_TYPE_PREFER_HUGE, +-} sxe2_alloc_type_e; +- +-typedef struct sxe2_hugetlb_mem { +- int shmid; +- __u32 reserved; +- void *shmaddr; +- unsigned long *bitmap; +- unsigned long bmp_size; +- struct list_node entry; +-} sxe2_hugetlb_mem_s; +- +-typedef struct sxe2_buf { +- void *buf; +- size_t length; +- __u64 base; +- __u32 reserved; +- sxe2_hugetlb_mem_s *hmem; +- sxe2_alloc_type_e type; +- __u64 resource_type; +- size_t req_alignment; +-} sxe2_buf_s; +- +-#define SXE2_RING_INIT(_ring, _size) \ +- { \ +- (_ring).head = 0; \ +- (_ring).tail = 0; \ +- (_ring).size = (_size); \ +- } +- +-struct sxe2_ring { +- __u32 head; +- __u32 tail; +- __u32 size; +-}; +- +-struct sxe2_sq_common_wr_trk_info { +- __u64 wrid; +- __u32 wr_len; +- __u16 quanta; +- __u8 reserved[2]; +-}; +- +-struct sxe2_qp_quanta { +- __le64 elem[SXE2_WQE_SIZE]; +-}; +- +-struct sxe2_common_attrs { +- __u64 feature_flags; +- __u32 max_hw_wq_frags; +- __u32 max_hw_read_sges; +- __u32 max_hw_inline; +- __u32 max_hw_rq_quanta; +- __u32 max_hw_wq_quanta; +- __u32 min_hw_cq_size; +- __u32 max_hw_cq_size; +- __u16 max_hw_push_len; +- __u16 max_hw_sq_chunk; +- __u16 min_hw_wq_size; +- __u8 hw_rev; +- __u8 rsv; +- __u32 max_hw_srq_quanta; +- __u32 max_hw_srq_wr; +-}; +- +-typedef struct sxe2_qp_common_init_info { +- struct sxe2_qp_quanta *sq; +- struct sxe2_qp_quanta *rq; +- struct sxe2_srq_verbs *srq; +- struct sxe2_common_attrs *common_attrs; +- __u32 *qp_db_no_llwqe; +- __u32 *doorbell_note; +- struct sxe2_sq_common_wr_trk_info *sq_wrtrk_array; +- __u64 *rq_wrid_array; +- __u32 qpn; +- __u32 qp_caps; +- __u32 sq_size; +- __u32 rq_size; +- __u32 max_sq_sge_cnt; +- __u32 max_rq_sge_cnt; +- __u32 max_inline_data; +- __u32 sq_depth; +- __u32 rq_depth; +- __u8 qp_type; +- __u8 sq_shift; +- __u8 rq_shift; +- bool legacy_mode; +-} sxe2_qp_common_init_info_s; +- +-struct sxe2_io_info { +- __u32 total_sqe_cnt; +- __u32 total_rqe_cnt; +- __u32 finished_sqe_cnt; +- __u32 finished_rqe_cnt; +- __u32 finished_rqe_insrq_cnt; +- __u32 flushed_sq_cnt; +- __u32 flushed_rq_cnt; +- __u32 cleaned_sq_cnt; +- __u32 cleaned_rq_cnt; +- __u32 cleaned_flushsq_cnt; +- __u32 cleaned_flushrq_cnt; +- __u32 total_signal_cnt; +- __u32 send_cnt; +- __u32 send_inv_cnt; +- __u32 read_cnt; +- __u32 write_cnt; +- __u32 local_inv_cnt; +- __u32 bind_mw_cnt; +- __u32 fast_regmr_cnt; +- __u64 last_send_sqwrid; +- __u64 last_rcvd_sqwrid; +- __u64 last_send_rqwrid; +- __u64 last_rcvd_rqwrid; +-}; +- +-typedef struct sxe2_qp_common { +- struct sxe2_qp_quanta *sq_base; +- struct sxe2_qp_quanta *rq_base; +- struct sxe2_srq_verbs *srq; +- struct sxe2_common_attrs *common_attrs; +- __u32 *qp_db_no_llwqe; +- struct sxe2_sq_common_wr_trk_info *sq_wrtrk_array; +- __u64 *rq_wrid_array; +- __u32 *doorbell_note; +- struct sxe2_verbs_llwqe *verbs_llwqe; +- __le32 *push_db; +- __u64 *push_wqe; +- struct sxe2_ring sq_ring; +- struct sxe2_ring rq_ring; +- struct sxe2_ring initial_ring; +- __u32 qpn; +- __u32 qp_caps; +- __u32 sq_size; +- __u32 rq_size; +- __u32 max_sq_sge_cnt; +- __u32 max_rq_sge_cnt; +- __u32 max_inline_data; +- __u8 swqe_polarity; +- __u8 rwqe_polarity; +- __u8 rq_wqe_size; +- __u8 rq_wqe_size_multiplier; +- __u8 rd_fence_rate; +- __u16 ord_cnt; +- bool llwqe_enable : 1; +- bool llwqe_mode : 1; +- bool push_dropped : 1; +- bool destroy_pending : 1; +- void *back_qp; +- pthread_spinlock_t *lock; +- struct sxe2_io_info statistics; +-} sxe2_qp_common_s; +- +-typedef struct sxe2_uqp { +- struct verbs_qp verbs_qp; +- struct sxe2_ucq *send_cq; +- struct sxe2_ucq *recv_cq; +- size_t buf_size; +- pthread_spinlock_t lock; +- __u32 sq_sig_all; +- sxe2_qp_common_s qp; +- enum ibv_qp_type qp_type; +- int err; +- __u32 wqe_idx; +- __le64 *cur_wqe; +- __u16 quanta; +- __le64 *wqe_hdr; +- unsigned int rb_sq_head; +- enum sxe2_disp_id funid; +- struct sxe2_qp_quanta wqebuf[SXE2_QP_WQE_MIN_QUANTA]; +- sxe2_buf_s buf; +-} sxe2_uqp_s; +- +-struct sxe2_cqe { +- __le64 buf[SXE2_CQE_SIZE]; +-}; +- +-struct sxe2_cq_db_note { +- __le64 buf[SXE2_CQE_SIZE]; +-}; +- +-struct sxe2_cqe_info { +- __u64 wr_id; +- __u32 bytes; +- union { +- struct { +- __u64 payload_len : 32; +- __u64 packet_seq : 24; +- __u64 rsvd1 : 8; +- __u64 qpc; +- __u64 l_r_key : 32; +- __u64 qp_id : 18; +- __u64 rsvd2 : 14; +- __u64 minor_err : 16; +- __u64 major_err : 16; +- __u64 wq_desc_idx : 15; +- __u64 rsvd3 : 3; +- __u64 extended_cqe : 1; +- __u64 push_dropped : 1; +- __u64 ipv4 : 1; +- __u64 stag_or_lrkey : 1; +- __u64 solicited_evt : 1; +- __u64 error : 1; +- __u64 op : 6; +- __u64 qp_type : 1; +- __u64 rsvd4 : 1; +- __u64 imme_data : 32; +- __u64 srqn : 18; +- __u64 is_srq : 1; +- __u64 rsvd5 : 13; +- __u64 cqe_timestamp; +- __u64 ud_smac : 48; +- __u64 ud_vlan_tag : 16; +- __u64 ud_src_qpn : 24; +- __u64 rsvd6 : 8; +- __u64 rsvd7 : 6; +- __u64 vsi_index : 10; +- __u64 rsvd8 : 12; +- __u64 vlan_tag_flag : 1; +- __u64 ud_smac_valid : 1; +- __u64 imm_data_flag : 1; +- __u64 cqe_valid : 1; +- } field; +- __u64 buf[SXE2_CQE_SIZE]; +- }info; +-}; +- +-struct sxe2_cq_uk_init_info { +- __le64 *cqe_alloc_db; +- struct sxe2_cqe *cq_base; +- __le32 *doorbell_note; +- __u32 cq_id; +- __u32 ncqe; +-}; +- +-struct sxe2_cq_uk { +- struct sxe2_cqe *cq_base; +- __le64 *cqe_alloc_db; +- __le32 *doorbell_note; +- __u32 arm_sn; +- __u32 cq_id; +- __u32 ncqe; +- struct sxe2_ring cq_ring; +- __u8 polarity; +-}; +- +-struct sxe2_ucq { +- struct verbs_cq verbs_cq; +- pthread_spinlock_t lock; +- sxe2_buf_s buf; +- size_t buf_size; +- uint32_t comp_vector; +- uint32_t report_rtt; +- struct sxe2_uqp *uqp; +- struct sxe2_cq_uk cq; +- struct sxe2_cqe_info cur_cqe; +-}; +- +-struct sxe2_upd { +- struct ibv_pd ibv_pd; +- void *arm_cq_page; +- void *arm_cq; +- uint32_t pd_id; +-}; +- +-struct sxe2_srq_verbs { +- struct sxe2_qp_quanta *srq_base; +- struct sxe2_common_attrs *common_attrs; +- __u64 *db_note; +- struct sxe2_ring srq_ring; +- __u32 srq_id; +- __u32 srq_size; +- __u32 max_srq_frag_cnt; +- __u8 srq_polarity; +- __u8 wqe_size; +- __u8 wqe_size_multiplier; +- __u32 srq_buf_size; +- __u64 *srq_wrid_array; +- __u8 *srqe_array; +-}; +- +-struct sxe2_srq_verbs_init_info { +- struct sxe2_qp_quanta *srq; +- struct sxe2_common_attrs *common_attrs; +- __le64 *db_note; +- __u64 *srq_wrid_array; +- __u8 *srqe_array; +- __u32 srq_id; +- __u32 srq_size; +- __u32 max_srq_frag_cnt; +- __u32 srq_buf_size; +-}; +- +-struct sxe2_usrq { +- struct verbs_srq vsrq; +- struct sxe2_srq_verbs srq_verbs; +- pthread_spinlock_t lock; +- __u32 max_wr; +- __u32 max_sge; +- sxe2_buf_s buf; +-}; +- +-struct sxe2_rdma_hw_attrs { +- struct sxe2_common_attrs uk_attrs; +- __u64 max_hw_outbound_msg_size; +- __u64 max_hw_inbound_msg_size; +- __u64 max_mr_size; +- __u32 min_hw_qp_id; +- __u32 min_hw_aeq_size; +- __u32 max_hw_aeq_size; +- __u32 min_hw_ceq_size; +- __u32 max_hw_ceq_size; +- __u32 max_hw_device_pages; +- __u32 max_hw_vf_fpm_id; +- __u32 first_hw_vf_fpm_id; +- __u32 max_rra; +- __u32 max_sra; +- __u32 max_hw_wqes; +- __u32 max_hw_pds; +- __u32 max_hw_ena_vf_count; +- __u32 max_qp_wr; +- __u32 max_pe_ready_count; +- __u32 max_done_count; +- __u32 max_sleep_count; +- __u32 max_mq_compl_wait_time_ms; +-}; +- +-typedef struct sxe2_spinlock { +- pthread_spinlock_t lock; +- int in_use; +- int need_lock; +-} sxe2_spinlock_s; +- +-struct sxe2_rdma_ucontext { +- struct verbs_context ibv_ctx; +- int abi_ver; +- struct sxe2_upd *sxe2_upd; +- struct sxe2_common_attrs uk_attrs; +- FILE *dbg_fp; +- char bdf[SXE2_RDMA_BDF_BUFF_LEN]; +- int ll_mode; +- __u32 tot_ll_wqes; +- __u32 ded_ll_wqes; +- __u32 shared_ll_wqes; +- struct list_head shared_llwqe_list; +- struct list_head dedicated_llwqe_list; +- struct list_head mmap_page_addr_list; +- __u16 alloc_dedicated_llwqes; +- __u16 alloc_shared_llwqes; +- pthread_mutex_t alloc_llwqe_mutex; +- void *qp_db_no_llwqe; +- void *cq_db_arm; +- void *cq_db_info; +- __u32 enable_io_log; +- __u32 log_level; +- sxe2_spinlock_s hugetlb_lock; +- struct list_head hugetlb_list; +-}; +- +-struct sxe2_umr { +- struct verbs_mr vmr; +- int acc_flags; +-}; +- +-struct sxe2_uah { +- struct ibv_ah ibv_ah; +- __u32 ah_id; +-}; +- +-struct sxe2_verbs_llwqe { +- void *wqe_addr; +- void *db_addr; +- pthread_spinlock_t lock; +- __u32 wqe_buf_size; +- __u32 num_llwqe; +- int need_lock; +- void *db_page_addr; +- off_t db_mmap_offset; +- __u8 mmaped_entry : 1; +- __u8 no_ll_mode : 1; +- __u8 qp_dedicated : 1; +- __u8 qp_shared : 1; +- __u32 count; +- struct list_node list_entry; +- __u32 db_handle; +- __u32 db_mmap_size; +- __u32 db_page_id; +-}; +- +-struct sxe2_db_mmap_db_page_addr { +- void* db_page_addr; +- __u32 mmap_size; +- __u32 db_page_id; +- struct list_node list_entry; +-}; +-struct ibv_cq *sxe2_ucreate_cq(struct ibv_context *context, int cqe, +- struct ibv_comp_channel *channel, int comp_vector); +-struct ibv_cq_ex *sxe2_ucreate_cq_ex(struct ibv_context *context, +- struct ibv_cq_init_attr_ex *cq_attr); +-int sxe2_umodify_cq(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr); +-int sxe2_udestroy_cq(struct ibv_cq *cq); +-struct ibv_pd *sxe2_ualloc_pd(struct ibv_context *context); +-int sxe2_ufree_pd(struct ibv_pd *pd); +-struct ibv_mr *sxe2_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, +- uint64_t hca_va, int acc); +-int sxe2_udereg_mr(struct verbs_mr *vmr); +-int sxe2_urereg_mr(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, +- void *addr, size_t length, int access); +-struct ibv_mr *sxe2_ureg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, +- size_t length, uint64_t iova, int fd, +- int acc); +-struct ibv_mw *sxe2_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type); +-struct ibv_ah *sxe2_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr); +-int sxe2_udestroy_ah(struct ibv_ah *ibah); +-int sxe2_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, +- struct ibv_send_wr **bad_wr); +-int sxe2_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, +- struct ibv_recv_wr **bad_wr); +-int sxe2_upoll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc); +-int sxe2_uarm_cq(struct ibv_cq *cq, int solicited); +-void sxe2_ucq_event(struct ibv_cq *cq); +-int sxe2_upost_srq_recv(struct ibv_srq *ibv_srq, +- struct ibv_recv_wr *ibv_wr, struct ibv_recv_wr **bad_wr); +-void sxe2_wr_start(struct ibv_qp_ex *qp_ex); +-int sxe2_wr_complete(struct ibv_qp_ex *qp_ex); +-void sxe2_wr_abort(struct ibv_qp_ex *qp_ex); +-void sxe2_wr_send_rc_ud(struct ibv_qp_ex *qp_ex); +-void sxe2_wr_send_imm_rc_ud(struct ibv_qp_ex *qp_ex, __be32 imm_data); +-void sxe2_wr_send_inv_rc(struct ibv_qp_ex *qp_ex, uint32_t invalidate_rkey); +-void sxe2_wr_rdma_read_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr); +-void sxe2_wr_rdma_write_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr); +-void sxe2_wr_rdma_write_imm_rc(struct ibv_qp_ex *qp_ex, uint32_t rkey, +- uint64_t remote_addr, __be32 imm_data); +-void sxe2_wr_set_ud_addr(struct ibv_qp_ex *qp_ex, struct ibv_ah *ah, +- uint32_t remote_qpn, uint32_t remote_qkey); +-void sxe2_wr_set_inline_data_rc_ud(struct ibv_qp_ex *qp_ex, +- void *addr, size_t length); +-void sxe2_wr_set_inline_data_list_rc_ud(struct ibv_qp_ex *qp_ex, +- size_t num_buf, const struct ibv_data_buf *buf_list); +-void sxe2_wr_set_sge_rc_ud(struct ibv_qp_ex *qp_ex, uint32_t lkey, +- uint64_t addr, uint32_t length); +-void sxe2_wr_set_sge_list_rc_ud(struct ibv_qp_ex *qp_ex, +- size_t num_sge, const struct ibv_sge *sg_list); +-void sxe2_wr_bind_mw_rc(struct ibv_qp_ex *qp_ex, struct ibv_mw *mw, +- uint32_t rkey, const struct ibv_mw_bind_info *bind_info); +-void sxe2_wr_local_inv_rc(struct ibv_qp_ex *qp_ex, uint32_t invalidate_rkey); +- +-static inline struct sxe2_rdma_ucontext *to_sctx(struct ibv_context *ibctx) +-{ +- return container_of(ibctx, struct sxe2_rdma_ucontext, ibv_ctx.context); +-} +- +-int sxe2_uget_single_threaded_env(void); +- +-int sxe2_uget_ll_mode(void); +- +-int sxe2_uget_tot_llwqe(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_get_context_resp *resp); +- +-int sxe2_uget_ded_llwqe(struct sxe2_rdma_ucontext *sctx, int total_llwqe); +- +-int sxe2_uinit_doorbell(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_get_context_resp *resp); +- +-void sxe2_ufree_doorbell(struct ibv_context *context); +- +- +-struct sxe2_verbs_llwqe *alloc_db_page_and_get_qp_llwqe(struct ibv_context *context, +- __u32 db_mmap_size, +- __u32 db_page_id, +- __u64 db_mmap_offset, +- __u8 alloc_page_type); +-struct sxe2_verbs_llwqe *db_uget_qp_llwqe(struct ibv_context *context, +- bool *need_alloc_page, +- __u8 *alloc_page_type); +- +-struct sxe2_verbs_llwqe * +-db_ualloc_page_and_llwqes(struct ibv_context *context, bool dedicated, +- __u32 db_mmap_size, +- __u32 db_page_id, +- __u64 db_mmap_offset); +- +-void db_uput_qp_llwqe(struct sxe2_rdma_ucontext *sctx, +- struct sxe2_verbs_llwqe *llwqe); +- +-void *sxe2_alloc_hw_buf(size_t size); +- +-void sxe2_free_hw_buf(void *buf, size_t size); +- +-__u32 sxe2_round_up_pow_2(__u32 value); +- +-struct ibv_qp *sxe2_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr); +- +-struct ibv_qp *sxe2_ucreate_qp_ex(struct ibv_context *context, +- struct ibv_qp_init_attr_ex *attr_ex); +- +-int sxe2_udestroy_qp(struct ibv_qp *ibqp); +- +-int sxe2_umodify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask); +- +-int sxe2_uquery_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr, int attr_mask, +- struct ibv_qp_init_attr *init_attr); +-int sxe2_start_poll(struct ibv_cq_ex *ibvcq_ex, struct ibv_poll_cq_attr *attr); +- +-int sxe2_next_poll(struct ibv_cq_ex *ibvcq_ex); +-void sxe2_end_poll(struct ibv_cq_ex *ibvcq_ex); +-uint64_t sxe2_wc_read_completion_ts(struct ibv_cq_ex *ibvcq_ex); +-uint64_t sxe2_wc_read_completion_wallclock_ns(struct ibv_cq_ex *ibvcq_ex); +-enum ibv_wc_opcode sxe2_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex); +-uint32_t sxe2_wc_read_vendor_err(struct ibv_cq_ex *ibvcq_ex); +-unsigned int sxe2_wc_read_wc_flags(struct ibv_cq_ex *ibvcq_ex); +-uint32_t sxe2_wc_read_byte_len(struct ibv_cq_ex *ibvcq_ex); +-__be32 sxe2_wc_read_imm_data(struct ibv_cq_ex *ibvcq_ex); +-uint32_t sxe2_wc_read_qp_num(struct ibv_cq_ex *ibvcq_ex); +-uint32_t sxe2_wc_read_src_qp(struct ibv_cq_ex *ibvcq_ex); +-uint32_t sxe2_wc_read_slid(struct ibv_cq_ex *ibvcq_ex); +-uint8_t sxe2_wc_read_sl(struct ibv_cq_ex *ibvcq_ex); +-uint8_t sxe2_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcq_ex); +- +-int sxe2_uget_srq_num(struct ibv_srq *ibv_srq, uint32_t *srqn); +- +-struct ibv_srq *sxe2_ucreate_srq(struct ibv_pd *pd, +- struct ibv_srq_init_attr *attr); +- +-struct ibv_srq *sxe2_ucreate_srq_ex(struct ibv_context *context, +- struct ibv_srq_init_attr_ex *attr); +- +-int sxe2_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, +- int attr_mask); +- +-int sxe2_uquery_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr); +- +-int sxe2_udestroy_srq(struct ibv_srq *ibv_srq); +- +-static inline struct sxe2_usrq *to_usrq(struct ibv_srq *ibv_srq) +-{ +- return container_of(ibv_srq, struct sxe2_usrq, vsrq.srq); +-} +- +-int sxe2_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, __u16 lid); +- +-int sxe2_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, __u16 lid); +- +-static inline int sxe2_spin_lock(sxe2_spinlock_s *lock) +-{ +- int ret = 0; +- if (lock->need_lock) { +- ret = pthread_spin_lock(&lock->lock); +- goto end; +- } +- +- if (unlikely(lock->in_use)) { +- fprintf(stderr, "*** ERROR: multithreading violation ***\n" +- "You are running a multithreaded application but\n" +- "you set SXE2_SINGLE_THREADED=1. Please unset it.\n"); +- abort(); +- } else { +- lock->in_use = 1; +- atomic_thread_fence(memory_order_acq_rel); +- } +-end: +- return ret; +-} +- +-static inline int sxe2_spin_unlock(sxe2_spinlock_s *lock) +-{ +- int ret = 0; +- +- if (lock->need_lock) { +- ret = pthread_spin_unlock(&lock->lock); +- goto end; +- } +- +- lock->in_use = 0; +-end: +- return ret; +-} +- +-static inline int sxe2_spinlock_init(sxe2_spinlock_s *lock, int need_lock) +-{ +- lock->in_use = 0; +- lock->need_lock = need_lock; +- +- return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_SHARED); +-} +- +-static inline int sxe2_spinlock_destroy(sxe2_spinlock_s *lock) +-{ +- return pthread_spin_destroy(&lock->lock); +-} +- +-#endif +diff -Naur rdma-core-48.0/providers/sxe2/verbs.c rdma-core-48.0.bak/providers/sxe2/verbs.c +--- rdma-core-48.0/providers/sxe2/verbs.c 2026-05-26 10:42:01.854075208 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/verbs.c 1970-01-01 08:00:00.000000000 +0800 +@@ -1 +0,0 @@ +- +diff -Naur rdma-core-48.0/providers/sxe2/verbs.h rdma-core-48.0.bak/providers/sxe2/verbs.h +--- rdma-core-48.0/providers/sxe2/verbs.h 2026-05-26 10:42:01.853075201 +0800 ++++ rdma-core-48.0.bak/providers/sxe2/verbs.h 1970-01-01 08:00:00.000000000 +0800 +@@ -1,6 +0,0 @@ +-#ifndef __SXE2_USER_VERBS_H__ +-#define __SXE2_USER_VERBS_H__ +- +-#include "sxe2_common.h" +- +-#endif +\ 文件尾没有换行符 diff --git a/SPECS/rdma-core.spec b/SPECS/rdma-core.spec index 5fa1e499e34ce9a1ad1ba139bb0a07d2ed2ece97..99f076dfb50e23d48de7dd0ed21160544c116abc 100644 --- a/SPECS/rdma-core.spec +++ b/SPECS/rdma-core.spec @@ -1,6 +1,6 @@ Name: rdma-core Version: 48.0 -Release: 1%{?dist} +Release: 2%{?dist} Summary: RDMA core userspace libraries and daemons # Almost everything is licensed under the OFA dual GPLv2, 2 Clause BSD license @@ -22,6 +22,7 @@ Patch301: 0002-systemd-drop-Protect-options-not-supported-in-RHEL-8.patch Patch9000: 0003-CMakeLists-disable-providers-that-were-not-enabled-i.patch Patch9998: 9998-kernel-boot-Do-not-perform-device-rename-on-OPA-devi.patch Patch9999: 9999-udev-keep-NAME_KERNEL-as-default-interface-naming-co.patch +Patch10000:10000-add-sxe2-rdma-core.patch # Do not build static libs by default. %define with_static %{?_with_static: 1} %{?!_with_static: 0} @@ -273,6 +274,7 @@ easy, object-oriented access to IB verbs. %patch9000 -p1 %patch9999 -p1 %endif +%patch10000 -p1 %build @@ -537,6 +539,7 @@ fi %{_libdir}/libmlx4.so.* %{_libdir}/libmana.so.* %endif +%{_libdir}/libibverbs/libsxe2-rdmav34.so %config(noreplace) %{_sysconfdir}/libibverbs.d/*.driver %doc %{_docdir}/%{name}/libibverbs.md @@ -631,6 +634,9 @@ fi %endif %changelog +* Tue May 26 2026 Jie Liu - 48.0-2 +- support SXE2 rdma-core + * Thu Sep 21 2023 Kamal Heib - 48.0-1 - Rebase to upstream release v48.0 - Resolves: RHEL-930, RHEL-4240, RHEL-6281, RHEL-6078