jcs.org / qemu
qemu with hax to log dma reads & writes jcs.org/2018/11/12/vfio
fork atom

hw/pvrdma: Remove max-sge command-line param

This parameter has no effect, fix it.

The function init_dev_caps sets the front-end's max-sge to MAX_SGE. Then
it checks backend's max-sge and adjust it accordingly (we can't send
more than what the device supports).

On send and recv we need to make sure the num_sge in the WQE does not
exceeds the backend device capability.
This check is done in pvrdma level so check on rdma level is deleted.

Signed-off-by: Yuval Shaia <yuval.shaia@oracle.com>
Message-Id: <20190109194123.3468-1-yuval.shaia@oracle.com>
Reviewed-by: Marcel Apfelbaum<marcel.apfelbaum@gmail.com>
Signed-off-by: Marcel Apfelbaum <marcel.apfelbaum@gmail.com>

authored by

Yuval Shaia and committed by
Marcel Apfelbaum ffef4775 5a301bb9

+42 -27
-1
docs/pvrdma.txt
··· 156 156 specify the port to use. If not set 1 will be used. 157 157 - dev-caps-max-mr-size: The maximum size of MR. 158 158 - dev-caps-max-qp: Maximum number of QPs. 159 - - dev-caps-max-sge: Maximum number of SGE elements in WR. 160 159 - dev-caps-max-cq: Maximum number of CQs. 161 160 - dev-caps-max-mr: Maximum number of MRs. 162 161 - dev-caps-max-pd: Maximum number of PDs.
+2 -21
hw/rdma/rdma_backend.c
··· 32 32 #include "rdma_rm.h" 33 33 #include "rdma_backend.h" 34 34 35 - /* Vendor Errors */ 36 - #define VENDOR_ERR_FAIL_BACKEND 0x201 37 - #define VENDOR_ERR_TOO_MANY_SGES 0x202 38 - #define VENDOR_ERR_NOMEM 0x203 39 - #define VENDOR_ERR_QP0 0x204 40 - #define VENDOR_ERR_INV_NUM_SGE 0x205 41 - #define VENDOR_ERR_MAD_SEND 0x206 42 - #define VENDOR_ERR_INVLKEY 0x207 43 - #define VENDOR_ERR_MR_SMALL 0x208 44 - #define VENDOR_ERR_INV_MAD_BUFF 0x209 45 - 46 35 #define THR_NAME_LEN 16 47 36 #define THR_POLL_TO 5000 48 37 ··· 475 464 } 476 465 477 466 pr_dbg("num_sge=%d\n", num_sge); 478 - if (!num_sge || num_sge > MAX_SGE) { 479 - pr_dbg("invalid num_sge=%d\n", num_sge); 480 - complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_NUM_SGE, ctx); 481 - return; 482 - } 483 467 484 468 bctx = g_malloc0(sizeof(*bctx)); 485 469 bctx->up_ctx = ctx; ··· 602 586 } 603 587 604 588 pr_dbg("num_sge=%d\n", num_sge); 605 - if (!num_sge || num_sge > MAX_SGE) { 606 - pr_dbg("invalid num_sge=%d\n", num_sge); 607 - complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_NUM_SGE, ctx); 608 - return; 609 - } 610 589 611 590 bctx = g_malloc0(sizeof(*bctx)); 612 591 bctx->up_ctx = ctx; ··· 941 920 if (ibv_query_device(backend_dev->context, &backend_dev->dev_attr)) { 942 921 return -EIO; 943 922 } 923 + 924 + dev_attr->max_sge = MAX_SGE; 944 925 945 926 CHK_ATTR(dev_attr, backend_dev->dev_attr, max_mr_size, "%" PRId64); 946 927 CHK_ATTR(dev_attr, backend_dev->dev_attr, max_qp, "%d");
+11
hw/rdma/rdma_backend.h
··· 22 22 #include "rdma_rm_defs.h" 23 23 #include "rdma_backend_defs.h" 24 24 25 + /* Vendor Errors */ 26 + #define VENDOR_ERR_FAIL_BACKEND 0x201 27 + #define VENDOR_ERR_TOO_MANY_SGES 0x202 28 + #define VENDOR_ERR_NOMEM 0x203 29 + #define VENDOR_ERR_QP0 0x204 30 + #define VENDOR_ERR_INV_NUM_SGE 0x205 31 + #define VENDOR_ERR_MAD_SEND 0x206 32 + #define VENDOR_ERR_INVLKEY 0x207 33 + #define VENDOR_ERR_MR_SMALL 0x208 34 + #define VENDOR_ERR_INV_MAD_BUFF 0x209 35 + 25 36 /* Add definition for QP0 and QP1 as there is no userspace enums for them */ 26 37 enum ibv_special_qp_type { 27 38 IBV_QPT_SMI = 0,
+5 -5
hw/rdma/vmw/pvrdma_main.c
··· 43 43 DEFINE_PROP_UINT64("dev-caps-max-mr-size", PVRDMADev, dev_attr.max_mr_size, 44 44 MAX_MR_SIZE), 45 45 DEFINE_PROP_INT32("dev-caps-max-qp", PVRDMADev, dev_attr.max_qp, MAX_QP), 46 - DEFINE_PROP_INT32("dev-caps-max-sge", PVRDMADev, dev_attr.max_sge, MAX_SGE), 47 46 DEFINE_PROP_INT32("dev-caps-max-cq", PVRDMADev, dev_attr.max_cq, MAX_CQ), 48 47 DEFINE_PROP_INT32("dev-caps-max-mr", PVRDMADev, dev_attr.max_mr, MAX_MR), 49 48 DEFINE_PROP_INT32("dev-caps-max-pd", PVRDMADev, dev_attr.max_pd, MAX_PD), ··· 549 548 sizeof(struct pvrdma_rq_wqe_hdr)); 550 549 551 550 dev->dev_attr.max_qp_wr = pg_tbl_bytes / 552 - (wr_sz + sizeof(struct pvrdma_sge) * MAX_SGE) - 553 - TARGET_PAGE_SIZE; /* First page is ring state */ 551 + (wr_sz + sizeof(struct pvrdma_sge) * 552 + dev->dev_attr.max_sge) - TARGET_PAGE_SIZE; 553 + /* First page is ring state ^^^^ */ 554 554 pr_dbg("max_qp_wr=%d\n", dev->dev_attr.max_qp_wr); 555 555 556 556 dev->dev_attr.max_cqe = pg_tbl_bytes / sizeof(struct pvrdma_cqe) - ··· 626 626 627 627 init_regs(pdev); 628 628 629 - init_dev_caps(dev); 630 - 631 629 rc = init_msix(pdev, errp); 632 630 if (rc) { 633 631 goto out; ··· 639 637 if (rc) { 640 638 goto out; 641 639 } 640 + 641 + init_dev_caps(dev); 642 642 643 643 rc = rdma_rm_init(&dev->rdma_dev_res, &dev->dev_attr, errp); 644 644 if (rc) {
+24
hw/rdma/vmw/pvrdma_qp_ops.c
··· 121 121 g_free(ctx); 122 122 } 123 123 124 + static void complete_with_error(uint32_t vendor_err, void *ctx) 125 + { 126 + struct ibv_wc wc = {0}; 127 + 128 + wc.status = IBV_WC_GENERAL_ERR; 129 + wc.vendor_err = vendor_err; 130 + 131 + pvrdma_qp_ops_comp_handler(ctx, &wc); 132 + } 133 + 124 134 void pvrdma_qp_ops_fini(void) 125 135 { 126 136 rdma_backend_unregister_comp_handler(); ··· 182 192 return -EIO; 183 193 } 184 194 195 + if (wqe->hdr.num_sge > dev->dev_attr.max_sge) { 196 + pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge, 197 + dev->dev_attr.max_sge); 198 + complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx); 199 + continue; 200 + } 201 + 185 202 rdma_backend_post_send(&dev->backend_dev, &qp->backend_qp, qp->qp_type, 186 203 (struct ibv_sge *)&wqe->sge[0], wqe->hdr.num_sge, 187 204 sgid_idx, sgid, ··· 226 243 comp_ctx->cqe.wr_id = wqe->hdr.wr_id; 227 244 comp_ctx->cqe.qp = qp_handle; 228 245 comp_ctx->cqe.opcode = IBV_WC_RECV; 246 + 247 + if (wqe->hdr.num_sge > dev->dev_attr.max_sge) { 248 + pr_dbg("Invalid num_sge=%d (max %d)\n", wqe->hdr.num_sge, 249 + dev->dev_attr.max_sge); 250 + complete_with_error(VENDOR_ERR_INV_NUM_SGE, comp_ctx); 251 + continue; 252 + } 229 253 230 254 rdma_backend_post_recv(&dev->backend_dev, &dev->rdma_dev_res, 231 255 &qp->backend_qp, qp->qp_type,