diff --git a/Documentation/target/tcmu-design.txt b/Documentation/target/tcmu-design.txt index bef81e42788f2b..4cebc1ebf99a6f 100644 --- a/Documentation/target/tcmu-design.txt +++ b/Documentation/target/tcmu-design.txt @@ -117,7 +117,9 @@ userspace (respectively) to put commands on the ring, and indicate when the commands are completed. version - 1 (userspace should abort if otherwise) -flags - none yet defined. +flags: +- TCMU_MAILBOX_FLAG_CAP_OOOC: indicates out-of-order completion is + supported. See "The Command Ring" for details. cmdr_off - The offset of the start of the command ring from the start of the memory region, to account for the mailbox size. cmdr_size - The size of the command ring. This does *not* need to be a @@ -162,6 +164,13 @@ rsp.sense_buffer if necessary. Userspace then increments mailbox.cmd_tail by entry.hdr.length (mod cmdr_size) and signals the kernel via the UIO method, a 4-byte write to the file descriptor. +If TCMU_MAILBOX_FLAG_CAP_OOOC is set for mailbox->flags, kernel is +capable of handling out-of-order completions. In this case, userspace can +handle command in different order other than original. Since kernel would +still process the commands in the same order it appeared in the command +ring, userspace need to update the cmd->id when completing the +command(a.k.a steal the original command's entry). + When the opcode is PAD, userspace only updates cmd_tail as above -- it's a no-op. (The kernel inserts PAD entries to ensure each CMD entry is contiguous within the command ring.) diff --git a/MAINTAINERS b/MAINTAINERS index 4978dc19a4d277..7bddfee55154ec 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5360,6 +5360,14 @@ S: Supported F: drivers/scsi/ibmvscsi/ibmvscsi* F: drivers/scsi/ibmvscsi/viosrp.h +IBM Power Virtual SCSI Device Target Driver +M: Bryant G. Ly +L: linux-scsi@vger.kernel.org +L: target-devel@vger.kernel.org +S: Supported +F: drivers/scsi/ibmvscsi/ibmvscsis.* +F: drivers/scsi/libsrp.* + IBM Power Virtual FC Device Drivers M: Tyrel Datwyler L: linux-scsi@vger.kernel.org diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index f28754c497e506..db475d41b57a5d 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -701,3 +701,31 @@ _GLOBAL(kexec_sequence) li r5,0 blr /* image->start(physid, image->start, 0); */ #endif /* CONFIG_KEXEC */ + +#ifdef CONFIG_MODULES +#if defined(_CALL_ELF) && _CALL_ELF == 2 + +#ifdef CONFIG_MODVERSIONS +.weak __crc_TOC. +.section "___kcrctab+TOC.","a" +.globl __kcrctab_TOC. +__kcrctab_TOC.: + .llong __crc_TOC. +#endif + +/* + * Export a fake .TOC. since both modpost and depmod will complain otherwise. + * Both modpost and depmod strip the leading . so we do the same here. + */ +.section "__ksymtab_strings","a" +__kstrtab_TOC.: + .asciz "TOC." + +.section "___ksymtab+TOC.","a" +/* This symbol name is important: it's used by modpost to find exported syms */ +.globl __ksymtab_TOC. +__ksymtab_TOC.: + .llong 0 /* .value */ + .llong __kstrtab_TOC. +#endif /* ELFv2 */ +#endif /* MODULES */ diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 08b7a40de5f85a..59663af9315fc1 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -326,10 +326,7 @@ static void dedotify_versions(struct modversion_info *vers, } } -/* - * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC. - * seem to be defined (value set later). - */ +/* Undefined symbols which refer to .funcname, hack to funcname (or .TOC.) */ static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab) { unsigned int i; @@ -337,11 +334,8 @@ static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab) for (i = 1; i < numsyms; i++) { if (syms[i].st_shndx == SHN_UNDEF) { char *name = strtab + syms[i].st_name; - if (name[0] == '.') { - if (strcmp(name+1, "TOC.") == 0) - syms[i].st_shndx = SHN_ABS; - syms[i].st_name++; - } + if (name[0] == '.') + memmove(name, name+1, strlen(name)); } } } @@ -357,7 +351,7 @@ static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs, numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym); for (i = 1; i < numsyms; i++) { - if (syms[i].st_shndx == SHN_ABS + if (syms[i].st_shndx == SHN_UNDEF && strcmp(strtab + syms[i].st_name, "TOC.") == 0) return &syms[i]; } diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index f121e612933913..60b30d338a8134 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -49,22 +49,25 @@ static struct workqueue_struct *isert_release_wq; static void isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); static int -isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, - struct isert_rdma_wr *wr); +isert_map_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn); static void isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); static int -isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, - struct isert_rdma_wr *wr); +isert_reg_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn); static int isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); static int -isert_rdma_post_recvl(struct isert_conn *isert_conn); +isert_login_post_recv(struct isert_conn *isert_conn); static int isert_rdma_accept(struct isert_conn *isert_conn); struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); static void isert_release_work(struct work_struct *work); +static void isert_wait4flush(struct isert_conn *isert_conn); +static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); static inline bool isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) @@ -177,12 +180,6 @@ isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) return ret; } -static void -isert_cq_event_callback(struct ib_event *e, void *context) -{ - isert_dbg("event: %d\n", e->event); -} - static int isert_alloc_rx_descriptors(struct isert_conn *isert_conn) { @@ -212,6 +209,7 @@ isert_alloc_rx_descriptors(struct isert_conn *isert_conn) rx_sg->addr = rx_desc->dma_addr; rx_sg->length = ISER_RX_PAYLOAD_SIZE; rx_sg->lkey = device->pd->local_dma_lkey; + rx_desc->rx_cqe.done = isert_recv_done; } return 0; @@ -250,9 +248,6 @@ isert_free_rx_descriptors(struct isert_conn *isert_conn) isert_conn->rx_descs = NULL; } -static void isert_cq_work(struct work_struct *); -static void isert_cq_callback(struct ib_cq *, void *); - static void isert_free_comps(struct isert_device *device) { @@ -261,10 +256,8 @@ isert_free_comps(struct isert_device *device) for (i = 0; i < device->comps_used; i++) { struct isert_comp *comp = &device->comps[i]; - if (comp->cq) { - cancel_work_sync(&comp->work); - ib_destroy_cq(comp->cq); - } + if (comp->cq) + ib_free_cq(comp->cq); } kfree(device->comps); } @@ -293,28 +286,17 @@ isert_alloc_comps(struct isert_device *device) max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); for (i = 0; i < device->comps_used; i++) { - struct ib_cq_init_attr cq_attr = {}; struct isert_comp *comp = &device->comps[i]; comp->device = device; - INIT_WORK(&comp->work, isert_cq_work); - cq_attr.cqe = max_cqe; - cq_attr.comp_vector = i; - comp->cq = ib_create_cq(device->ib_device, - isert_cq_callback, - isert_cq_event_callback, - (void *)comp, - &cq_attr); + comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, + IB_POLL_WORKQUEUE); if (IS_ERR(comp->cq)) { isert_err("Unable to allocate cq\n"); ret = PTR_ERR(comp->cq); comp->cq = NULL; goto out_cq; } - - ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); - if (ret) - goto out_cq; } return 0; @@ -582,7 +564,6 @@ isert_init_conn(struct isert_conn *isert_conn) INIT_LIST_HEAD(&isert_conn->node); init_completion(&isert_conn->login_comp); init_completion(&isert_conn->login_req_comp); - init_completion(&isert_conn->wait); kref_init(&isert_conn->kref); mutex_init(&isert_conn->mutex); spin_lock_init(&isert_conn->pool_lock); @@ -596,11 +577,13 @@ isert_free_login_buf(struct isert_conn *isert_conn) struct ib_device *ib_dev = isert_conn->device->ib_device; ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, - ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); + ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); + kfree(isert_conn->login_rsp_buf); + ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, - ISCSI_DEF_MAX_RECV_SEG_LEN, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); - kfree(isert_conn->login_buf); + kfree(isert_conn->login_req_buf); } static int @@ -609,50 +592,48 @@ isert_alloc_login_buf(struct isert_conn *isert_conn, { int ret; - isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + - ISER_RX_LOGIN_SIZE, GFP_KERNEL); - if (!isert_conn->login_buf) { + isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), + GFP_KERNEL); + if (!isert_conn->login_req_buf) { isert_err("Unable to allocate isert_conn->login_buf\n"); return -ENOMEM; } - isert_conn->login_req_buf = isert_conn->login_buf; - isert_conn->login_rsp_buf = isert_conn->login_buf + - ISCSI_DEF_MAX_RECV_SEG_LEN; - - isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", - isert_conn->login_buf, isert_conn->login_req_buf, - isert_conn->login_rsp_buf); - isert_conn->login_req_dma = ib_dma_map_single(ib_dev, - (void *)isert_conn->login_req_buf, - ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); - + isert_conn->login_req_buf, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); if (ret) { isert_err("login_req_dma mapping error: %d\n", ret); isert_conn->login_req_dma = 0; - goto out_login_buf; + goto out_free_login_req_buf; } - isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, - (void *)isert_conn->login_rsp_buf, - ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); + isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); + if (!isert_conn->login_rsp_buf) { + isert_err("Unable to allocate isert_conn->login_rspbuf\n"); + goto out_unmap_login_req_buf; + } + isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, + isert_conn->login_rsp_buf, + ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); if (ret) { isert_err("login_rsp_dma mapping error: %d\n", ret); isert_conn->login_rsp_dma = 0; - goto out_req_dma_map; + goto out_free_login_rsp_buf; } return 0; -out_req_dma_map: +out_free_login_rsp_buf: + kfree(isert_conn->login_rsp_buf); +out_unmap_login_req_buf: ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, - ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); -out_login_buf: - kfree(isert_conn->login_buf); + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); +out_free_login_req_buf: + kfree(isert_conn->login_req_buf); return ret; } @@ -726,7 +707,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) if (ret) goto out_conn_dev; - ret = isert_rdma_post_recvl(isert_conn); + ret = isert_login_post_recv(isert_conn); if (ret) goto out_conn_dev; @@ -773,7 +754,7 @@ isert_connect_release(struct isert_conn *isert_conn) ib_destroy_qp(isert_conn->qp); } - if (isert_conn->login_buf) + if (isert_conn->login_req_buf) isert_free_login_buf(isert_conn); isert_device_put(device); @@ -820,12 +801,30 @@ isert_put_conn(struct isert_conn *isert_conn) kref_put(&isert_conn->kref, isert_release_kref); } +static void +isert_handle_unbound_conn(struct isert_conn *isert_conn) +{ + struct isert_np *isert_np = isert_conn->cm_id->context; + + mutex_lock(&isert_np->mutex); + if (!list_empty(&isert_conn->node)) { + /* + * This means iscsi doesn't know this connection + * so schedule a cleanup ourselves + */ + list_del_init(&isert_conn->node); + isert_put_conn(isert_conn); + queue_work(isert_release_wq, &isert_conn->release_work); + } + mutex_unlock(&isert_np->mutex); +} + /** * isert_conn_terminate() - Initiate connection termination * @isert_conn: isert connection struct * * Notes: - * In case the connection state is FULL_FEATURE, move state + * In case the connection state is BOUND, move state * to TEMINATING and start teardown sequence (rdma_disconnect). * In case the connection state is UP, complete flush as well. * @@ -837,23 +836,16 @@ isert_conn_terminate(struct isert_conn *isert_conn) { int err; - switch (isert_conn->state) { - case ISER_CONN_TERMINATING: - break; - case ISER_CONN_UP: - case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ - isert_info("Terminating conn %p state %d\n", - isert_conn, isert_conn->state); - isert_conn->state = ISER_CONN_TERMINATING; - err = rdma_disconnect(isert_conn->cm_id); - if (err) - isert_warn("Failed rdma_disconnect isert_conn %p\n", - isert_conn); - break; - default: - isert_warn("conn %p teminating in state %d\n", - isert_conn, isert_conn->state); - } + if (isert_conn->state >= ISER_CONN_TERMINATING) + return; + + isert_info("Terminating conn %p state %d\n", + isert_conn, isert_conn->state); + isert_conn->state = ISER_CONN_TERMINATING; + err = rdma_disconnect(isert_conn->cm_id); + if (err) + isert_warn("Failed rdma_disconnect isert_conn %p\n", + isert_conn); } static int @@ -887,35 +879,27 @@ static int isert_disconnected_handler(struct rdma_cm_id *cma_id, enum rdma_cm_event_type event) { - struct isert_np *isert_np = cma_id->context; - struct isert_conn *isert_conn; - bool terminating = false; - - if (isert_np->cm_id == cma_id) - return isert_np_cma_handler(cma_id->context, event); - - isert_conn = cma_id->qp->qp_context; + struct isert_conn *isert_conn = cma_id->qp->qp_context; mutex_lock(&isert_conn->mutex); - terminating = (isert_conn->state == ISER_CONN_TERMINATING); - isert_conn_terminate(isert_conn); - mutex_unlock(&isert_conn->mutex); - - isert_info("conn %p completing wait\n", isert_conn); - complete(&isert_conn->wait); - - if (terminating) - goto out; - - mutex_lock(&isert_np->mutex); - if (!list_empty(&isert_conn->node)) { - list_del_init(&isert_conn->node); - isert_put_conn(isert_conn); - queue_work(isert_release_wq, &isert_conn->release_work); + switch (isert_conn->state) { + case ISER_CONN_TERMINATING: + break; + case ISER_CONN_UP: + isert_conn_terminate(isert_conn); + isert_wait4flush(isert_conn); + isert_handle_unbound_conn(isert_conn); + break; + case ISER_CONN_BOUND: + case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + break; + default: + isert_warn("conn %p teminating in state %d\n", + isert_conn, isert_conn->state); } - mutex_unlock(&isert_np->mutex); + mutex_unlock(&isert_conn->mutex); -out: return 0; } @@ -934,12 +918,16 @@ isert_connect_error(struct rdma_cm_id *cma_id) static int isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) { + struct isert_np *isert_np = cma_id->context; int ret = 0; isert_info("%s (%d): status %d id %p np %p\n", rdma_event_msg(event->event), event->event, event->status, cma_id, cma_id->context); + if (isert_np->cm_id == cma_id) + return isert_np_cma_handler(cma_id->context, event->event); + switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: ret = isert_connect_request(cma_id, event); @@ -977,7 +965,8 @@ isert_post_recvm(struct isert_conn *isert_conn, u32 count) for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { rx_desc = &isert_conn->rx_descs[i]; - rx_wr->wr_id = (uintptr_t)rx_desc; + + rx_wr->wr_cqe = &rx_desc->rx_cqe; rx_wr->sg_list = &rx_desc->rx_sg; rx_wr->num_sge = 1; rx_wr->next = rx_wr + 1; @@ -985,13 +974,10 @@ isert_post_recvm(struct isert_conn *isert_conn, u32 count) rx_wr--; rx_wr->next = NULL; /* mark end of work requests list */ - isert_conn->post_recv_buf_count += count; ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, &rx_wr_failed); - if (ret) { + if (ret) isert_err("ib_post_recv() failed with ret: %d\n", ret); - isert_conn->post_recv_buf_count -= count; - } return ret; } @@ -1002,23 +988,20 @@ isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) struct ib_recv_wr *rx_wr_failed, rx_wr; int ret; - rx_wr.wr_id = (uintptr_t)rx_desc; + rx_wr.wr_cqe = &rx_desc->rx_cqe; rx_wr.sg_list = &rx_desc->rx_sg; rx_wr.num_sge = 1; rx_wr.next = NULL; - isert_conn->post_recv_buf_count++; ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); - if (ret) { + if (ret) isert_err("ib_post_recv() failed with ret: %d\n", ret); - isert_conn->post_recv_buf_count--; - } return ret; } static int -isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) +isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) { struct ib_device *ib_dev = isert_conn->cm_id->device; struct ib_send_wr send_wr, *send_wr_failed; @@ -1027,8 +1010,10 @@ isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); + tx_desc->tx_cqe.done = isert_login_send_done; + send_wr.next = NULL; - send_wr.wr_id = (uintptr_t)tx_desc; + send_wr.wr_cqe = &tx_desc->tx_cqe; send_wr.sg_list = tx_desc->tx_sg; send_wr.num_sge = tx_desc->num_sge; send_wr.opcode = IB_WR_SEND; @@ -1056,7 +1041,6 @@ isert_create_send_desc(struct isert_conn *isert_conn, tx_desc->iser_header.flags = ISCSI_CTRL; tx_desc->num_sge = 1; - tx_desc->isert_cmd = isert_cmd; if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; @@ -1097,8 +1081,9 @@ isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, { struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; - isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; - send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; + isert_cmd->iser_ib_op = ISER_IB_SEND; + tx_desc->tx_cqe.done = isert_send_done; + send_wr->wr_cqe = &tx_desc->tx_cqe; if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { send_wr->opcode = IB_WR_SEND_WITH_INV; @@ -1113,7 +1098,7 @@ isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, } static int -isert_rdma_post_recvl(struct isert_conn *isert_conn) +isert_login_post_recv(struct isert_conn *isert_conn) { struct ib_recv_wr rx_wr, *rx_wr_fail; struct ib_sge sge; @@ -1121,23 +1106,22 @@ isert_rdma_post_recvl(struct isert_conn *isert_conn) memset(&sge, 0, sizeof(struct ib_sge)); sge.addr = isert_conn->login_req_dma; - sge.length = ISER_RX_LOGIN_SIZE; + sge.length = ISER_RX_PAYLOAD_SIZE; sge.lkey = isert_conn->device->pd->local_dma_lkey; isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", sge.addr, sge.length, sge.lkey); + isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; + memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); - rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; + rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; rx_wr.sg_list = &sge; rx_wr.num_sge = 1; - isert_conn->post_recv_buf_count++; ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); - if (ret) { + if (ret) isert_err("ib_post_recv() failed: %d\n", ret); - isert_conn->post_recv_buf_count--; - } return ret; } @@ -1203,12 +1187,12 @@ isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, goto post_send; } - ret = isert_rdma_post_recvl(isert_conn); + ret = isert_login_post_recv(isert_conn); if (ret) return ret; } post_send: - ret = isert_post_send(isert_conn, tx_desc); + ret = isert_login_post_send(isert_conn, tx_desc); if (ret) return ret; @@ -1218,7 +1202,7 @@ isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, static void isert_rx_login_req(struct isert_conn *isert_conn) { - struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; + struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; int rx_buflen = isert_conn->login_req_len; struct iscsi_conn *conn = isert_conn->conn; struct iscsi_login *login = conn->conn_login; @@ -1551,12 +1535,42 @@ isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, } static void -isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) +isert_print_wc(struct ib_wc *wc, const char *type) +{ + if (wc->status != IB_WC_WR_FLUSH_ERR) + isert_err("%s failure: %s (%d) vend_err %x\n", type, + ib_wc_status_msg(wc->status), wc->status, + wc->vendor_err); + else + isert_dbg("%s failure: %s (%d)\n", type, + ib_wc_status_msg(wc->status), wc->status); +} + +static void +isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) { + struct isert_conn *isert_conn = wc->qp->qp_context; + struct ib_device *ib_dev = isert_conn->cm_id->device; + struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); + struct iscsi_hdr *hdr = &rx_desc->iscsi_header; struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; uint64_t read_va = 0, write_va = 0; uint32_t read_stag = 0, write_stag = 0; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "recv"); + if (wc->status != IB_WC_WR_FLUSH_ERR) + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + return; + } + + ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); + + isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", + rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, + (int)(wc->byte_len - ISER_HEADERS_LEN)); + switch (iser_ctrl->flags & 0xF0) { case ISCSI_CTRL: if (iser_ctrl->flags & ISER_RSV) { @@ -1584,56 +1598,40 @@ isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) isert_rx_opcode(isert_conn, rx_desc, read_stag, read_va, write_stag, write_va); + + ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); } static void -isert_rcv_completion(struct iser_rx_desc *desc, - struct isert_conn *isert_conn, - u32 xfer_len) +isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) { + struct isert_conn *isert_conn = wc->qp->qp_context; struct ib_device *ib_dev = isert_conn->cm_id->device; - struct iscsi_hdr *hdr; - u64 rx_dma; - int rx_buflen; - - if ((char *)desc == isert_conn->login_req_buf) { - rx_dma = isert_conn->login_req_dma; - rx_buflen = ISER_RX_LOGIN_SIZE; - isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", - rx_dma, rx_buflen); - } else { - rx_dma = desc->dma_addr; - rx_buflen = ISER_RX_PAYLOAD_SIZE; - isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", - rx_dma, rx_buflen); + + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "login recv"); + return; } - ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); + ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); - hdr = &desc->iscsi_header; - isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", - hdr->opcode, hdr->itt, hdr->flags, - (int)(xfer_len - ISER_HEADERS_LEN)); + isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; - if ((char *)desc == isert_conn->login_req_buf) { - isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; - if (isert_conn->conn) { - struct iscsi_login *login = isert_conn->conn->conn_login; + if (isert_conn->conn) { + struct iscsi_login *login = isert_conn->conn->conn_login; - if (login && !login->first_request) - isert_rx_login_req(isert_conn); - } - mutex_lock(&isert_conn->mutex); - complete(&isert_conn->login_req_comp); - mutex_unlock(&isert_conn->mutex); - } else { - isert_rx_do_work(desc, isert_conn); + if (login && !login->first_request) + isert_rx_login_req(isert_conn); } - ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, - DMA_FROM_DEVICE); + mutex_lock(&isert_conn->mutex); + complete(&isert_conn->login_req_comp); + mutex_unlock(&isert_conn->mutex); - isert_conn->post_recv_buf_count--; + ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); } static int @@ -1683,54 +1681,50 @@ isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) static void isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) { - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; - isert_dbg("Cmd %p\n", isert_cmd); - if (wr->data.sg) { + if (isert_cmd->data.sg) { isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); - isert_unmap_data_buf(isert_conn, &wr->data); + isert_unmap_data_buf(isert_conn, &isert_cmd->data); } - if (wr->rdma_wr) { + if (isert_cmd->rdma_wr) { isert_dbg("Cmd %p free send_wr\n", isert_cmd); - kfree(wr->rdma_wr); - wr->rdma_wr = NULL; + kfree(isert_cmd->rdma_wr); + isert_cmd->rdma_wr = NULL; } - if (wr->ib_sge) { + if (isert_cmd->ib_sge) { isert_dbg("Cmd %p free ib_sge\n", isert_cmd); - kfree(wr->ib_sge); - wr->ib_sge = NULL; + kfree(isert_cmd->ib_sge); + isert_cmd->ib_sge = NULL; } } static void isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) { - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; - isert_dbg("Cmd %p\n", isert_cmd); - if (wr->fr_desc) { - isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); - if (wr->fr_desc->ind & ISERT_PROTECTED) { - isert_unmap_data_buf(isert_conn, &wr->prot); - wr->fr_desc->ind &= ~ISERT_PROTECTED; + if (isert_cmd->fr_desc) { + isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, isert_cmd->fr_desc); + if (isert_cmd->fr_desc->ind & ISERT_PROTECTED) { + isert_unmap_data_buf(isert_conn, &isert_cmd->prot); + isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; } spin_lock_bh(&isert_conn->pool_lock); - list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); + list_add_tail(&isert_cmd->fr_desc->list, &isert_conn->fr_pool); spin_unlock_bh(&isert_conn->pool_lock); - wr->fr_desc = NULL; + isert_cmd->fr_desc = NULL; } - if (wr->data.sg) { + if (isert_cmd->data.sg) { isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); - isert_unmap_data_buf(isert_conn, &wr->data); + isert_unmap_data_buf(isert_conn, &isert_cmd->data); } - wr->ib_sge = NULL; - wr->rdma_wr = NULL; + isert_cmd->ib_sge = NULL; + isert_cmd->rdma_wr = NULL; } static void @@ -1882,52 +1876,70 @@ isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) } static void -isert_completion_rdma_write(struct iser_tx_desc *tx_desc, - struct isert_cmd *isert_cmd) +isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) { - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; - struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; - struct se_cmd *se_cmd = &cmd->se_cmd; - struct isert_conn *isert_conn = isert_cmd->conn; + struct isert_conn *isert_conn = wc->qp->qp_context; struct isert_device *device = isert_conn->device; + struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); + struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); + struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; int ret = 0; - if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { - ret = isert_check_pi_status(se_cmd, - wr->fr_desc->pi_ctx->sig_mr); - wr->fr_desc->ind &= ~ISERT_PROTECTED; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "rdma write"); + if (wc->status != IB_WC_WR_FLUSH_ERR) + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + isert_completion_put(desc, isert_cmd, device->ib_device, true); + return; + } + + isert_dbg("Cmd %p\n", isert_cmd); + + if (isert_cmd->fr_desc && isert_cmd->fr_desc->ind & ISERT_PROTECTED) { + ret = isert_check_pi_status(cmd, + isert_cmd->fr_desc->pi_ctx->sig_mr); + isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; } device->unreg_rdma_mem(isert_cmd, isert_conn); - wr->rdma_wr_num = 0; + isert_cmd->rdma_wr_num = 0; if (ret) - transport_send_check_condition_and_sense(se_cmd, - se_cmd->pi_err, 0); + transport_send_check_condition_and_sense(cmd, cmd->pi_err, 0); else - isert_put_response(isert_conn->conn, cmd); + isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); } static void -isert_completion_rdma_read(struct iser_tx_desc *tx_desc, - struct isert_cmd *isert_cmd) +isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) { - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; + struct isert_conn *isert_conn = wc->qp->qp_context; + struct isert_device *device = isert_conn->device; + struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); + struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; struct se_cmd *se_cmd = &cmd->se_cmd; - struct isert_conn *isert_conn = isert_cmd->conn; - struct isert_device *device = isert_conn->device; int ret = 0; - if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "rdma read"); + if (wc->status != IB_WC_WR_FLUSH_ERR) + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + isert_completion_put(desc, isert_cmd, device->ib_device, true); + return; + } + + isert_dbg("Cmd %p\n", isert_cmd); + + if (isert_cmd->fr_desc && isert_cmd->fr_desc->ind & ISERT_PROTECTED) { ret = isert_check_pi_status(se_cmd, - wr->fr_desc->pi_ctx->sig_mr); - wr->fr_desc->ind &= ~ISERT_PROTECTED; + isert_cmd->fr_desc->pi_ctx->sig_mr); + isert_cmd->fr_desc->ind &= ~ISERT_PROTECTED; } iscsit_stop_dataout_timer(cmd); device->unreg_rdma_mem(isert_cmd, isert_conn); - cmd->write_data_done = wr->data.len; - wr->rdma_wr_num = 0; + cmd->write_data_done = isert_cmd->data.len; + isert_cmd->rdma_wr_num = 0; isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); spin_lock_bh(&cmd->istate_lock); @@ -1975,170 +1987,56 @@ isert_do_control_comp(struct work_struct *work) } static void -isert_response_completion(struct iser_tx_desc *tx_desc, - struct isert_cmd *isert_cmd, - struct isert_conn *isert_conn, - struct ib_device *ib_dev) +isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) { - struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; - - if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || - cmd->i_state == ISTATE_SEND_LOGOUTRSP || - cmd->i_state == ISTATE_SEND_REJECT || - cmd->i_state == ISTATE_SEND_TEXTRSP) { - isert_unmap_tx_desc(tx_desc, ib_dev); + struct isert_conn *isert_conn = wc->qp->qp_context; + struct ib_device *ib_dev = isert_conn->cm_id->device; + struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); - INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); - queue_work(isert_comp_wq, &isert_cmd->comp_work); - return; + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "login send"); + if (wc->status != IB_WC_WR_FLUSH_ERR) + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); } - cmd->i_state = ISTATE_SENT_STATUS; - isert_completion_put(tx_desc, isert_cmd, ib_dev, false); + isert_unmap_tx_desc(tx_desc, ib_dev); } static void -isert_snd_completion(struct iser_tx_desc *tx_desc, - struct isert_conn *isert_conn) +isert_send_done(struct ib_cq *cq, struct ib_wc *wc) { + struct isert_conn *isert_conn = wc->qp->qp_context; struct ib_device *ib_dev = isert_conn->cm_id->device; - struct isert_cmd *isert_cmd = tx_desc->isert_cmd; - struct isert_rdma_wr *wr; + struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); + struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); - if (!isert_cmd) { - isert_unmap_tx_desc(tx_desc, ib_dev); + if (unlikely(wc->status != IB_WC_SUCCESS)) { + isert_print_wc(wc, "send"); + if (wc->status != IB_WC_WR_FLUSH_ERR) + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + isert_completion_put(tx_desc, isert_cmd, ib_dev, true); return; } - wr = &isert_cmd->rdma_wr; - isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); + isert_dbg("Cmd %p\n", isert_cmd); - switch (wr->iser_ib_op) { - case ISER_IB_SEND: - isert_response_completion(tx_desc, isert_cmd, - isert_conn, ib_dev); - break; - case ISER_IB_RDMA_WRITE: - isert_completion_rdma_write(tx_desc, isert_cmd); - break; - case ISER_IB_RDMA_READ: - isert_completion_rdma_read(tx_desc, isert_cmd); - break; + switch (isert_cmd->iscsi_cmd->i_state) { + case ISTATE_SEND_TASKMGTRSP: + case ISTATE_SEND_LOGOUTRSP: + case ISTATE_SEND_REJECT: + case ISTATE_SEND_TEXTRSP: + isert_unmap_tx_desc(tx_desc, ib_dev); + + INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); + queue_work(isert_comp_wq, &isert_cmd->comp_work); + return; default: - isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); - dump_stack(); + isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; + isert_completion_put(tx_desc, isert_cmd, ib_dev, false); break; } } -/** - * is_isert_tx_desc() - Indicate if the completion wr_id - * is a TX descriptor or not. - * @isert_conn: iser connection - * @wr_id: completion WR identifier - * - * Since we cannot rely on wc opcode in FLUSH errors - * we must work around it by checking if the wr_id address - * falls in the iser connection rx_descs buffer. If so - * it is an RX descriptor, otherwize it is a TX. - */ -static inline bool -is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) -{ - void *start = isert_conn->rx_descs; - int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); - - if (wr_id >= start && wr_id < start + len) - return false; - - return true; -} - -static void -isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) -{ - if (wc->wr_id == ISER_BEACON_WRID) { - isert_info("conn %p completing wait_comp_err\n", - isert_conn); - complete(&isert_conn->wait_comp_err); - } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { - struct ib_device *ib_dev = isert_conn->cm_id->device; - struct isert_cmd *isert_cmd; - struct iser_tx_desc *desc; - - desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; - isert_cmd = desc->isert_cmd; - if (!isert_cmd) - isert_unmap_tx_desc(desc, ib_dev); - else - isert_completion_put(desc, isert_cmd, ib_dev, true); - } else { - isert_conn->post_recv_buf_count--; - if (!isert_conn->post_recv_buf_count) - iscsit_cause_connection_reinstatement(isert_conn->conn, 0); - } -} - -static void -isert_handle_wc(struct ib_wc *wc) -{ - struct isert_conn *isert_conn; - struct iser_tx_desc *tx_desc; - struct iser_rx_desc *rx_desc; - - isert_conn = wc->qp->qp_context; - if (likely(wc->status == IB_WC_SUCCESS)) { - if (wc->opcode == IB_WC_RECV) { - rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; - isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); - } else { - tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; - isert_snd_completion(tx_desc, isert_conn); - } - } else { - if (wc->status != IB_WC_WR_FLUSH_ERR) - isert_err("%s (%d): wr id %llx vend_err %x\n", - ib_wc_status_msg(wc->status), wc->status, - wc->wr_id, wc->vendor_err); - else - isert_dbg("%s (%d): wr id %llx\n", - ib_wc_status_msg(wc->status), wc->status, - wc->wr_id); - - if (wc->wr_id != ISER_FASTREG_LI_WRID) - isert_cq_comp_err(isert_conn, wc); - } -} - -static void -isert_cq_work(struct work_struct *work) -{ - enum { isert_poll_budget = 65536 }; - struct isert_comp *comp = container_of(work, struct isert_comp, - work); - struct ib_wc *const wcs = comp->wcs; - int i, n, completed = 0; - - while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { - for (i = 0; i < n; i++) - isert_handle_wc(&wcs[i]); - - completed += n; - if (completed >= isert_poll_budget) - break; - } - - ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); -} - -static void -isert_cq_callback(struct ib_cq *cq, void *context) -{ - struct isert_comp *comp = context; - - queue_work(isert_comp_wq, &comp->work); -} - static int isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) { @@ -2395,7 +2293,8 @@ isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, page_off = offset % PAGE_SIZE; rdma_wr->wr.sg_list = ib_sge; - rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; + rdma_wr->wr.wr_cqe = &isert_cmd->tx_desc.tx_cqe; + /* * Perform mapping of TCM scatterlist memory ib_sge dma_addr. */ @@ -2428,24 +2327,23 @@ isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, } static int -isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, - struct isert_rdma_wr *wr) +isert_map_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn) { + struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; struct se_cmd *se_cmd = &cmd->se_cmd; - struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); struct isert_conn *isert_conn = conn->context; - struct isert_data_buf *data = &wr->data; + struct isert_data_buf *data = &isert_cmd->data; struct ib_rdma_wr *rdma_wr; struct ib_sge *ib_sge; u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; int ret = 0, i, ib_sge_cnt; - isert_cmd->tx_desc.isert_cmd = isert_cmd; - - offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; + offset = isert_cmd->iser_ib_op == ISER_IB_RDMA_READ ? + cmd->write_data_done : 0; ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, se_cmd->t_data_nents, se_cmd->data_length, - offset, wr->iser_ib_op, &wr->data); + offset, isert_cmd->iser_ib_op, + &isert_cmd->data); if (ret) return ret; @@ -2458,41 +2356,44 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, ret = -ENOMEM; goto unmap_cmd; } - wr->ib_sge = ib_sge; + isert_cmd->ib_sge = ib_sge; - wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); - wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num, - GFP_KERNEL); - if (!wr->rdma_wr) { - isert_dbg("Unable to allocate wr->rdma_wr\n"); + isert_cmd->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); + isert_cmd->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * + isert_cmd->rdma_wr_num, GFP_KERNEL); + if (!isert_cmd->rdma_wr) { + isert_dbg("Unable to allocate isert_cmd->rdma_wr\n"); ret = -ENOMEM; goto unmap_cmd; } - wr->isert_cmd = isert_cmd; rdma_write_max = isert_conn->max_sge * PAGE_SIZE; - for (i = 0; i < wr->rdma_wr_num; i++) { - rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i]; + for (i = 0; i < isert_cmd->rdma_wr_num; i++) { + rdma_wr = &isert_cmd->rdma_wr[i]; data_len = min(data_left, rdma_write_max); rdma_wr->wr.send_flags = 0; - if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { + if (isert_cmd->iser_ib_op == ISER_IB_RDMA_WRITE) { + isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; + rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; rdma_wr->remote_addr = isert_cmd->read_va + offset; rdma_wr->rkey = isert_cmd->read_stag; - if (i + 1 == wr->rdma_wr_num) + if (i + 1 == isert_cmd->rdma_wr_num) rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr; else - rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; + rdma_wr->wr.next = &isert_cmd->rdma_wr[i + 1].wr; } else { + isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; + rdma_wr->wr.opcode = IB_WR_RDMA_READ; rdma_wr->remote_addr = isert_cmd->write_va + va_offset; rdma_wr->rkey = isert_cmd->write_stag; - if (i + 1 == wr->rdma_wr_num) + if (i + 1 == isert_cmd->rdma_wr_num) rdma_wr->wr.send_flags = IB_SEND_SIGNALED; else - rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; + rdma_wr->wr.next = &isert_cmd->rdma_wr[i + 1].wr; } ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, @@ -2517,7 +2418,7 @@ isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) u32 rkey; memset(inv_wr, 0, sizeof(*inv_wr)); - inv_wr->wr_id = ISER_FASTREG_LI_WRID; + inv_wr->wr_cqe = NULL; inv_wr->opcode = IB_WR_LOCAL_INV; inv_wr->ex.invalidate_rkey = mr->rkey; @@ -2573,7 +2474,7 @@ isert_fast_reg_mr(struct isert_conn *isert_conn, reg_wr.wr.next = NULL; reg_wr.wr.opcode = IB_WR_REG_MR; - reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID; + reg_wr.wr.wr_cqe = NULL; reg_wr.wr.send_flags = 0; reg_wr.wr.num_sge = 0; reg_wr.mr = mr; @@ -2660,10 +2561,10 @@ isert_set_prot_checks(u8 prot_checks) static int isert_reg_sig_mr(struct isert_conn *isert_conn, - struct se_cmd *se_cmd, - struct isert_rdma_wr *rdma_wr, + struct isert_cmd *isert_cmd, struct fast_reg_descriptor *fr_desc) { + struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; struct ib_sig_handover_wr sig_wr; struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; struct pi_context *pi_ctx = fr_desc->pi_ctx; @@ -2684,14 +2585,14 @@ isert_reg_sig_mr(struct isert_conn *isert_conn, memset(&sig_wr, 0, sizeof(sig_wr)); sig_wr.wr.opcode = IB_WR_REG_SIG_MR; - sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID; - sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA]; + sig_wr.wr.wr_cqe = NULL; + sig_wr.wr.sg_list = &isert_cmd->ib_sg[DATA]; sig_wr.wr.num_sge = 1; sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; sig_wr.sig_attrs = &sig_attrs; sig_wr.sig_mr = pi_ctx->sig_mr; if (se_cmd->t_prot_sg) - sig_wr.prot = &rdma_wr->ib_sg[PROT]; + sig_wr.prot = &isert_cmd->ib_sg[PROT]; if (!wr) wr = &sig_wr.wr; @@ -2705,35 +2606,34 @@ isert_reg_sig_mr(struct isert_conn *isert_conn, } fr_desc->ind &= ~ISERT_SIG_KEY_VALID; - rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; - rdma_wr->ib_sg[SIG].addr = 0; - rdma_wr->ib_sg[SIG].length = se_cmd->data_length; + isert_cmd->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; + isert_cmd->ib_sg[SIG].addr = 0; + isert_cmd->ib_sg[SIG].length = se_cmd->data_length; if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) /* * We have protection guards on the wire * so we need to set a larget transfer */ - rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; + isert_cmd->ib_sg[SIG].length += se_cmd->prot_length; isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", - rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, - rdma_wr->ib_sg[SIG].lkey); + isert_cmd->ib_sg[SIG].addr, isert_cmd->ib_sg[SIG].length, + isert_cmd->ib_sg[SIG].lkey); err: return ret; } static int isert_handle_prot_cmd(struct isert_conn *isert_conn, - struct isert_cmd *isert_cmd, - struct isert_rdma_wr *wr) + struct isert_cmd *isert_cmd) { struct isert_device *device = isert_conn->device; struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; int ret; - if (!wr->fr_desc->pi_ctx) { - ret = isert_create_pi_ctx(wr->fr_desc, + if (!isert_cmd->fr_desc->pi_ctx) { + ret = isert_create_pi_ctx(isert_cmd->fr_desc, device->ib_device, device->pd); if (ret) { @@ -2748,16 +2648,20 @@ isert_handle_prot_cmd(struct isert_conn *isert_conn, se_cmd->t_prot_sg, se_cmd->t_prot_nents, se_cmd->prot_length, - 0, wr->iser_ib_op, &wr->prot); + 0, + isert_cmd->iser_ib_op, + &isert_cmd->prot); if (ret) { isert_err("conn %p failed to map protection buffer\n", isert_conn); return ret; } - memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); - ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, - ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); + memset(&isert_cmd->ib_sg[PROT], 0, sizeof(isert_cmd->ib_sg[PROT])); + ret = isert_fast_reg_mr(isert_conn, isert_cmd->fr_desc, + &isert_cmd->prot, + ISERT_PROT_KEY_VALID, + &isert_cmd->ib_sg[PROT]); if (ret) { isert_err("conn %p failed to fast reg mr\n", isert_conn); @@ -2765,29 +2669,28 @@ isert_handle_prot_cmd(struct isert_conn *isert_conn, } } - ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); + ret = isert_reg_sig_mr(isert_conn, isert_cmd, isert_cmd->fr_desc); if (ret) { isert_err("conn %p failed to fast reg mr\n", isert_conn); goto unmap_prot_cmd; } - wr->fr_desc->ind |= ISERT_PROTECTED; + isert_cmd->fr_desc->ind |= ISERT_PROTECTED; return 0; unmap_prot_cmd: if (se_cmd->t_prot_sg) - isert_unmap_data_buf(isert_conn, &wr->prot); + isert_unmap_data_buf(isert_conn, &isert_cmd->prot); return ret; } static int -isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, - struct isert_rdma_wr *wr) +isert_reg_rdma(struct isert_cmd *isert_cmd, struct iscsi_conn *conn) { + struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; struct se_cmd *se_cmd = &cmd->se_cmd; - struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); struct isert_conn *isert_conn = conn->context; struct fast_reg_descriptor *fr_desc = NULL; struct ib_rdma_wr *rdma_wr; @@ -2796,57 +2699,61 @@ isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int ret = 0; unsigned long flags; - isert_cmd->tx_desc.isert_cmd = isert_cmd; - - offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; + offset = isert_cmd->iser_ib_op == ISER_IB_RDMA_READ ? + cmd->write_data_done : 0; ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, se_cmd->t_data_nents, se_cmd->data_length, - offset, wr->iser_ib_op, &wr->data); + offset, isert_cmd->iser_ib_op, + &isert_cmd->data); if (ret) return ret; - if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { + if (isert_cmd->data.dma_nents != 1 || + isert_prot_cmd(isert_conn, se_cmd)) { spin_lock_irqsave(&isert_conn->pool_lock, flags); fr_desc = list_first_entry(&isert_conn->fr_pool, struct fast_reg_descriptor, list); list_del(&fr_desc->list); spin_unlock_irqrestore(&isert_conn->pool_lock, flags); - wr->fr_desc = fr_desc; + isert_cmd->fr_desc = fr_desc; } - ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, - ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); + ret = isert_fast_reg_mr(isert_conn, fr_desc, &isert_cmd->data, + ISERT_DATA_KEY_VALID, &isert_cmd->ib_sg[DATA]); if (ret) goto unmap_cmd; if (isert_prot_cmd(isert_conn, se_cmd)) { - ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); + ret = isert_handle_prot_cmd(isert_conn, isert_cmd); if (ret) goto unmap_cmd; - ib_sg = &wr->ib_sg[SIG]; + ib_sg = &isert_cmd->ib_sg[SIG]; } else { - ib_sg = &wr->ib_sg[DATA]; + ib_sg = &isert_cmd->ib_sg[DATA]; } - memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); - wr->ib_sge = &wr->s_ib_sge; - wr->rdma_wr_num = 1; - memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr)); - wr->rdma_wr = &wr->s_rdma_wr; - wr->isert_cmd = isert_cmd; + memcpy(&isert_cmd->s_ib_sge, ib_sg, sizeof(*ib_sg)); + isert_cmd->ib_sge = &isert_cmd->s_ib_sge; + isert_cmd->rdma_wr_num = 1; + memset(&isert_cmd->s_rdma_wr, 0, sizeof(isert_cmd->s_rdma_wr)); + isert_cmd->rdma_wr = &isert_cmd->s_rdma_wr; - rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr; - rdma_wr->wr.sg_list = &wr->s_ib_sge; + rdma_wr = &isert_cmd->s_rdma_wr; + rdma_wr->wr.sg_list = &isert_cmd->s_ib_sge; rdma_wr->wr.num_sge = 1; - rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; - if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { + rdma_wr->wr.wr_cqe = &isert_cmd->tx_desc.tx_cqe; + if (isert_cmd->iser_ib_op == ISER_IB_RDMA_WRITE) { + isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; + rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; rdma_wr->remote_addr = isert_cmd->read_va; rdma_wr->rkey = isert_cmd->read_stag; rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 0 : IB_SEND_SIGNALED; } else { + isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; + rdma_wr->wr.opcode = IB_WR_RDMA_READ; rdma_wr->remote_addr = isert_cmd->write_va; rdma_wr->rkey = isert_cmd->write_stag; @@ -2861,7 +2768,7 @@ isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, list_add_tail(&fr_desc->list, &isert_conn->fr_pool); spin_unlock_irqrestore(&isert_conn->pool_lock, flags); } - isert_unmap_data_buf(isert_conn, &wr->data); + isert_unmap_data_buf(isert_conn, &isert_cmd->data); return ret; } @@ -2871,7 +2778,6 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) { struct se_cmd *se_cmd = &cmd->se_cmd; struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; struct isert_conn *isert_conn = conn->context; struct isert_device *device = isert_conn->device; struct ib_send_wr *wr_failed; @@ -2880,8 +2786,8 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", isert_cmd, se_cmd->data_length); - wr->iser_ib_op = ISER_IB_RDMA_WRITE; - rc = device->reg_rdma_mem(conn, cmd, wr); + isert_cmd->iser_ib_op = ISER_IB_RDMA_WRITE; + rc = device->reg_rdma_mem(isert_cmd, conn); if (rc) { isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); return rc; @@ -2898,8 +2804,8 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); isert_init_send_wr(isert_conn, isert_cmd, &isert_cmd->tx_desc.send_wr); - isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; - wr->rdma_wr_num += 1; + isert_cmd->s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; + isert_cmd->rdma_wr_num += 1; rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); if (rc) { @@ -2908,7 +2814,7 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) } } - rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); + rc = ib_post_send(isert_conn->qp, &isert_cmd->rdma_wr->wr, &wr_failed); if (rc) isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); @@ -2927,7 +2833,6 @@ isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) { struct se_cmd *se_cmd = &cmd->se_cmd; struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); - struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; struct isert_conn *isert_conn = conn->context; struct isert_device *device = isert_conn->device; struct ib_send_wr *wr_failed; @@ -2935,14 +2840,14 @@ isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", isert_cmd, se_cmd->data_length, cmd->write_data_done); - wr->iser_ib_op = ISER_IB_RDMA_READ; - rc = device->reg_rdma_mem(conn, cmd, wr); + isert_cmd->iser_ib_op = ISER_IB_RDMA_READ; + rc = device->reg_rdma_mem(isert_cmd, conn); if (rc) { isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); return rc; } - rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); + rc = ib_post_send(isert_conn->qp, &isert_cmd->rdma_wr->wr, &wr_failed); if (rc) isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); @@ -3214,6 +3119,7 @@ isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) conn->context = isert_conn; isert_conn->conn = conn; + isert_conn->state = ISER_CONN_BOUND; isert_set_conn_info(np, conn, isert_conn); @@ -3274,8 +3180,6 @@ static void isert_release_work(struct work_struct *work) isert_info("Starting release conn %p\n", isert_conn); - wait_for_completion(&isert_conn->wait); - mutex_lock(&isert_conn->mutex); isert_conn->state = ISER_CONN_DOWN; mutex_unlock(&isert_conn->mutex); @@ -3309,15 +3213,27 @@ isert_wait4cmds(struct iscsi_conn *conn) } } +static void +isert_beacon_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct isert_conn *isert_conn = wc->qp->qp_context; + + isert_print_wc(wc, "beacon"); + + isert_info("conn %p completing wait_comp_err\n", isert_conn); + complete(&isert_conn->wait_comp_err); +} + static void isert_wait4flush(struct isert_conn *isert_conn) { struct ib_recv_wr *bad_wr; + static struct ib_cqe cqe = { .done = isert_beacon_done }; isert_info("conn %p\n", isert_conn); init_completion(&isert_conn->wait_comp_err); - isert_conn->beacon.wr_id = ISER_BEACON_WRID; + isert_conn->beacon.wr_cqe = &cqe; /* post an indication that all flush errors were consumed */ if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { isert_err("conn %p failed to post beacon", isert_conn); @@ -3369,14 +3285,6 @@ static void isert_wait_conn(struct iscsi_conn *conn) isert_info("Starting conn %p\n", isert_conn); mutex_lock(&isert_conn->mutex); - /* - * Only wait for wait_comp_err if the isert_conn made it - * into full feature phase.. - */ - if (isert_conn->state == ISER_CONN_INIT) { - mutex_unlock(&isert_conn->mutex); - return; - } isert_conn_terminate(isert_conn); mutex_unlock(&isert_conn->mutex); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index 8d50453eef66ce..192788a4820cde 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -36,9 +36,7 @@ /* Constant PDU lengths calculations */ #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \ sizeof(struct iscsi_hdr)) -#define ISER_RECV_DATA_SEG_LEN 8192 -#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN) -#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) +#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) /* QP settings */ /* Maximal bounds on received asynchronous PDUs */ @@ -62,12 +60,11 @@ ISERT_MAX_TX_MISC_PDUS + \ ISERT_MAX_RX_MISC_PDUS) -#define ISER_RX_PAD_SIZE (ISER_RECV_DATA_SEG_LEN + 4096 - \ - (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge))) +#define ISER_RX_PAD_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 4096 - \ + (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \ + sizeof(struct ib_cqe))) #define ISCSI_ISER_SG_TABLESIZE 256 -#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL -#define ISER_BEACON_WRID 0xfffffffffffffffeULL enum isert_desc_type { ISCSI_TX_CONTROL, @@ -84,6 +81,7 @@ enum iser_ib_op_code { enum iser_conn_state { ISER_CONN_INIT, ISER_CONN_UP, + ISER_CONN_BOUND, ISER_CONN_FULL_FEATURE, ISER_CONN_TERMINATING, ISER_CONN_DOWN, @@ -92,23 +90,35 @@ enum iser_conn_state { struct iser_rx_desc { struct iser_ctrl iser_header; struct iscsi_hdr iscsi_header; - char data[ISER_RECV_DATA_SEG_LEN]; + char data[ISCSI_DEF_MAX_RECV_SEG_LEN]; u64 dma_addr; struct ib_sge rx_sg; + struct ib_cqe rx_cqe; char pad[ISER_RX_PAD_SIZE]; } __packed; +static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe) +{ + return container_of(cqe, struct iser_rx_desc, rx_cqe); +} + struct iser_tx_desc { struct iser_ctrl iser_header; struct iscsi_hdr iscsi_header; enum isert_desc_type type; u64 dma_addr; struct ib_sge tx_sg[2]; + struct ib_cqe tx_cqe; int num_sge; - struct isert_cmd *isert_cmd; struct ib_send_wr send_wr; } __packed; +static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe) +{ + return container_of(cqe, struct iser_tx_desc, tx_cqe); +} + + enum isert_indicator { ISERT_PROTECTED = 1 << 0, ISERT_DATA_KEY_VALID = 1 << 1, @@ -144,20 +154,6 @@ enum { SIG = 2, }; -struct isert_rdma_wr { - struct isert_cmd *isert_cmd; - enum iser_ib_op_code iser_ib_op; - struct ib_sge *ib_sge; - struct ib_sge s_ib_sge; - int rdma_wr_num; - struct ib_rdma_wr *rdma_wr; - struct ib_rdma_wr s_rdma_wr; - struct ib_sge ib_sg[3]; - struct isert_data_buf data; - struct isert_data_buf prot; - struct fast_reg_descriptor *fr_desc; -}; - struct isert_cmd { uint32_t read_stag; uint32_t write_stag; @@ -170,22 +166,34 @@ struct isert_cmd { struct iscsi_cmd *iscsi_cmd; struct iser_tx_desc tx_desc; struct iser_rx_desc *rx_desc; - struct isert_rdma_wr rdma_wr; + enum iser_ib_op_code iser_ib_op; + struct ib_sge *ib_sge; + struct ib_sge s_ib_sge; + int rdma_wr_num; + struct ib_rdma_wr *rdma_wr; + struct ib_rdma_wr s_rdma_wr; + struct ib_sge ib_sg[3]; + struct isert_data_buf data; + struct isert_data_buf prot; + struct fast_reg_descriptor *fr_desc; struct work_struct comp_work; struct scatterlist sg; }; +static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) +{ + return container_of(desc, struct isert_cmd, tx_desc); +} + struct isert_device; struct isert_conn { enum iser_conn_state state; - int post_recv_buf_count; u32 responder_resources; u32 initiator_depth; bool pi_support; u32 max_sge; - char *login_buf; - char *login_req_buf; + struct iser_rx_desc *login_req_buf; char *login_rsp_buf; u64 login_req_dma; int login_req_len; @@ -201,7 +209,6 @@ struct isert_conn { struct ib_qp *qp; struct isert_device *device; struct mutex mutex; - struct completion wait; struct completion wait_comp_err; struct kref kref; struct list_head fr_pool; @@ -221,17 +228,13 @@ struct isert_conn { * * @device: pointer to device handle * @cq: completion queue - * @wcs: work completion array * @active_qps: Number of active QPs attached * to completion context - * @work: completion work handle */ struct isert_comp { struct isert_device *device; struct ib_cq *cq; - struct ib_wc wcs[16]; int active_qps; - struct work_struct work; }; struct isert_device { @@ -243,9 +246,8 @@ struct isert_device { struct isert_comp *comps; int comps_used; struct list_head dev_node; - int (*reg_rdma_mem)(struct iscsi_conn *conn, - struct iscsi_cmd *cmd, - struct isert_rdma_wr *wr); + int (*reg_rdma_mem)(struct isert_cmd *isert_cmd, + struct iscsi_conn *conn); void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); }; diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 0c37fee363b1d6..b0707a7aac029b 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1261,40 +1261,26 @@ static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch, */ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) { + struct se_session *se_sess; struct srpt_send_ioctx *ioctx; - unsigned long flags; + int tag; BUG_ON(!ch); + se_sess = ch->sess; - ioctx = NULL; - spin_lock_irqsave(&ch->spinlock, flags); - if (!list_empty(&ch->free_list)) { - ioctx = list_first_entry(&ch->free_list, - struct srpt_send_ioctx, free_list); - list_del(&ioctx->free_list); + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, TASK_RUNNING); + if (tag < 0) { + pr_err("Unable to obtain tag for srpt_send_ioctx\n"); + return NULL; } - spin_unlock_irqrestore(&ch->spinlock, flags); - - if (!ioctx) - return ioctx; - - BUG_ON(ioctx->ch != ch); + ioctx = &((struct srpt_send_ioctx *)se_sess->sess_cmd_map)[tag]; + memset(ioctx, 0, sizeof(struct srpt_send_ioctx)); + ioctx->ch = ch; spin_lock_init(&ioctx->spinlock); ioctx->state = SRPT_STATE_NEW; - ioctx->n_rbuf = 0; - ioctx->rbufs = NULL; - ioctx->n_rdma = 0; - ioctx->n_rdma_wrs = 0; - ioctx->rdma_wrs = NULL; - ioctx->mapped_sg_count = 0; init_completion(&ioctx->tx_done); - ioctx->queue_status_only = false; - /* - * transport_init_se_cmd() does not initialize all fields, so do it - * here. - */ - memset(&ioctx->cmd, 0, sizeof(ioctx->cmd)); - memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data)); + + ioctx->cmd.map_tag = tag; return ioctx; } @@ -2240,9 +2226,8 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, struct srp_login_rej *rej; struct ib_cm_rep_param *rep_param; struct srpt_rdma_ch *ch, *tmp_ch; - struct se_node_acl *se_acl; u32 it_iu_len; - int i, ret = 0; + int ret = 0; unsigned char *p; WARN_ON_ONCE(irqs_disabled()); @@ -2371,12 +2356,6 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, if (!ch->ioctx_ring) goto free_ch; - INIT_LIST_HEAD(&ch->free_list); - for (i = 0; i < ch->rq_size; i++) { - ch->ioctx_ring[i]->ch = ch; - list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list); - } - ret = srpt_create_ch_ib(ch); if (ret) { rej->reason = cpu_to_be32( @@ -2406,19 +2385,13 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, pr_debug("registering session %s\n", ch->sess_name); p = &ch->sess_name[0]; - ch->sess = transport_init_session(TARGET_PROT_NORMAL); - if (IS_ERR(ch->sess)) { - rej->reason = cpu_to_be32( - SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); - pr_debug("Failed to create session\n"); - goto destroy_ib; - } - try_again: - se_acl = core_tpg_get_initiator_node_acl(&sport->port_tpg_1, p); - if (!se_acl) { + ch->sess = target_alloc_session(&sport->port_tpg_1, ch->rq_size, + sizeof(struct srpt_send_ioctx), + TARGET_PROT_NORMAL, p, ch, NULL); + if (IS_ERR(ch->sess)) { pr_info("Rejected login because no ACL has been" - " configured yet for initiator %s.\n", ch->sess_name); + " configured yet for initiator %s.\n", p); /* * XXX: Hack to retry of ch->i_port_id without leading '0x' */ @@ -2426,14 +2399,11 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id, p += 2; goto try_again; } - rej->reason = cpu_to_be32( + rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ? + SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES : SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED); - transport_free_session(ch->sess); goto destroy_ib; } - ch->sess->se_node_acl = se_acl; - - transport_register_session(&sport->port_tpg_1, se_acl, ch->sess, ch); pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess, ch->sess_name, ch->cm_id); @@ -3205,7 +3175,7 @@ static void srpt_release_cmd(struct se_cmd *se_cmd) struct srpt_send_ioctx *ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); struct srpt_rdma_ch *ch = ioctx->ch; - unsigned long flags; + struct se_session *se_sess = ch->sess; WARN_ON(ioctx->state != SRPT_STATE_DONE); WARN_ON(ioctx->mapped_sg_count != 0); @@ -3216,9 +3186,7 @@ static void srpt_release_cmd(struct se_cmd *se_cmd) ioctx->n_rbuf = 0; } - spin_lock_irqsave(&ch->spinlock, flags); - list_add(&ioctx->free_list, &ch->free_list); - spin_unlock_irqrestore(&ch->spinlock, flags); + percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); } /** diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h index 09037f2b0b5143..6fbb6e79407c6b 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.h +++ b/drivers/infiniband/ulp/srpt/ib_srpt.h @@ -179,7 +179,6 @@ struct srpt_recv_ioctx { * struct srpt_send_ioctx - SRPT send I/O context. * @ioctx: See above. * @ch: Channel pointer. - * @free_list: Node in srpt_rdma_ch.free_list. * @n_rbuf: Number of data buffers in the received SRP command. * @rbufs: Pointer to SRP data buffer array. * @single_rbuf: SRP data buffer if the command has only a single buffer. @@ -202,7 +201,6 @@ struct srpt_send_ioctx { struct srp_direct_buf *rbufs; struct srp_direct_buf single_rbuf; struct scatterlist *sg; - struct list_head free_list; spinlock_t spinlock; enum srpt_command_state state; struct se_cmd cmd; diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index e2f31c93717db6..0a1366cba81b41 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -184,7 +184,7 @@ config CHR_DEV_SCH changers are listed as "Type: Medium Changer" in /proc/scsi/scsi. If you have such hardware and want to use it with linux, say Y here. Check for details. - + If you want to compile this as a module ( = code which can be inserted in and removed from the running kernel whenever you want), say M here and read and @@ -491,7 +491,7 @@ config SCSI_DPT_I2O tristate "Adaptec I2O RAID support " depends on SCSI && PCI && VIRT_TO_BUS help - This driver supports all of Adaptec's I2O based RAID controllers as + This driver supports all of Adaptec's I2O based RAID controllers as well as the DPT SmartRaid V cards. This is an Adaptec maintained driver by Deanna Bonds. See . @@ -760,7 +760,7 @@ config SCSI_GDTH ---help--- Formerly called GDT SCSI Disk Array Controller Support. - This is a driver for RAID/SCSI Disk Array Controllers (EISA/ISA/PCI) + This is a driver for RAID/SCSI Disk Array Controllers (EISA/ISA/PCI) manufactured by Intel Corporation/ICP vortex GmbH. It is documented in the kernel source in and . @@ -803,7 +803,7 @@ config SCSI_GENERIC_NCR5380_MMIO select SCSI_SPI_ATTRS ---help--- This is a driver for the old NCR 53c80 series of SCSI controllers - on boards using memory mapped I/O. + on boards using memory mapped I/O. It is explained in section 3.8 of the SCSI-HOWTO, available from . If it doesn't work out of the box, you may have to change some settings in @@ -847,6 +847,20 @@ config SCSI_IBMVSCSI To compile this driver as a module, choose M here: the module will be called ibmvscsi. +config SCSI_IBMVSCSIS + tristate "IBM Virtual SCSI Server support" + depends on PPC_PSERIES && SCSI_SRP && TARGET_CORE + help + This is the SRP target driver for IBM pSeries virtual environments. + + The userspace component needed to initialize the driver and + documentation can be found: + + http://stgt.berlios.de/ + + To compile this driver as a module, choose M here: the + module will be called ibmvstgt. + config SCSI_IBMVFC tristate "IBM Virtual FC support" depends on PPC_PSERIES && SCSI @@ -1728,6 +1742,15 @@ config SCSI_PM8001 This driver supports PMC-Sierra PCIE SAS/SATA 8x6G SPC 8001 chip based host adapters. +config SCSI_SRP + tristate "SCSI RDMA Protocol helper library" + depends on SCSI && PCI + help + If you wish to use SRP target drivers, say Y. + + To compile this driver as a module, choose M here: the + module will be called libsrp. + config SCSI_BFA_FC tristate "Brocade BFA Fibre Channel Support" depends on PCI && SCSI diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 862ab4efad61e9..4b5ec85ed9f740 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -84,7 +84,7 @@ obj-$(CONFIG_SCSI_NCR_Q720) += NCR_Q720_mod.o obj-$(CONFIG_SCSI_SYM53C416) += sym53c416.o obj-$(CONFIG_SCSI_QLOGIC_FAS) += qlogicfas408.o qlogicfas.o obj-$(CONFIG_PCMCIA_QLOGIC) += qlogicfas408.o -obj-$(CONFIG_SCSI_QLOGIC_1280) += qla1280.o +obj-$(CONFIG_SCSI_QLOGIC_1280) += qla1280.o obj-$(CONFIG_SCSI_QLA_FC) += qla2xxx/ obj-$(CONFIG_SCSI_QLA_ISCSI) += libiscsi.o qla4xxx/ obj-$(CONFIG_SCSI_LPFC) += lpfc/ @@ -127,7 +127,9 @@ obj-$(CONFIG_SCSI_LASI700) += 53c700.o lasi700.o obj-$(CONFIG_SCSI_SNI_53C710) += 53c700.o sni_53c710.o obj-$(CONFIG_SCSI_NSP32) += nsp32.o obj-$(CONFIG_SCSI_IPR) += ipr.o +obj-$(CONFIG_SCSI_SRP) += libsrp.o obj-$(CONFIG_SCSI_IBMVSCSI) += ibmvscsi/ +obj-$(CONFIG_SCSI_IBMVSCSIS) += ibmvscsi/ obj-$(CONFIG_SCSI_IBMVFC) += ibmvscsi/ obj-$(CONFIG_SCSI_HPTIOP) += hptiop.o obj-$(CONFIG_SCSI_STEX) += stex.o @@ -146,6 +148,8 @@ obj-$(CONFIG_XEN_SCSI_FRONTEND) += xen-scsifront.o obj-$(CONFIG_HYPERV_STORAGE) += hv_storvsc.o obj-$(CONFIG_SCSI_WD719X) += wd719x.o +CFLAGS_libsrp.o := -DDEBUG + obj-$(CONFIG_ARM) += arm/ obj-$(CONFIG_CHR_DEV_ST) += st.o diff --git a/drivers/scsi/ibmvscsi/Makefile b/drivers/scsi/ibmvscsi/Makefile index 3840c64f2966bd..015fba7e207c3f 100644 --- a/drivers/scsi/ibmvscsi/Makefile +++ b/drivers/scsi/ibmvscsi/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_SCSI_IBMVSCSI) += ibmvscsi.o +obj-$(CONFIG_SCSI_IBMVSCSIS) += ibmvscsis.o obj-$(CONFIG_SCSI_IBMVFC) += ibmvfc.o diff --git a/drivers/scsi/ibmvscsi/ibmvscsis.c b/drivers/scsi/ibmvscsi/ibmvscsis.c new file mode 100644 index 00000000000000..9246f05bbc95d6 --- /dev/null +++ b/drivers/scsi/ibmvscsi/ibmvscsis.c @@ -0,0 +1,1938 @@ +/* + * IBM eServer i/pSeries Virtual SCSI Target Driver + * Copyright (C) 2003-2005 Dave Boutcher (boutcher@us.ibm.com) IBM Corp. + * Santiago Leon (santil@us.ibm.com) IBM Corp. + * Linda Xie (lxie@us.ibm.com) IBM Corp. + * + * Copyright (C) 2005-2011 FUJITA Tomonori + * Copyright (C) 2010 Nicholas A. Bellinger + * Copyright (C) 2016 Bryant G. Ly IBM Corp. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "ibmvscsi.h" +#include "viosrp.h" + +#define GETTARGET(x) ((int)((((u64)(x)) >> 56) & 0x003f)) +#define GETBUS(x) ((int)((((u64)(x)) >> 53) & 0x0007)) +#define GETLUN(x) ((int)((((u64)(x)) >> 48) & 0x001f)) + +#define IBMVSCSIS_VERSION "v0.1" +#define IBMVSCSIS_NAMELEN 32 + +#define INITIAL_SRP_LIMIT 800 +#define DEFAULT_MAX_SECTORS 256 + +#define MAX_H_COPY_RDMA (128*1024) + +#define SRP_RSP_SENSE_DATA_LEN 18 +#define NO_SUCH_LUN ((uint64_t)-1LL) + +static struct workqueue_struct *vtgtd; +static unsigned max_vdma_size = MAX_H_COPY_RDMA; + +/* Adapter list and lock to control it */ +static DEFINE_SPINLOCK(ibmvscsis_dev_lock); +static LIST_HEAD(ibmvscsis_dev_list); + +struct ibmvscsis_cmnd { + /* Used for libsrp processing callbacks */ + struct scsi_cmnd sc; + /* Used for TCM Core operations */ + struct se_cmd se_cmd; + /* Sense buffer that will be mapped into outgoing status */ + unsigned char sense_buf[TRANSPORT_SENSE_BUFFER]; + u32 lun; +}; + +struct ibmvscsis_crq_msg { + u8 valid; + u8 format; + u8 rsvd; + u8 status; + u16 rsvd1; + __be16 IU_length; + __be64 IU_data_ptr; +}; + +struct ibmvscsis_tport { + /* SCSI protocol the tport is providing */ + u8 tport_proto_id; + /* ASCII formatted WWPN for SRP Target port */ + char tport_name[IBMVSCSIS_NAMELEN]; + /* Returned by ibmvscsis_make_tport() */ + struct se_wwn tport_wwn; + int lun_count; + /* Returned by ibmvscsis_make_tpg() */ + struct se_portal_group se_tpg; + /* ibmvscsis port target portal group tag for TCM */ + u16 tport_tpgt; + /* Pointer to TCM session for I_T Nexus */ + struct se_session *se_sess; + struct ibmvscsis_cmnd *cmd; + bool enabled; + bool releasing; +}; + +struct ibmvscsis_adapter { + struct device dev; + struct vio_dev *dma_dev; + struct list_head siblings; + + struct crq_queue crq_queue; + struct work_struct crq_work; + + atomic_t req_lim_delta; + u32 liobn; + u32 riobn; + + struct srp_target *target; + + struct list_head list; + struct ibmvscsis_tport tport; +}; + +struct ibmvscsis_nacl { + /* Returned by ibmvscsis_make_nexus */ + struct se_node_acl se_node_acl; +}; + +struct inquiry_data { + u8 qual_type; + u8 rmb_reserve; + u8 version; + u8 aerc_naca_hisup_format; + u8 addl_len; + u8 sccs_reserved; + u8 bque_encserv_vs_multip_mchngr_reserved; + u8 reladr_reserved_linked_cmdqueue_vs; + char vendor[8]; + char product[16]; + char revision[4]; + char vendor_specific[20]; + char reserved1[2]; + char version_descriptor[16]; + char reserved2[22]; + char unique[158]; +}; + +enum scsi_lun_addr_method { + SCSI_LUN_ADDR_METHOD_PERIPHERAL = 0, + SCSI_LUN_ADDR_METHOD_FLAT = 1, + SCSI_LUN_ADDR_METHOD_LUN = 2, + SCSI_LUN_ADDR_METHOD_EXTENDED_LUN = 3, +}; + +static int ibmvscsis_probe(struct vio_dev *vdev, + const struct vio_device_id *id); +static void ibmvscsis_dev_release(struct device *dev); +static int read_dma_window(struct vio_dev *vdev, + struct ibmvscsis_adapter *adapter); +static char *ibmvscsis_get_fabric_name(void); +static char *ibmvscsis_get_fabric_wwn(struct se_portal_group *se_tpg); +static u16 ibmvscsis_get_tag(struct se_portal_group *se_tpg); +static u32 ibmvscsis_get_default_depth(struct se_portal_group *se_tpg); +static int ibmvscsis_check_true(struct se_portal_group *se_tpg); +static int ibmvscsis_check_false(struct se_portal_group *se_tpg); +static u32 ibmvscsis_tpg_get_inst_index(struct se_portal_group *se_tpg); +static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd); +static void ibmvscsis_release_cmd(struct se_cmd *se_cmd); +static int ibmvscsis_shutdown_session(struct se_session *se_sess); +static void ibmvscsis_close_session(struct se_session *se_sess); +static u32 ibmvscsis_sess_get_index(struct se_session *se_sess); +static int ibmvscsis_write_pending(struct se_cmd *se_cmd); +static int ibmvscsis_write_pending_status(struct se_cmd *se_cmd); +static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl); +static int ibmvscsis_get_cmd_state(struct se_cmd *se_cmd); +static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd); +static int ibmvscsis_queue_status(struct se_cmd *se_cmd); +static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd); +static void ibmvscsis_aborted_task(struct se_cmd *se_cmd); +static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf, + struct config_group *group, + const char *name); +static void ibmvscsis_drop_tport(struct se_wwn *wwn); +static struct se_portal_group *ibmvscsis_make_tpg(struct se_wwn *wwn, + struct config_group *group, + const char *name); +static void ibmvscsis_drop_tpg(struct se_portal_group *se_tpg); +static int ibmvscsis_remove(struct vio_dev *vdev); +static ssize_t system_id_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t partition_number_show(struct device *dev, + struct device_attribute *attr, + char *buf); +static ssize_t unit_address_show(struct device *dev, + struct device_attribute *attr, char *buf); +static int get_system_info(void); +static irqreturn_t ibmvscsis_interrupt(int dummy, void *data); +static int process_srp_iu(struct iu_entry *iue); +static void process_iu(struct viosrp_crq *crq, + struct ibmvscsis_adapter *adapter); +static void process_crq(struct viosrp_crq *crq, + struct ibmvscsis_adapter *adapter); +static void handle_crq(struct work_struct *work); +static int ibmvscsis_reset_crq_queue(struct ibmvscsis_adapter *adapter); +static void crq_queue_destroy(struct ibmvscsis_adapter *adapter); +static inline struct viosrp_crq *next_crq(struct crq_queue *queue); +static int send_iu(struct iu_entry *iue, u64 length, u8 format); +static int send_rsp(struct iu_entry *iue, struct scsi_cmnd *sc, + unsigned char status, unsigned char asc); +static int send_adapter_info(struct iu_entry *iue, + dma_addr_t remote_buffer, u16 length); +static int process_mad_iu(struct iu_entry *iue); +static void process_login(struct iu_entry *iue); +static void process_tsk_mgmt(struct iu_entry *iue); +static int ibmvscsis_rdma(struct scsi_cmnd *sc, struct scatterlist *sg, + int nsg, struct srp_direct_buf *md, int nmd, + enum dma_data_direction dir, unsigned int rest); +static int ibmvscsis_cmnd_done(struct se_cmd *se_cmd); +static int ibmvscsis_queuecommand(struct ibmvscsis_adapter *adapter, + struct iu_entry *iue); +static uint64_t ibmvscsis_unpack_lun(const uint8_t *lun, int len); +static int tcm_queuecommand(struct ibmvscsis_adapter *adapter, + struct ibmvscsis_cmnd *vsc, + struct srp_cmd *scmd); + +/* + * Hypervisor calls. + */ +#define h_reg_crq(ua, tok, sz)\ + plpar_hcall_norets(H_REG_CRQ, ua, tok, sz); + +static inline long h_copy_rdma(s64 length, u64 sliobn, u64 slioba, + u64 dliobn, u64 dlioba) +{ + long rc = 0; + + mb(); + + rc = plpar_hcall_norets(H_COPY_RDMA, length, sliobn, slioba, + dliobn, dlioba); + return rc; +} + +static inline void h_free_crq(uint32_t unit_address) +{ + long rc = 0; + + do { + if (H_IS_LONG_BUSY(rc)) + msleep(get_longbusy_msecs(rc)); + + rc = plpar_hcall_norets(H_FREE_CRQ, unit_address); + } while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc))); +} + +static inline long h_send_crq(struct ibmvscsis_adapter *adapter, + u64 word1, u64 word2) +{ + long rc; + struct vio_dev *vdev = adapter->dma_dev; + + pr_debug("ibmvscsis: ibmvscsis_send_crq(0x%x, 0x%016llx, 0x%016llx)\n", + vdev->unit_address, word1, word2); + + /* + * Ensure the command buffer is flushed to memory before handing it + * over to the other side to prevent it from fetching any stale data. + */ + mb(); + rc = plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2); + pr_debug("ibmvscsis: ibmvcsis_send_crq rc = 0x%lx\n", rc); + + return rc; +} + +/*****************************************************************************/ +/* Global device driver data areas */ +/*****************************************************************************/ + +static const char ibmvscsis_driver_name[] = "ibmvscsis"; +static char system_id[64] = ""; +static char partition_name[97] = "UNKNOWN"; +static unsigned int partition_number = -1; + +static struct class_attribute ibmvscsis_class_attrs[] = { + __ATTR_NULL, +}; + +static struct device_attribute dev_attr_system_id = + __ATTR(system_id, S_IRUGO, system_id_show, NULL); + +static struct device_attribute dev_attr_partition_number = + __ATTR(partition_number, S_IRUGO, partition_number_show, NULL); + +static struct device_attribute dev_attr_unit_address = + __ATTR(unit_address, S_IRUGO, unit_address_show, NULL); + +static struct attribute *ibmvscsis_dev_attrs[] = { + &dev_attr_system_id.attr, + &dev_attr_partition_number.attr, + &dev_attr_unit_address.attr, +}; +ATTRIBUTE_GROUPS(ibmvscsis_dev); + +static struct class ibmvscsis_class = { + .name = "ibmvscsis", + .dev_release = ibmvscsis_dev_release, + .class_attrs = ibmvscsis_class_attrs, + .dev_groups = ibmvscsis_dev_groups, +}; + +static ssize_t ibmvscsis_wwn_version_show(struct config_item *item, + char *page) +{ + return sprintf(page, "IBMVSCSIS fabric module %s on %s/%s" + "on "UTS_RELEASE"\n", IBMVSCSIS_VERSION, utsname()->sysname, + utsname()->machine); +} + +CONFIGFS_ATTR_RO(ibmvscsis_wwn_, version); + +static struct configfs_attribute *ibmvscsis_wwn_attrs[] = { + &ibmvscsis_wwn_attr_version, + NULL, +}; + +static ssize_t ibmvscsis_tpg_enable_show(struct config_item *item, + char *page) +{ + struct se_portal_group *se_tpg = to_tpg(item); + struct ibmvscsis_tport *tport = container_of(se_tpg, + struct ibmvscsis_tport, se_tpg); + + return snprintf(page, PAGE_SIZE, "%d\n", (tport->enabled) ? 1: 0); +} + +static ssize_t ibmvscsis_tpg_enable_store(struct config_item *item, + const char *page, size_t count) +{ + + struct se_portal_group *se_tpg = to_tpg(item); + struct ibmvscsis_tport *tport = container_of(se_tpg, + struct ibmvscsis_tport, se_tpg); + unsigned long tmp; + int ret; + + ret = kstrtoul(page, 0, &tmp); + if (ret < 0) { + pr_err("Unable to extract srpt_tpg_store_enable\n"); + return -EINVAL; + } + + if ((tmp != 0) && (tmp != 1)) { + pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", + tmp); + return -EINVAL; + } + + if (tmp == 1) + tport->enabled = true; + else + tport->enabled = false; + + return count; +} + +CONFIGFS_ATTR(ibmvscsis_tpg_, enable); + +static struct configfs_attribute *ibmvscsis_tpg_attrs[] = { + &ibmvscsis_tpg_attr_enable, + NULL, +}; + +static const struct target_core_fabric_ops ibmvscsis_ops = { + .module = THIS_MODULE, + .name = "ibmvscsis", + .max_data_sg_nents = SCSI_MAX_SG_SEGMENTS, + .get_fabric_name = ibmvscsis_get_fabric_name, + .tpg_get_wwn = ibmvscsis_get_fabric_wwn, + .tpg_get_tag = ibmvscsis_get_tag, + .tpg_get_default_depth = ibmvscsis_get_default_depth, + .tpg_check_demo_mode = ibmvscsis_check_true, + .tpg_check_demo_mode_cache = ibmvscsis_check_true, + .tpg_check_demo_mode_write_protect = ibmvscsis_check_false, + .tpg_check_prod_mode_write_protect = ibmvscsis_check_false, + .tpg_get_inst_index = ibmvscsis_tpg_get_inst_index, + .check_stop_free = ibmvscsis_check_stop_free, + .release_cmd = ibmvscsis_release_cmd, + .shutdown_session = ibmvscsis_shutdown_session, + .close_session = ibmvscsis_close_session, + .sess_get_index = ibmvscsis_sess_get_index, + .write_pending = ibmvscsis_write_pending, + .write_pending_status = ibmvscsis_write_pending_status, + .set_default_node_attributes = ibmvscsis_set_default_node_attrs, + .get_cmd_state = ibmvscsis_get_cmd_state, + .queue_data_in = ibmvscsis_queue_data_in, + .queue_status = ibmvscsis_queue_status, + .queue_tm_rsp = ibmvscsis_queue_tm_rsp, + .aborted_task = ibmvscsis_aborted_task, + /* + * Setup function pointers for logic in target_cor_fabric_configfs.c + */ + .fabric_make_wwn = ibmvscsis_make_tport, + .fabric_drop_wwn = ibmvscsis_drop_tport, + .fabric_make_tpg = ibmvscsis_make_tpg, + .fabric_drop_tpg = ibmvscsis_drop_tpg, + + .tfc_wwn_attrs = ibmvscsis_wwn_attrs, + .tfc_tpg_base_attrs = ibmvscsis_tpg_attrs, +}; + +static struct vio_device_id ibmvscsis_device_table[] = { + {"v-scsi-host", "IBM,v-scsi-host"}, + {"", ""} +}; + +MODULE_DEVICE_TABLE(vio, ibmvscsis_device_table); + +static struct vio_driver ibmvscsis_driver = { + .name = ibmvscsis_driver_name, + .id_table = ibmvscsis_device_table, + .probe = ibmvscsis_probe, + .remove = ibmvscsis_remove, +}; + +/*****************************************************************************/ +/* End of global device driver data areas */ +/*****************************************************************************/ +static int crq_queue_create(struct crq_queue *queue, + struct ibmvscsis_adapter *adapter) +{ + int retrc; + int err; + struct vio_dev *vdev = adapter->dma_dev; + + queue->msgs = (struct viosrp_crq *)get_zeroed_page(GFP_KERNEL); + + if (!queue->msgs) + goto malloc_failed; + + queue->size = PAGE_SIZE / sizeof(*queue->msgs); + + queue->msg_token = dma_map_single(&vdev->dev, queue->msgs, + queue->size * sizeof(*queue->msgs), + DMA_BIDIRECTIONAL); + + if (dma_mapping_error(&vdev->dev, queue->msg_token)) { + goto map_failed; + } + + retrc = err = h_reg_crq(vdev->unit_address, queue->msg_token, + PAGE_SIZE); + + /* If the adapter was left active for some reason (like kexec) + * try freeing and re-registering + */ + if (err == H_RESOURCE) { + err = ibmvscsis_reset_crq_queue(adapter); + } + if( err == 2 ) { + pr_warn("ibmvscsis: Partner adapter not ready\n"); + retrc = 0; + } else if ( err != 0 ) { + pr_err("ibmvscsis: Error 0x%x opening virtual adapter\n", err); + goto reg_crq_failed; + } + + queue->cur = 0; + spin_lock_init(&queue->lock); + + INIT_WORK(&adapter->crq_work, handle_crq); + + err = request_irq(vdev->irq, &ibmvscsis_interrupt, + 0, "ibmvscsis", adapter); + if (err) { + pr_err("ibmvscsis: Error 0x%x h_send_crq\n", err); + goto req_irq_failed; + } + + err = vio_enable_interrupts(vdev); + if (err != 0 ) { + pr_err("ibmvscsis: Error %d enabling interrupts!!!\n", err); + goto req_irq_failed; + } + + return retrc; + +req_irq_failed: + h_free_crq(vdev->unit_address); +reg_crq_failed: + dma_unmap_single(&vdev->dev, queue->msg_token, + queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL); +map_failed: + free_page((unsigned long) queue->msgs); +malloc_failed: + return -1; +} + +/* + * ibmvscsis_probe - ibm vscsis target initialize entry point + * @param dev vio device struct + * @param id vio device id struct + * @return 0 - Success + * Non-zero - Failure + */ +static int ibmvscsis_probe(struct vio_dev *vdev, const struct vio_device_id *id) +{ + int ret = -ENOMEM; + struct ibmvscsis_adapter *adapter; + struct srp_target *target; + unsigned long flags; + + pr_debug("ibmvscsis: Probe for UA 0x%x\n", vdev->unit_address); + + adapter = kzalloc(sizeof(struct ibmvscsis_adapter), GFP_KERNEL); + if (!adapter) + return ret; + target = kzalloc(sizeof(struct srp_target), GFP_KERNEL); + if (!target) + goto free_adapter; + + adapter->dma_dev = vdev; + adapter->target = target; + snprintf(&adapter->tport.tport_name[0], 256, "%s", dev_name(&vdev->dev)); + + ret = read_dma_window(adapter->dma_dev, adapter); + if(ret != 0) { + pr_debug("ibmvscsis: probe failed read dma window\n"); + goto free_target; + } + pr_debug("ibmvscsis: Probe: liobn 0x%x, riobn 0x%x\n", adapter->liobn, + adapter->riobn); + + spin_lock_irqsave(&ibmvscsis_dev_lock, flags); + list_add_tail(&adapter->list, &ibmvscsis_dev_list); + spin_unlock_irqrestore(&ibmvscsis_dev_lock, flags); + + ret = srp_target_alloc(target, &vdev->dev, + INITIAL_SRP_LIMIT, + SRP_MAX_IU_LEN); + + adapter->target->ldata = adapter; + + if(ret) { + pr_debug("ibmvscsis: failed target alloc ret: %d\n", ret); + goto free_srp_target; + } + + ret = crq_queue_create(&adapter->crq_queue, adapter); + pr_debug("ibmvscsis: failed crq_queue_create ret: %d\n", ret); + if(ret != 0 && ret != H_RESOURCE) { + pr_debug("ibmvscsis: failed crq_queue_create ret: %d\n", ret); + ret = -1; + } + + if(h_send_crq(adapter, 0xC001000000000000LL, 0) != 0 + && ret != H_RESOURCE) { + pr_warn("ibmvscsis: Failed to send CRQ message\n"); + ret = 0; + } + + dev_set_drvdata(&vdev->dev, adapter); + + return 0; + +free_srp_target: + srp_target_free(target); +free_target: + kfree(target); +free_adapter: + kfree(adapter); + return ret; +} + +static int ibmvscsis_remove(struct vio_dev *dev) +{ + unsigned long flags; + struct ibmvscsis_adapter *adapter = dev_get_drvdata(&dev->dev); + + spin_lock_irqsave(&ibmvscsis_dev_lock, flags); + list_del(&adapter->list); + spin_unlock_irqrestore(&ibmvscsis_dev_lock, flags); + + crq_queue_destroy(adapter); + srp_target_free(adapter->target); + + kfree(adapter); + + return 0; +} + +static int read_dma_window(struct vio_dev *vdev, + struct ibmvscsis_adapter *adapter) +{ + const __be32 *dma_window; + const __be32 *prop; + + /* TODO Using of_parse_dma_window would be better, but it doesn't give + * a way to read multiple windows without already knowing the size of + * a window or the number of windows + */ + dma_window = + (const __be32 *)vio_get_attribute(vdev, "ibm,my-dma-window", + NULL); + if (!dma_window) { + pr_err("ibmvscsis: Couldn't find ibm,my-dma-window property\n"); + return -1; + } + + adapter->liobn = be32_to_cpu(*dma_window); + dma_window++; + + prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells", + NULL); + if (!prop) { + pr_warn("ibmvscsis: Couldn't find ibm, \ + #dma-address-cells property\n"); + dma_window++; + } else + dma_window += be32_to_cpu(*prop); + + prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells", + NULL); + if (!prop) { + pr_warn("ibmvscsis: Couldn't find ibm,#dma-size-cells property\n"); + dma_window++; + } else + dma_window += be32_to_cpu(*prop); + + /* dma_window should point to the second window now */ + adapter->riobn = be32_to_cpu(*dma_window); + + return 0; +} + +static inline union viosrp_iu *vio_iu(struct iu_entry *iue) +{ + return (union viosrp_iu *)(iue->sbuf->buf); +} + +static void ibmvscsis_dev_release(struct device *dev) {}; + +static char *ibmvscsis_get_fabric_name(void) +{ + return "ibmvscsis"; +} + +static char *ibmvscsis_get_fabric_wwn(struct se_portal_group *se_tpg) +{ + struct ibmvscsis_tport *tport = + container_of(se_tpg, struct ibmvscsis_tport, se_tpg); + + return &tport->tport_name[0]; +} + +static u16 ibmvscsis_get_tag(struct se_portal_group *se_tpg) +{ + struct ibmvscsis_tport *tport = + container_of(se_tpg, struct ibmvscsis_tport, se_tpg); + + return tport->tport_tpgt; +} + +static u32 ibmvscsis_get_default_depth(struct se_portal_group *se_tpg) +{ + return 1; +} + +static int ibmvscsis_check_true(struct se_portal_group *se_tpg) +{ + return 1; +} + +static int ibmvscsis_check_false(struct se_portal_group *se_tpg) +{ + return 0; +} + +static u32 ibmvscsis_tpg_get_inst_index(struct se_portal_group *se_tpg) +{ + return 1; +} + +static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd) +{ + return target_put_sess_cmd(se_cmd); +} + +static void ibmvscsis_release_cmd(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = + container_of(se_cmd, struct ibmvscsis_cmnd, se_cmd); + + kfree(cmd); +} + +static int ibmvscsis_shutdown_session(struct se_session *se_sess) +{ + return 0; +} + +static void ibmvscsis_close_session(struct se_session *se_sess) +{ + return; +} + +static u32 ibmvscsis_sess_get_index(struct se_session *se_sess) +{ + return 0; +} + +static int ibmvscsis_write_pending(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = container_of(se_cmd, + struct ibmvscsis_cmnd, se_cmd); + struct scsi_cmnd *sc = &cmd->sc; + struct iu_entry *iue = (struct iu_entry *) sc->SCp.ptr; + int ret; + + pr_debug("ibmvscsis: ibmvscsis_write_pending\n"); + sc->sdb.table.nents = se_cmd->t_data_nents; + sc->sdb.table.sgl = se_cmd->t_data_sg; + + ret = srp_transfer_data(sc, &vio_iu(iue)->srp.cmd, + ibmvscsis_rdma, 1, 1); + if (ret) { + pr_err("ibmvscsis: srp_transfer_data() failed: %d\n", ret); + return -EAGAIN; /* Signal QUEUE_FULL */ + } + /* + * We now tell TCM to add this WRITE CDB directly into the TCM storage + * object execution queue. + */ + target_execute_cmd(se_cmd); + return 0; +} + +static int ibmvscsis_write_pending_status(struct se_cmd *se_cmd) +{ + return 0; +} + +static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl) +{ + return; +} + +static int ibmvscsis_get_cmd_state(struct se_cmd *se_cmd) +{ + return 0; +} + +static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = container_of(se_cmd, + struct ibmvscsis_cmnd, se_cmd); + struct scsi_cmnd *sc = &cmd->sc; + /* + * Check for overflow residual count + */ + pr_debug("ibmvscsis: ibmvscsis_queue_data_in\n"); + + if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) + scsi_set_resid(sc, se_cmd->residual_count); + + sc->sdb.length = se_cmd->data_length; + + sc->sdb.table.nents = se_cmd->t_data_nents; + sc->sdb.table.sgl = se_cmd->t_data_sg; + + /* + * This will call srp_transfer_data() and post the response + * to VIO via libsrp. + */ + ibmvscsis_cmnd_done(se_cmd); + pr_debug("ibmvscsis: queue_data_in"); + return 0; +} + +static int ibmvscsis_queue_status(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = container_of(se_cmd, + struct ibmvscsis_cmnd, se_cmd); + struct scsi_cmnd *sc = &cmd->sc; + /* + * Copy any generated SENSE data into sc->sense_buffer and + * set the appropriate sc->result to be translated by + * ibmvscsis_cmnd_done() + */ + pr_debug("ibmvscsis: ibmvscsis_queue_status\n"); + if (se_cmd->sense_buffer && + ((se_cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || + (se_cmd->se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { + memcpy(sc->sense_buffer, se_cmd->sense_buffer, + SCSI_SENSE_BUFFERSIZE); + sc->result = SAM_STAT_CHECK_CONDITION; + set_driver_byte(sc, DRIVER_SENSE); + } else + sc->result = se_cmd->scsi_status; + + set_host_byte(sc, DID_OK); + if ((se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) || + (se_cmd->se_cmd_flags & SCF_UNDERFLOW_BIT)) + scsi_set_resid(sc, se_cmd->residual_count); +// sc->scsi_done(sc); + /* + * Finally post the response to VIO via libsrp. + */ + ibmvscsis_cmnd_done(se_cmd); + return 0; +} + +static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = container_of(se_cmd, + struct ibmvscsis_cmnd, se_cmd); + struct scsi_cmnd *sc = &cmd->sc; + struct iu_entry *iue = (struct iu_entry *) sc->SCp.ptr; + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + struct srp_cmd *srp_cmd; + int ret; + enum dma_data_direction dir; + + pr_debug("ibmvscsis: ibmvscsis_queue_tm_rsp\n"); + + if (unlikely(transport_check_aborted_status(se_cmd, false))) { + atomic_inc(&adapter->req_lim_delta); + srp_iu_put(iue); + goto out; + } + + srp_cmd = &vio_iu(iue)->srp.cmd; + dir = srp_cmd_direction(srp_cmd); + + if (dir == DMA_FROM_DEVICE) { + ret = srp_transfer_data(sc, srp_cmd, ibmvscsis_rdma, 1, 1); + if( ret == -ENOMEM) + pr_debug("ibmvscsis: res queue full\n"); + else if (ret) + pr_err("ibmvscsis: tm_rsp failed\n"); + } + + send_rsp(iue, sc, NO_SENSE, 0x00); + return; +out: + return; +} + +static void ibmvscsis_aborted_task(struct se_cmd *se_cmd) +{ + pr_debug("ibmvscsis: ibmvscsis_aborted_task\n"); + return; +} + +static struct se_portal_group *ibmvscsis_make_nexus(struct ibmvscsis_tport *tport, + const char *name) +{ + struct se_node_acl *acl; + + pr_debug("ibmvscsis: make nexus"); + if (tport->se_sess) { + pr_debug("tport->se_sess already exists\n"); + ERR_PTR(-EEXIST); + } + + /* + * Initialize the struct se_session pointer and setup tagpool + * for struct ibmvscsis_cmd descriptors + */ + tport->se_sess = transport_init_session(TARGET_PROT_NORMAL); + if (IS_ERR(tport->se_sess)) { + goto transport_init_fail; + } + pr_debug("ibmvscsis: make_nexus: se_sess:%p, tport(%p)\n", + tport->se_sess, tport); + pr_debug("ibmvsciss: initiator name:%s, se_tpg:%p\n", + tport->se_sess->se_node_acl->initiatorname, + tport->se_sess->se_tpg); + /* + * Since we are running in 'demo mode' this call will generate a + * struct se_node_acl for the ibmvscsis struct se_portal_group with + * the SCSI Initiator port name of the passed configfs group 'name'. + */ + + acl = core_tpg_check_initiator_node_acl(&tport->se_tpg, + (unsigned char *)name); + if (!acl) { + pr_debug("core_tpg_check_initiator_node_acl() failed" + " for %s\n", name); + goto acl_failed; + } + tport->se_sess->se_node_acl = acl; + + /* + * Now register the TCM ibmvscsis virtual I_T Nexus as active. + */ + transport_register_session(&tport->se_tpg, + tport->se_sess->se_node_acl, + tport->se_sess, tport); + + return &tport->se_tpg; + +acl_failed: + transport_free_session(tport->se_sess); +transport_init_fail: + kfree(tport); + return ERR_PTR(-ENOMEM); +} + +static int ibmvscsis_drop_nexus(struct ibmvscsis_tport *tport) +{ + struct se_session *se_sess; + + pr_debug("ibmvscsis: drop nexus"); + + se_sess = tport->se_sess; + if (!se_sess) { + return -ENODEV; + } + + /* + * Release the SCSI I_T Nexus to the emulated ibmvscsis Target Port + */ + transport_deregister_session(tport->se_sess); + + transport_free_session(tport->se_sess); + + return 0; +} + +static struct ibmvscsis_tport *ibmvscsis_lookup_port(const char *name) +{ + struct ibmvscsis_tport *tport; + struct vio_dev *vdev; + struct ibmvscsis_adapter *adapter; + int ret; + unsigned long flags; + + spin_lock_irqsave(&ibmvscsis_dev_lock, flags); + list_for_each_entry(adapter, &ibmvscsis_dev_list, list) { + vdev = adapter->dma_dev; + ret = strcmp(dev_name(&vdev->dev), name); + if(ret == 0) { + tport = &adapter->tport; + } + if(tport) + goto found; + } + spin_unlock_irqrestore(&ibmvscsis_dev_lock, flags); + return NULL; +found: + spin_unlock_irqrestore(&ibmvscsis_dev_lock, flags); + return tport; +} + +static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf, + struct config_group *group, + const char *name) +{ + struct ibmvscsis_tport *tport; + int ret; + + tport = ibmvscsis_lookup_port(name); + ret = -EINVAL; + if(!tport) + goto err; + + tport->tport_proto_id = SCSI_PROTOCOL_SRP; + pr_debug("ibmvscsis: make_tport(%s), pointer:%p tport_id:%x\n", name, + tport, tport->tport_proto_id); + return &tport->tport_wwn; +err: + return ERR_PTR(ret); +} + +static void ibmvscsis_drop_tport(struct se_wwn *wwn) +{ + struct ibmvscsis_tport *tport = container_of(wwn, + struct ibmvscsis_tport, tport_wwn); + + pr_debug("drop_tport(%s\n", + config_item_name(&tport->tport_wwn.wwn_group.cg_item)); +} + +static struct se_portal_group *ibmvscsis_make_tpg(struct se_wwn *wwn, + struct config_group *group, + const char *name) +{ + struct ibmvscsis_tport *tport = + container_of(wwn, struct ibmvscsis_tport, tport_wwn); + int ret; + + tport->releasing = false; + + ret = core_tpg_register(&tport->tport_wwn, + &tport->se_tpg, + tport->tport_proto_id); + if(ret) + return ERR_PTR(ret); + + return &tport->se_tpg; +} + +static void ibmvscsis_drop_tpg(struct se_portal_group *se_tpg) +{ + struct ibmvscsis_tport *tport = container_of(se_tpg, + struct ibmvscsis_tport, se_tpg); + + tport->releasing = true; + tport->enabled = false; + + /* + * Release the virtual I_T Nexus for this ibmvscsis TPG + */ + ibmvscsis_drop_nexus(tport); + /* + * Deregister the se_tpg from TCM.. + */ + core_tpg_deregister(se_tpg); +} + +static ssize_t system_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%s\n", system_id); +} + +static ssize_t partition_number_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%x\n", partition_number); +} + +static ssize_t unit_address_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ibmvscsis_adapter *adapter = + container_of(dev, struct ibmvscsis_adapter, dev); + return snprintf(buf, PAGE_SIZE, "%x\n", adapter->dma_dev->unit_address); +} + +static int get_system_info(void) +{ + struct device_node *rootdn, *vdevdn; + const char *id, *model, *name; + const unsigned int *num; + + pr_debug("ibmvscsis: getsysteminfo"); + rootdn = of_find_node_by_path("/"); + if (!rootdn) + return -ENOENT; + + model = of_get_property(rootdn, "model", NULL); + id = of_get_property(rootdn, "system-id", NULL); + if (model && id) + snprintf(system_id, sizeof(system_id), "%s-%s", model, id); + + name = of_get_property(rootdn, "ibm,partition-name", NULL); + if (name) + strncpy(partition_name, name, sizeof(partition_name)); + + num = of_get_property(rootdn, "ibm,partition-no", NULL); + if (num) + partition_number = of_read_number(num, 1); + + of_node_put(rootdn); + + vdevdn = of_find_node_by_path("/vdevice"); + vdevdn = of_find_node_by_path("/vdevice"); + if (vdevdn) { + const unsigned *mvds; + + mvds = of_get_property(vdevdn, "ibm,max-virtual-dma-size", + NULL); + if (mvds) + max_vdma_size = *mvds; + of_node_put(vdevdn); + } + + return 0; +}; + +static irqreturn_t ibmvscsis_interrupt(int dummy, void *data) +{ + struct ibmvscsis_adapter *adapter = data; + + pr_debug("ibmvscsis: there is an interrupt\n"); + vio_disable_interrupts(adapter->dma_dev); + queue_work(vtgtd, &adapter->crq_work); + + return IRQ_HANDLED; +} + +static int process_srp_iu(struct iu_entry *iue) +{ + union viosrp_iu *iu = vio_iu(iue); + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + u8 opcode = iu->srp.rsp.opcode; + unsigned long flags; + int err = 1; + + spin_lock_irqsave(&target->lock, flags); + if (adapter->tport.releasing) { + spin_unlock_irqrestore(&target->lock, flags); + pr_err("ibmvscsis: process_srp_iu, no tpg, releasing:%x\n", + adapter->tport.releasing); + srp_iu_put(iue); + goto done; + } + spin_unlock_irqrestore(&target->lock, flags); + + switch (opcode) { + case SRP_LOGIN_REQ: + pr_debug("ibmvscsis: srploginreq"); + process_login(iue); + break; + case SRP_TSK_MGMT: + pr_debug("ibmvscsis: srp task mgmt"); + process_tsk_mgmt(iue); + break; + case SRP_CMD: + pr_debug("ibmvscsis: srpcmd"); + pr_debug("ibmvscsis: process_srp_iu, iu_entry: %llx\n", + (u64)iue->sbuf->buf); + err = ibmvscsis_queuecommand(adapter, iue); + if(err) { + srp_iu_put(iue); + pr_debug("ibmvscsis: can't queue cmd\n"); + } + break; + case SRP_LOGIN_RSP: + case SRP_I_LOGOUT: + case SRP_T_LOGOUT: + case SRP_RSP: + case SRP_CRED_REQ: + case SRP_CRED_RSP: + case SRP_AER_REQ: + case SRP_AER_RSP: + pr_err("ibmvscsis: Unsupported type %u\n", opcode); + break; + default: + pr_err("ibmvscsis: Unknown type %u\n", opcode); + } +done: + return err; +} + +static void process_iu(struct viosrp_crq *crq, + struct ibmvscsis_adapter *adapter) +{ + struct iu_entry *iue; + long err; + + iue = srp_iu_get(adapter->target); + if (!iue) { + pr_err("Error getting IU from pool %p\n", iue); + return; + } + + iue->remote_token = crq->IU_data_ptr; + + err = h_copy_rdma(be16_to_cpu(crq->IU_length), adapter->riobn, + be64_to_cpu(crq->IU_data_ptr), + adapter->liobn, iue->sbuf->dma); + + if (err != H_SUCCESS) { + pr_err("ibmvscsis: %ld transferring data error %p\n", err, iue); + srp_iu_put(iue); + } + + if (crq->format == VIOSRP_MAD_FORMAT) { + process_mad_iu(iue); + } + else { + pr_debug("ibmvscsis: process srpiu"); + process_srp_iu(iue); + } +} + +static void process_crq(struct viosrp_crq *crq, + struct ibmvscsis_adapter *adapter) +{ + switch (crq->valid) { + case 0xC0: + /* initialization */ + switch (crq->format) { + case 0x01: + h_send_crq(adapter, 0xC002000000000000, 0); + break; + case 0x02: + break; + default: + pr_err("ibmvscsis: Unknown format %u\n", crq->format); + } + break; + case 0xFF: + /* transport event */ + break; + case 0x80: + /* real payload */ + switch (crq->format) { + case VIOSRP_SRP_FORMAT: + case VIOSRP_MAD_FORMAT: + pr_debug("ibmvscsis: case viosrp mad crq: 0x%x, 0x%x," + " 0x%x, 0x%x, 0x%x, 0x%x, 0x%llx\n", + crq->valid, crq->format, crq->reserved, + crq->status, be16_to_cpu(crq->timeout), + be16_to_cpu(crq->IU_length), + be64_to_cpu(crq->IU_data_ptr)); + process_iu(crq, adapter); + break; + case VIOSRP_OS400_FORMAT: + case VIOSRP_AIX_FORMAT: + case VIOSRP_LINUX_FORMAT: + case VIOSRP_INLINE_FORMAT: + pr_err("ibmvscsis: Unsupported format %u\n", + crq->format); + break; + default: + pr_err("ibmvscsis: Unknown format %u\n", + crq->format); + } + break; + default: + pr_err("ibmvscsis: unknown message type 0x%02x!?\n", + crq->valid); + } +} + +static void handle_crq(struct work_struct *work) +{ + struct ibmvscsis_adapter *adapter = + container_of(work, struct ibmvscsis_adapter, crq_work); + struct viosrp_crq *crq; + int done = 0; + + while (!done) { + while ((crq = next_crq(&adapter->crq_queue)) != NULL) { + process_crq(crq, adapter); + crq->valid = 0x00; + } + + vio_enable_interrupts(adapter->dma_dev); + + crq = next_crq(&adapter->crq_queue); + if (crq) { + vio_disable_interrupts(adapter->dma_dev); + process_crq(crq, adapter); + crq->valid = 0x00; + } else + done = 1; + } +} + +static int ibmvscsis_reset_crq_queue(struct ibmvscsis_adapter *adapter) +{ + int rc = 0; + struct vio_dev *vdev = adapter->dma_dev; + struct crq_queue *queue = &adapter->crq_queue; + + /* Close the CRQ */ + h_free_crq(vdev->unit_address); + + /* Clean out the queue */ + memset(queue->msgs, 0x00, PAGE_SIZE); + queue->cur = 0; + + /* And re-open it again */ + rc = h_reg_crq(vdev->unit_address, queue->msg_token, + PAGE_SIZE); + if (rc == 2) + /* Adapter is good, but other end is not ready */ + pr_warn("ibmvscsis: Partner adapter not ready\n"); + else if (rc != 0) + pr_err("ibmvscsis: couldn't register crq--rc 0x%x\n", rc); + + return rc; +} + +static void crq_queue_destroy(struct ibmvscsis_adapter *adapter) +{ + struct vio_dev *vdev = adapter->dma_dev; + struct crq_queue *queue = &adapter->crq_queue; + + free_irq(vdev->irq, (void *)adapter); + flush_work(&adapter->crq_work); + h_free_crq(vdev->unit_address); + dma_unmap_single(&adapter->dma_dev->dev, queue->msg_token, + queue->size * sizeof(*queue->msgs), + DMA_BIDIRECTIONAL); + + free_page((unsigned long)queue->msgs); +} + +static inline struct viosrp_crq *next_crq(struct crq_queue *queue) +{ + struct viosrp_crq *crq; + unsigned long flags; + + spin_lock_irqsave(&queue->lock, flags); + crq = &queue->msgs[queue->cur]; + if (crq->valid & 0x80) { + if (++queue->cur == queue->size) + queue->cur = 0; + rmb(); + } else + crq = NULL; + spin_unlock_irqrestore(&queue->lock, flags); + + return crq; +} + +static int send_iu(struct iu_entry *iue, u64 length, u8 format) +{ + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + struct ibmvscsis_crq_msg crq_msg; + __be64 *crq_as_u64 = (__be64*)&crq_msg; + long rc, rc1; + + pr_debug("ibmvscsis: send_iu: 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n", + (unsigned long)length, + (unsigned long)adapter->liobn, + (unsigned long)iue->sbuf->dma, + (unsigned long)adapter->riobn, + (unsigned long)be64_to_cpu(iue->remote_token)); + + /* First copy the SRP */ + rc = h_copy_rdma(length, adapter->liobn, iue->sbuf->dma, + adapter->riobn, be64_to_cpu(iue->remote_token)); + + if (rc) + pr_err("ibmvscsis: Error %ld transferring data\n", rc); + + pr_debug("ibmvscsis: crq pre cooked: 0x%x, 0x%llx, 0x%llx\n", + format, length, vio_iu(iue)->srp.rsp.tag); + + crq_msg.valid = 0x80; + crq_msg.format = format; + crq_msg.rsvd = 0; + if (rc == 0) { + crq_msg.status = 0x99; + } else { + crq_msg.status = 0; + } + crq_msg.rsvd1 = 0; + crq_msg.IU_length = cpu_to_be16(length); + crq_msg.IU_data_ptr = vio_iu(iue)->srp.rsp.tag; + + pr_debug("ibmvscsis: send crq: 0x%x, 0x%llx, 0x%llx\n", + adapter->dma_dev->unit_address, + be64_to_cpu(crq_as_u64[0]), + be64_to_cpu(crq_as_u64[1])); + + srp_iu_put(iue); + + rc1 = h_send_crq(adapter, be64_to_cpu(crq_as_u64[0]), + be64_to_cpu(crq_as_u64[1])); + + if (rc1) { + pr_err("ibmvscsis: %ld sending response\n", rc1); + return rc1; + } + + return rc; +} + +static int send_rsp(struct iu_entry *iue, struct scsi_cmnd *sc, + unsigned char status, unsigned char asc) +{ + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + union viosrp_iu *iu = vio_iu(iue); + uint64_t tag = iu->srp.rsp.tag; + + /* If the linked bit is on and status is good */ + if (test_bit(V_LINKED, &iue->flags) && (status == NO_SENSE)) + status = 0x10; + + memset(iu, 0, sizeof(struct srp_rsp)); + iu->srp.rsp.opcode = SRP_RSP; +// iu->srp.rsp.req_lim_delta = 1; + iu->srp.rsp.req_lim_delta = cpu_to_be32(1 + + atomic_xchg(&adapter->req_lim_delta, 0)); + iu->srp.rsp.tag = tag; + + if (test_bit(V_DIOVER, &iue->flags)) + iu->srp.rsp.flags |= SRP_RSP_FLAG_DIOVER; + + iu->srp.rsp.data_in_res_cnt = 0; + iu->srp.rsp.data_out_res_cnt = 0; + + iu->srp.rsp.flags &= ~SRP_RSP_FLAG_RSPVALID; + + iu->srp.rsp.resp_data_len = 0; + iu->srp.rsp.status = status; + if (status) { + uint8_t *sense = iu->srp.rsp.data; + + if (sc) { + iu->srp.rsp.flags |= SRP_RSP_FLAG_SNSVALID; + iu->srp.rsp.sense_data_len = SCSI_SENSE_BUFFERSIZE; + memcpy(sense, sc->sense_buffer, SCSI_SENSE_BUFFERSIZE); + } else { + iu->srp.rsp.status = SAM_STAT_CHECK_CONDITION; + iu->srp.rsp.flags |= SRP_RSP_FLAG_SNSVALID; + iu->srp.rsp.sense_data_len = SRP_RSP_SENSE_DATA_LEN; + + /* Valid bit and 'current errors' */ + sense[0] = (0x1 << 7 | 0x70); + /* Sense key */ + sense[2] = status; + /* Additional sense length */ + sense[7] = 0xa; /* 10 bytes */ + /* Additional sense code */ + sense[12] = asc; + } + } + + send_iu(iue, sizeof(iu->srp.rsp) + SRP_RSP_SENSE_DATA_LEN, + VIOSRP_SRP_FORMAT); + + return 0; +} + +static int send_adapter_info(struct iu_entry *iue, + dma_addr_t remote_buffer, u16 length) +{ + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + dma_addr_t data_token; + struct mad_adapter_info_data *info; + int err; + + info = dma_alloc_coherent(&adapter->dma_dev->dev, sizeof(*info), + &data_token, GFP_KERNEL); + if (!info) { + pr_err("ibmvscsis: bad dma_alloc_coherent %p\n", target); + return 1; + } + + pr_debug("ibmvscsis: get_remote_info: 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n", + (unsigned long)sizeof(*info), + (unsigned long)adapter->liobn, + (unsigned long)data_token, + (unsigned long)adapter->riobn, + (unsigned long)remote_buffer); + + /* Get remote info */ + err = h_copy_rdma(sizeof(*info), adapter->riobn, + be64_to_cpu(remote_buffer), + adapter->liobn, data_token); + + if (err == H_SUCCESS) { + pr_err("ibmvscsis: Client connect: %s (%d)\n", + info->partition_name, info->partition_number); + } + + memset(info, 0, sizeof(*info)); + + strcpy(info->srp_version, "16.a"); + strncpy(info->partition_name, partition_name, + sizeof(info->partition_name)); + pr_debug("ibmvscsis: partition_number: %x\n", partition_number); + + info->partition_number = cpu_to_be32(partition_number); + info->mad_version = cpu_to_be32(1); + info->os_type = cpu_to_be32(2); + info->port_max_txu[0] = cpu_to_be32(SCSI_MAX_SG_SEGMENTS * PAGE_SIZE); + + pr_debug("ibmvscsis: send info to remote: 0x%lx 0x%lx 0x%lx \ + 0x%lx 0x%lx\n",(unsigned long)sizeof(*info), + (unsigned long)adapter->liobn, + (unsigned long)data_token, + (unsigned long)adapter->riobn, + (unsigned long)remote_buffer); + + /* Send our info to remote */ + err = h_copy_rdma(sizeof(*info), adapter->liobn, data_token, + adapter->riobn, be64_to_cpu(remote_buffer)); + + dma_free_coherent(&adapter->dma_dev->dev, sizeof(*info), info, + data_token); + if (err != H_SUCCESS) { + pr_err("ibmvscsis: Error sending adapter info %d\n", err); + return 1; + } + + return 0; +} + +static int process_mad_iu(struct iu_entry *iue) +{ + union viosrp_iu *iu = vio_iu(iue); + struct viosrp_adapter_info *info; + struct viosrp_host_config *conf; + + switch (be32_to_cpu(iu->mad.empty_iu.common.type)) { + case VIOSRP_EMPTY_IU_TYPE: + pr_err("ibmvscsis: %s\n", "Unsupported EMPTY MAD IU"); + break; + case VIOSRP_ERROR_LOG_TYPE: + pr_err("ibmvscsis: %s\n", "Unsupported ERROR LOG MAD IU"); + iu->mad.error_log.common.status = 1; + send_iu(iue, sizeof(iu->mad.error_log), VIOSRP_MAD_FORMAT); + break; + case VIOSRP_ADAPTER_INFO_TYPE: + info = &iu->mad.adapter_info; + info->common.status = send_adapter_info(iue, info->buffer, + info->common.length); + send_iu(iue, sizeof(*info), VIOSRP_MAD_FORMAT); + break; + case VIOSRP_HOST_CONFIG_TYPE: + conf = &iu->mad.host_config; + conf->common.status = 1; + send_iu(iue, sizeof(*conf), VIOSRP_MAD_FORMAT); + break; + default: + pr_err("ibmvscsis: Unknown type %u\n", iu->srp.rsp.opcode); + iu->mad.empty_iu.common.status = + cpu_to_be16(VIOSRP_MAD_NOT_SUPPORTED); + send_iu(iue, sizeof(iu->mad), VIOSRP_MAD_FORMAT); + break; + } + + return 1; +} + +static void process_login(struct iu_entry *iue) +{ + union viosrp_iu *iu = vio_iu(iue); + struct srp_login_rsp *rsp = &iu->srp.login_rsp; + struct srp_login_rej *rej = &iu->srp.login_rej; + u64 tag = iu->srp.rsp.tag; + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + struct vio_dev *vdev = adapter->dma_dev; + char name[16]; + struct se_portal_group *se_tpg; + + /* + * TODO handle case that requested size is wrong and buffer + * format is wrong + */ + memset(iu, 0, max(sizeof(*rsp), sizeof(*rej))); + + snprintf(name, sizeof(name), "%x", vdev->unit_address); + + if(!&adapter->tport.enabled) { + rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); + pr_err("ibmvscsis: Rejected SRP_LOGIN_REQ because target %s" + " has not yet been enabled", name); + goto reject; + } + + se_tpg = ibmvscsis_make_nexus(&adapter->tport, + &adapter->tport.tport_name[0]); + if(!se_tpg) { + pr_debug("ibmvscsis: login make nexus fail se_tpg(%p)\n", + se_tpg); + goto reject; + } + + rsp->opcode = SRP_LOGIN_RSP; + + rsp->req_lim_delta = cpu_to_be32(INITIAL_SRP_LIMIT); + + pr_debug("ibmvscsis: process_login, tag:%llu\n", tag); + + rsp->tag = tag; + rsp->max_it_iu_len = cpu_to_be32(sizeof(union srp_iu)); + rsp->max_ti_iu_len = cpu_to_be32(sizeof(union srp_iu)); + /* direct and indirect */ + rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT + | SRP_BUF_FORMAT_INDIRECT); + + send_iu(iue, sizeof(*rsp), VIOSRP_SRP_FORMAT); + return; + +reject: + rej->opcode = SRP_LOGIN_REJ; + rej->tag = tag; + rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT + | SRP_BUF_FORMAT_INDIRECT); + + send_iu(iue, sizeof(*rej), VIOSRP_SRP_FORMAT); +} + +static void process_tsk_mgmt(struct iu_entry *iue) +{ + union viosrp_iu *iu = vio_iu(iue); + uint64_t tag = iu->srp.rsp.tag; + uint8_t *resp_data = iu->srp.rsp.data; + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + + memset(iu, 0, sizeof(struct srp_rsp)); + iu->srp.rsp.opcode = SRP_RSP; + iu->srp.rsp.req_lim_delta = cpu_to_be32(1 + + atomic_xchg(&adapter->req_lim_delta, 0)); +// iu->srp.rsp.req_lim_delta = 1; + iu->srp.rsp.tag = tag; + + iu->srp.rsp.data_in_res_cnt = 0; + iu->srp.rsp.data_out_res_cnt = 0; + + iu->srp.rsp.flags &= ~SRP_RSP_FLAG_RSPVALID; + + iu->srp.rsp.resp_data_len = 4; + /* TASK MANAGEMENT FUNCTION NOT SUPPORTED for now */ + resp_data[3] = 4; + + send_iu(iue, sizeof(iu->srp.rsp) + iu->srp.rsp.resp_data_len, + VIOSRP_SRP_FORMAT); +} + +static int ibmvscsis_rdma(struct scsi_cmnd *sc, struct scatterlist *sg, int nsg, + struct srp_direct_buf *md, int nmd, + enum dma_data_direction dir, unsigned int rest) +{ + struct iu_entry *iue = (struct iu_entry *) sc->SCp.ptr; + struct srp_target *target = iue->target; + struct ibmvscsis_adapter *adapter = target->ldata; + dma_addr_t token; + long err; + unsigned int done = 0; + int i, sidx, soff; + + sidx = soff = 0; + token = sg_dma_address(sg + sidx); + + for (i = 0; i < nmd && rest; i++) { + unsigned int mdone, mlen; + + mlen = min(rest, be32_to_cpu(md[i].len)); + for (mdone = 0; mlen;) { + int slen = min(sg_dma_len(sg + sidx) - soff, mlen); + + if (dir == DMA_TO_DEVICE) + err = h_copy_rdma(slen, + adapter->riobn, + be64_to_cpu(md[i].va) + mdone, + adapter->liobn, + token + soff); + else + err = h_copy_rdma(slen, + adapter->liobn, + token + soff, + adapter->riobn, + be64_to_cpu(md[i].va)+mdone); + + if (err != H_SUCCESS) { + pr_err("ibmvscsis: rdma error %d %d %ld\n", + dir, slen, err); + return -EIO; + } + + mlen -= slen; + mdone += slen; + soff += slen; + done += slen; + + if (soff == sg_dma_len(sg + sidx)) { + sidx++; + soff = 0; + token = sg_dma_address(sg + sidx); + + if (sidx > nsg) { + pr_err("ibmvscsis: out of sg %p %d %d\n", + iue, sidx, nsg); + return -EIO; + } + } + } + rest -= mlen; + } + return 0; +} + +static int ibmvscsis_cmnd_done(struct se_cmd *se_cmd) +{ + struct ibmvscsis_cmnd *cmd = container_of(se_cmd, + struct ibmvscsis_cmnd, se_cmd); + struct scsi_cmnd *sc = &cmd->sc; + struct iu_entry *iue = (struct iu_entry *) sc->SCp.ptr; + enum dma_data_direction dir; + int err = 0; + + dir = sc->sc_data_direction; + + if (scsi_sg_count(sc)) + err = srp_transfer_data(sc, &vio_iu(iue)->srp.cmd, + ibmvscsis_rdma, 1, 1); + + if (err || sc->result != SAM_STAT_GOOD) { + pr_err("ibmvscsis: operation failed %p %d %x\n", + iue, sc->result, vio_iu(iue)->srp.cmd.cdb[0]); + send_rsp(iue, sc, HARDWARE_ERROR, 0x00); + } else + send_rsp(iue, sc, NO_SENSE, 0x00); + + srp_iu_put(iue); + return 0; +} + +static int ibmvscsis_queuecommand(struct ibmvscsis_adapter *adapter, + struct iu_entry *iue) +{ + struct srp_cmd *cmd = iue->sbuf->buf; + struct scsi_cmnd *sc; + struct ibmvscsis_cmnd *vsc; + int ret; + + pr_debug("ibmvscsis: ibmvscsis_queuecommand\n"); + + vsc = kzalloc(sizeof(*vsc), GFP_KERNEL); + sc = &vsc->sc; + sc->sense_buffer = vsc->se_cmd.sense_buffer; + sc->cmnd = cmd->cdb; + sc->SCp.ptr = (char *)iue; + + pr_debug("ibmvscsis: tcm_queuecommand\n"); + ret = tcm_queuecommand(adapter, vsc, cmd); + + return ret; +} + +static uint64_t ibmvscsis_unpack_lun(const uint8_t *lun, int len) +{ + uint64_t res = NO_SUCH_LUN; + int addressing_method; + + if (unlikely(len < 2)) { + pr_err("Illegal LUN length %d, expected 2 bytes or more\n", + len); + goto out; + } + + switch (len) { + case 8: + if ((*((__be64 *)lun) & + cpu_to_be64(0x0000FFFFFFFFFFFFLL)) != 0) + goto out_err; + break; + case 4: + if (*((__be16 *)&lun[2]) != 0) + goto out_err; + break; + case 6: + if (*((__be32 *)&lun[2]) != 0) + goto out_err; + break; + case 2: + break; + default: + goto out_err; + } + + addressing_method = (*lun) >> 6; /* highest two bits of byte 0 */ + switch (addressing_method) { + case SCSI_LUN_ADDR_METHOD_PERIPHERAL: + case SCSI_LUN_ADDR_METHOD_FLAT: + case SCSI_LUN_ADDR_METHOD_LUN: + res = *(lun + 1) | (((*lun) & 0x3f) << 8); + break; + + case SCSI_LUN_ADDR_METHOD_EXTENDED_LUN: + default: + pr_err("Unimplemented LUN addressing method %u\n", + addressing_method); + break; + } + +out: + return res; + +out_err: + pr_err("Support for multi-level LUNs has not yet been implemented\n"); + goto out; +} + +static int tcm_queuecommand(struct ibmvscsis_adapter *adapter, + struct ibmvscsis_cmnd *vsc, + struct srp_cmd *scmd) +{ + struct se_cmd *se_cmd; + int attr; + u64 data_len; + int ret; + uint64_t unpacked_lun; + + switch (scmd->task_attr) { + case SRP_SIMPLE_TASK: + attr = TCM_SIMPLE_TAG; + break; + case SRP_ORDERED_TASK: + attr = TCM_ORDERED_TAG; + break; + case SRP_HEAD_TASK: + attr = TCM_HEAD_TAG; + break; + case SRP_ACA_TASK: + attr = TCM_ACA_TAG; + break; + default: + pr_err("ibmvscsis: Task attribute %d not supported\n", + scmd->task_attr); + attr = TCM_SIMPLE_TAG; + } + + pr_debug("ibmvscsis: srp_data_length: %llx, srp_direction:%x\n", + srp_data_length(scmd, srp_cmd_direction(scmd)), + srp_cmd_direction(scmd)); + data_len = srp_data_length(scmd, srp_cmd_direction(scmd)); + + vsc->se_cmd.tag = scmd->tag; + se_cmd = &vsc->se_cmd; + + pr_debug("ibmvscsis: size of lun:%lx, lun:%s\n", sizeof(scmd->lun), + &scmd->lun.scsi_lun[0]); + + unpacked_lun = ibmvscsis_unpack_lun((uint8_t *)&scmd->lun, + sizeof(scmd->lun)); + + pr_debug("ibmvscsis: tcm_queuecommand- se_cmd(%p), se_sess(%p)," + " cdb: %x, sense: %x, unpacked_lun: %llx," + " data_length: %llx, task_attr: %x, data_dir: %x" + " flags: %x, tag:%llx, packed_lun:%llx\n", + se_cmd, adapter->tport.se_sess, + scmd->cdb[0], vsc->sense_buf[0], unpacked_lun, + data_len, attr, srp_cmd_direction(scmd), + TARGET_SCF_ACK_KREF, scmd->tag, + be64_to_cpu(&scmd->lun)); + + ret = target_submit_cmd(se_cmd, adapter->tport.se_sess, + &scmd->cdb[0], &vsc->sense_buf[0], unpacked_lun, + data_len, attr, srp_cmd_direction(scmd), + 0); + if(ret != 0) { + ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + pr_debug("ibmvscsis: tcm_queuecommand fail submit_cmd\n"); + goto send_sense; + } + return 0; + +send_sense: + transport_send_check_condition_and_sense(&vsc->se_cmd, ret, 0); + return -1; +} + +/* + * ibmvscsis_init() - Kernel Module initialization + * + * Note: vio_register_driver() registers callback functions, and atleast one + * of those call back functions calls TCM - Linux IO Target Subsystem, thus + * the SCSI Target template must be registered before vio_register_driver() + * is called. + */ +static int __init ibmvscsis_init(void) +{ + int ret = -ENOMEM; + + pr_info("IBMVSCSIS fabric module %s on %s/%s" + "on "UTS_RELEASE"\n", IBMVSCSIS_VERSION, utsname()->sysname, + utsname()->machine); + + ret = get_system_info(); + if (ret) { + pr_err("ibmvscsis: ret %d from get_system_info\n", ret); + goto out; + } + + ret = class_register(&ibmvscsis_class); + if(ret) { + pr_err("ibmvscsis failed class register\n"); + goto out; + } + + pr_debug("ibmvscsis: start register template"); + ret = target_register_template(&ibmvscsis_ops); + if (ret) { + pr_debug("ibmvscsis: ret %d from target_register_template\n", + ret); + goto unregister_class; + } + + vtgtd = create_workqueue("ibmvscsis"); + if(!vtgtd) + goto unregister_target; + + ret = vio_register_driver(&ibmvscsis_driver); + if (ret) { + pr_err("ibmvscsis: ret %d from vio_register_driver\n", ret); + goto destroy_wq; + } + + return 0; + +destroy_wq: + destroy_workqueue(vtgtd); +unregister_target: + target_unregister_template(&ibmvscsis_ops); +unregister_class: + class_unregister(&ibmvscsis_class); +out: + return ret; +}; + +static void __exit ibmvscsis_exit(void) +{ + pr_info("ibmvscsis: Unregister IBM virtual SCSI driver\n"); + vio_unregister_driver(&ibmvscsis_driver); + destroy_workqueue(vtgtd); + target_unregister_template(&ibmvscsis_ops); + class_unregister(&ibmvscsis_class); +}; + +MODULE_DESCRIPTION("IBMVSCSIS fabric driver"); +MODULE_AUTHOR("Bryant G. Ly"); +MODULE_LICENSE("GPL"); +module_init(ibmvscsis_init); +module_exit(ibmvscsis_exit); + diff --git a/drivers/scsi/libsrp.c b/drivers/scsi/libsrp.c new file mode 100644 index 00000000000000..3ec0157a9dceb6 --- /dev/null +++ b/drivers/scsi/libsrp.c @@ -0,0 +1,410 @@ +/* + * SCSI RDMA Protocol lib functions + * + * Copyright (C) 2006 FUJITA Tomonori + * Copyright (C) 2016 Bryant G. Ly IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int srp_iu_pool_alloc(struct srp_queue *q, size_t max, + struct srp_buf **ring) +{ + int i; + struct iu_entry *iue; + + q->pool = kcalloc(max, sizeof(struct iu_entry *), GFP_KERNEL); + if (!q->pool) + return -ENOMEM; + q->items = kcalloc(max, sizeof(struct iu_entry), GFP_KERNEL); + if (!q->items) + goto free_pool; + + spin_lock_init(&q->lock); + kfifo_init(&q->queue, (void *) q->pool, max * sizeof(void *)); + + for (i = 0, iue = q->items; i < max; i++) { + kfifo_in(&q->queue, (void *) &iue, sizeof(void *)); + iue->sbuf = ring[i]; + iue++; + } + return 0; + + kfree(q->items); +free_pool: + kfree(q->pool); + return -ENOMEM; +} + +static void srp_iu_pool_free(struct srp_queue *q) +{ + kfree(q->items); + kfree(q->pool); +} + +static struct srp_buf **srp_ring_alloc(struct device *dev, + size_t max, size_t size) +{ + int i; + struct srp_buf **ring; + + ring = kcalloc(max, sizeof(struct srp_buf *), GFP_KERNEL); + if (!ring) + return NULL; + + for (i = 0; i < max; i++) { + ring[i] = kzalloc(sizeof(struct srp_buf), GFP_KERNEL); + if (!ring[i]) + goto out; + ring[i]->buf = dma_alloc_coherent(dev, size, &ring[i]->dma, + GFP_KERNEL); + if (!ring[i]->buf) + goto out; + } + return ring; + +out: + for (i = 0; i < max && ring[i]; i++) { + if (ring[i]->buf) { + dma_free_coherent(dev, size, ring[i]->buf, + ring[i]->dma); + } + kfree(ring[i]); + } + kfree(ring); + + return NULL; +} + +static void srp_ring_free(struct device *dev, struct srp_buf **ring, size_t max, + size_t size) +{ + int i; + + for (i = 0; i < max; i++) { + dma_free_coherent(dev, size, ring[i]->buf, ring[i]->dma); + kfree(ring[i]); + } + kfree(ring); +} + +int srp_target_alloc(struct srp_target *target, struct device *dev, + size_t nr, size_t iu_size) +{ + int err; + + spin_lock_init(&target->lock); + + target->dev = dev; + + target->srp_iu_size = iu_size; + target->rx_ring_size = nr; + target->rx_ring = srp_ring_alloc(target->dev, nr, iu_size); + if (!target->rx_ring) + return -ENOMEM; + err = srp_iu_pool_alloc(&target->iu_queue, nr, target->rx_ring); + if (err) + goto free_ring; + + dev_set_drvdata(target->dev, target); + return 0; + +free_ring: + srp_ring_free(target->dev, target->rx_ring, nr, iu_size); + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(srp_target_alloc); + +void srp_target_free(struct srp_target *target) +{ + dev_set_drvdata(target->dev, NULL); + srp_ring_free(target->dev, target->rx_ring, target->rx_ring_size, + target->srp_iu_size); + srp_iu_pool_free(&target->iu_queue); +} +EXPORT_SYMBOL_GPL(srp_target_free); + +struct iu_entry *srp_iu_get(struct srp_target *target) +{ + struct iu_entry *iue = NULL; + + pr_info("libsrp: srp_iu_get\n"); + if (kfifo_out_locked(&target->iu_queue.queue, (void *) &iue, + sizeof(void *), + &target->iu_queue.lock) != sizeof(void *)) { + WARN_ONCE(1, "unexpected fifo state"); + return NULL; + } + if (!iue) + return iue; + iue->target = target; + iue->flags = 0; + return iue; +} +EXPORT_SYMBOL_GPL(srp_iu_get); + +void srp_iu_put(struct iu_entry *iue) +{ + pr_info("libsrp: srp_iu_put\n"); + kfifo_in_locked(&iue->target->iu_queue.queue, (void *) &iue, + sizeof(void *), &iue->target->iu_queue.lock); +} +EXPORT_SYMBOL_GPL(srp_iu_put); + +static int srp_direct_data(struct scsi_cmnd *sc, struct srp_direct_buf *md, + enum dma_data_direction dir, srp_rdma_t rdma_io, + int dma_map, int ext_desc) +{ + struct iu_entry *iue = NULL; + struct scatterlist *sg = NULL; + int err, nsg = 0, len; + + if (dma_map) { + iue = (struct iu_entry *) sc->SCp.ptr; + sg = scsi_sglist(sc); + + pr_debug("libsrp: iue: %p scsi_buff_len: %u srp_buff_len: %u" + " scsi_sg_count: %d va:%llx, key:%x, len:%x\n", + iue, scsi_bufflen(sc), be32_to_cpu(md->len), + scsi_sg_count(sc), be64_to_cpu(md->va), + be32_to_cpu(md->key), + be32_to_cpu(md->len)); + + nsg = dma_map_sg(iue->target->dev, sg, scsi_sg_count(sc), + DMA_BIDIRECTIONAL); + if (!nsg) { + pr_err("libsrp: fail to map %p %d\n", + iue, scsi_sg_count(sc)); + return 0; + } + len = min(scsi_bufflen(sc), be32_to_cpu(md->len)); + } else + len = be32_to_cpu(md->len); + + err = rdma_io(sc, sg, nsg, md, 1, dir, len); + + if (dma_map) + dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); + + return err; +} + +static int srp_indirect_data(struct scsi_cmnd *sc, struct srp_cmd *cmd, + struct srp_indirect_buf *id, + enum dma_data_direction dir, srp_rdma_t rdma_io, + int dma_map, int ext_desc) +{ + struct iu_entry *iue = NULL; + struct srp_direct_buf *md = NULL; + struct scatterlist dummy, *sg = NULL; + dma_addr_t token = 0; + int err = 0; + int nmd, nsg = 0, len; + + if (dma_map || ext_desc) { + iue = (struct iu_entry *) sc->SCp.ptr; + sg = scsi_sglist(sc); + + pr_debug("libsrp: iue: %p scsi_buff_len: %u srp_ind_len: %u" + " in_desc_count: %d out_desc_count: %d, va:%llx," + " key:%x, len:%x\n", + iue, scsi_bufflen(sc), id->len, + cmd->data_in_desc_cnt, cmd->data_out_desc_cnt, + be64_to_cpu(id->table_desc.va), + be32_to_cpu(id->table_desc.key), + be32_to_cpu(id->table_desc.len)); + } + + nmd = be32_to_cpu(id->table_desc.len) / sizeof(struct srp_direct_buf); + + if ((dir == DMA_FROM_DEVICE && nmd == cmd->data_in_desc_cnt) || + (dir == DMA_TO_DEVICE && nmd == cmd->data_out_desc_cnt)) { + md = &id->desc_list[0]; + goto rdma; + } + + if (ext_desc && dma_map) { + md = dma_alloc_coherent(iue->target->dev, + be32_to_cpu(id->table_desc.len), + &token, GFP_KERNEL); + if (!md) { + pr_err("libsrp: Can't get dma memory %u\n", + be32_to_cpu(id->table_desc.len)); + return -ENOMEM; + } + + sg_init_one(&dummy, md, be32_to_cpu(id->table_desc.len)); + sg_dma_address(&dummy) = token; + sg_dma_len(&dummy) = be32_to_cpu(id->table_desc.len); + err = rdma_io(sc, &dummy, 1, &id->table_desc, 1, DMA_TO_DEVICE, + be32_to_cpu(id->table_desc.len)); + if (err) { + pr_err("libsrp: Error copying indirect table %d\n", + err); + goto free_mem; + } + } else { + pr_err("libsrp: This command uses external indirect buffer\n"); + return -EINVAL; + } + +rdma: + if (dma_map) { + nsg = dma_map_sg(iue->target->dev, sg, scsi_sg_count(sc), + DMA_BIDIRECTIONAL); + if (!nsg) { + pr_err("libsrp: fail to map %p %d\n", + iue, scsi_sg_count(sc)); + err = -EIO; + goto free_mem; + } + len = min(scsi_bufflen(sc), be32_to_cpu(id->len)); + } else + len = be32_to_cpu(id->len); + + err = rdma_io(sc, sg, nsg, md, nmd, dir, len); + + if (dma_map) + dma_unmap_sg(iue->target->dev, sg, nsg, DMA_BIDIRECTIONAL); + +free_mem: + if (token && dma_map) { + dma_free_coherent(iue->target->dev, + be32_to_cpu(id->table_desc.len), md, token); + } + return err; +} + +static int data_out_desc_size(struct srp_cmd *cmd) +{ + int size = 0; + u8 fmt = cmd->buf_fmt >> 4; + + switch (fmt) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + size = sizeof(struct srp_direct_buf); + break; + case SRP_DATA_DESC_INDIRECT: + size = sizeof(struct srp_indirect_buf) + + sizeof(struct srp_direct_buf) * cmd->data_out_desc_cnt; + break; + default: + pr_err("libsrp: client error. Invalid data_out_format %x\n", + fmt); + break; + } + return size; +} + +/* + * TODO: this can be called multiple times for a single command if it + * has very long data. + */ +int srp_transfer_data(struct scsi_cmnd *sc, struct srp_cmd *cmd, + srp_rdma_t rdma_io, int dma_map, int ext_desc) +{ + struct srp_direct_buf *md; + struct srp_indirect_buf *id; + enum dma_data_direction dir; + int offset, err = 0; + u8 format; + + offset = cmd->add_cdb_len & ~3; + + dir = srp_cmd_direction(cmd); + if (dir == DMA_FROM_DEVICE) + offset += data_out_desc_size(cmd); + + if (dir == DMA_TO_DEVICE) + format = cmd->buf_fmt >> 4; + else + format = cmd->buf_fmt & ((1U << 4) - 1); + + switch (format) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + md = (struct srp_direct_buf *) + (cmd->add_data + offset); + err = srp_direct_data(sc, md, dir, rdma_io, dma_map, ext_desc); + break; + case SRP_DATA_DESC_INDIRECT: + id = (struct srp_indirect_buf *) + (cmd->add_data + offset); + err = srp_indirect_data(sc, cmd, id, dir, rdma_io, dma_map, + ext_desc); + break; + default: + pr_err("libsrp: Unknown format %d %x\n", dir, format); + err = -EINVAL; + } + + return err; +} +EXPORT_SYMBOL_GPL(srp_transfer_data); + +u64 srp_data_length(struct srp_cmd *cmd, enum dma_data_direction dir) +{ + struct srp_direct_buf *md; + struct srp_indirect_buf *id; + u64 len = 0; + unsigned offset = cmd->add_cdb_len & ~3; + u8 fmt; + + if (dir == DMA_TO_DEVICE) + fmt = cmd->buf_fmt >> 4; + else { + fmt = cmd->buf_fmt & ((1U << 4) - 1); + offset += data_out_desc_size(cmd); + } + + switch (fmt) { + case SRP_NO_DATA_DESC: + break; + case SRP_DATA_DESC_DIRECT: + md = (struct srp_direct_buf *) (cmd->add_data + offset); + len = be32_to_cpu(md->len); + break; + case SRP_DATA_DESC_INDIRECT: + id = (struct srp_indirect_buf *) (cmd->add_data + offset); + len = be32_to_cpu(id->len); + break; + default: + pr_err("invalid data format %x\n", fmt); + break; + } + return len; +} +EXPORT_SYMBOL_GPL(srp_data_length); + +MODULE_DESCRIPTION("SCSI RDMA Protocol lib functions"); +MODULE_AUTHOR("FUJITA Tomonori"); +MODULE_LICENSE("GPL"); diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index 9872f3429e53ae..e6c5bcf2416217 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -2929,6 +2929,7 @@ struct qlt_hw_data { uint8_t tgt_node_name[WWN_SIZE]; + struct dentry *dfs_tgt_sess; struct list_head q_full_list; uint32_t num_pend_cmds; uint32_t num_qfull_cmds_alloc; diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index cd8b96a4b0dd61..34272fde8a5b0d 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -12,6 +12,47 @@ static struct dentry *qla2x00_dfs_root; static atomic_t qla2x00_dfs_root_count; +static int +qla2x00_dfs_tgt_sess_show(struct seq_file *s, void *unused) +{ + scsi_qla_host_t *vha = s->private; + struct qla_hw_data *ha = vha->hw; + unsigned long flags; + struct qla_tgt_sess *sess = NULL; + struct qla_tgt *tgt= vha->vha_tgt.qla_tgt; + + seq_printf(s, "%s\n",vha->host_str); + if (tgt) { + seq_printf(s, "Port ID Port Name Handle\n"); + + spin_lock_irqsave(&ha->tgt.sess_lock, flags); + list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) { + seq_printf(s, "%02x:%02x:%02x %8phC %d\n", + sess->s_id.b.domain,sess->s_id.b.area, + sess->s_id.b.al_pa, sess->port_name, + sess->loop_id); + } + spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); + } + + return 0; +} + +static int +qla2x00_dfs_tgt_sess_open(struct inode *inode, struct file *file) +{ + scsi_qla_host_t *vha = inode->i_private; + return single_open(file, qla2x00_dfs_tgt_sess_show, vha); +} + + +static const struct file_operations dfs_tgt_sess_ops = { + .open = qla2x00_dfs_tgt_sess_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + static int qla_dfs_fw_resource_cnt_show(struct seq_file *s, void *unused) { @@ -248,6 +289,15 @@ qla2x00_dfs_setup(scsi_qla_host_t *vha) "Unable to create debugfs fce node.\n"); goto out; } + + ha->tgt.dfs_tgt_sess = debugfs_create_file("tgt_sess", + S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_sess_ops); + if (!ha->tgt.dfs_tgt_sess) { + ql_log(ql_log_warn, vha, 0xffff, + "Unable to create debugFS tgt_sess node.\n"); + goto out; + } + out: return 0; } @@ -257,6 +307,11 @@ qla2x00_dfs_remove(scsi_qla_host_t *vha) { struct qla_hw_data *ha = vha->hw; + if (ha->tgt.dfs_tgt_sess) { + debugfs_remove(ha->tgt.dfs_tgt_sess); + ha->tgt.dfs_tgt_sess = NULL; + } + if (ha->dfs_fw_resource_cnt) { debugfs_remove(ha->dfs_fw_resource_cnt); ha->dfs_fw_resource_cnt = NULL; diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index ee967becd257e1..985231900aca42 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -641,7 +641,8 @@ void qlt_unreg_sess(struct qla_tgt_sess *sess) { struct scsi_qla_host *vha = sess->vha; - vha->hw->tgt.tgt_ops->clear_nacl_from_fcport_map(sess); + if (sess->se_sess) + vha->hw->tgt.tgt_ops->clear_nacl_from_fcport_map(sess); if (!list_empty(&sess->del_list_entry)) list_del_init(&sess->del_list_entry); @@ -856,8 +857,12 @@ static void qlt_del_sess_work_fn(struct delayed_work *work) ql_dbg(ql_dbg_tgt_mgt, vha, 0xf004, "Timeout: sess %p about to be deleted\n", sess); - ha->tgt.tgt_ops->shutdown_sess(sess); - ha->tgt.tgt_ops->put_sess(sess); + if (sess->se_sess) { + ha->tgt.tgt_ops->shutdown_sess(sess); + ha->tgt.tgt_ops->put_sess(sess); + } else { + qlt_unreg_sess(sess); + } } else { schedule_delayed_work(&tgt->sess_del_work, sess->expires - elapsed); @@ -879,7 +884,6 @@ static struct qla_tgt_sess *qlt_create_sess( struct qla_hw_data *ha = vha->hw; struct qla_tgt_sess *sess; unsigned long flags; - unsigned char be_sid[3]; /* Check to avoid double sessions */ spin_lock_irqsave(&ha->tgt.sess_lock, flags); @@ -905,6 +909,14 @@ static struct qla_tgt_sess *qlt_create_sess( if (sess->deleted) qlt_undelete_sess(sess); + if (!sess->se_sess) { + if (ha->tgt.tgt_ops->check_initiator_node_acl(vha, + &sess->port_name[0], sess) < 0) { + spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); + return NULL; + } + } + kref_get(&sess->se_sess->sess_kref); ha->tgt.tgt_ops->update_sess(sess, fcport->d_id, fcport->loop_id, (fcport->flags & FCF_CONF_COMP_SUPPORTED)); @@ -948,26 +960,6 @@ static struct qla_tgt_sess *qlt_create_sess( "Adding sess %p to tgt %p via ->check_initiator_node_acl()\n", sess, vha->vha_tgt.qla_tgt); - be_sid[0] = sess->s_id.b.domain; - be_sid[1] = sess->s_id.b.area; - be_sid[2] = sess->s_id.b.al_pa; - /* - * Determine if this fc_port->port_name is allowed to access - * target mode using explict NodeACLs+MappedLUNs, or using - * TPG demo mode. If this is successful a target mode FC nexus - * is created. - */ - if (ha->tgt.tgt_ops->check_initiator_node_acl(vha, - &fcport->port_name[0], sess, &be_sid[0], fcport->loop_id) < 0) { - kfree(sess); - return NULL; - } - /* - * Take an extra reference to ->sess_kref here to handle qla_tgt_sess - * access across ->tgt.sess_lock reaquire. - */ - kref_get(&sess->se_sess->sess_kref); - sess->conf_compl_supported = (fcport->flags & FCF_CONF_COMP_SUPPORTED); BUILD_BUG_ON(sizeof(sess->port_name) != sizeof(fcport->port_name)); memcpy(sess->port_name, fcport->port_name, sizeof(sess->port_name)); @@ -985,6 +977,23 @@ static struct qla_tgt_sess *qlt_create_sess( fcport->loop_id, sess->s_id.b.domain, sess->s_id.b.area, sess->s_id.b.al_pa, sess->conf_compl_supported ? "" : "not "); + /* + * Determine if this fc_port->port_name is allowed to access + * target mode using explict NodeACLs+MappedLUNs, or using + * TPG demo mode. If this is successful a target mode FC nexus + * is created. + */ + if (ha->tgt.tgt_ops->check_initiator_node_acl(vha, + &fcport->port_name[0], sess) < 0) { + return NULL; + } else { + /* + * Take an extra reference to ->sess_kref here to handle qla_tgt_sess + * access across ->tgt.sess_lock reaquire. + */ + kref_get(&sess->se_sess->sess_kref); + } + return sess; } diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h index 22a6a767fe07b1..d857feeb65146e 100644 --- a/drivers/scsi/qla2xxx/qla_target.h +++ b/drivers/scsi/qla2xxx/qla_target.h @@ -731,7 +731,7 @@ struct qla_tgt_func_tmpl { void (*free_session)(struct qla_tgt_sess *); int (*check_initiator_node_acl)(struct scsi_qla_host *, unsigned char *, - void *, uint8_t *, uint16_t); + struct qla_tgt_sess *); void (*update_sess)(struct qla_tgt_sess *, port_id_t, uint16_t, bool); struct qla_tgt_sess *(*find_sess_by_loop_id)(struct scsi_qla_host *, const uint16_t); diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c index 1808a01cfb7e79..c1461d225f08b4 100644 --- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c +++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c @@ -1406,6 +1406,39 @@ static void tcm_qla2xxx_free_session(struct qla_tgt_sess *sess) transport_deregister_session(sess->se_sess); } +static int tcm_qla2xxx_session_cb(struct se_portal_group *se_tpg, + struct se_session *se_sess, void *p) +{ + struct tcm_qla2xxx_tpg *tpg = container_of(se_tpg, + struct tcm_qla2xxx_tpg, se_tpg); + struct tcm_qla2xxx_lport *lport = tpg->lport; + struct qla_hw_data *ha = lport->qla_vha->hw; + struct se_node_acl *se_nacl = se_sess->se_node_acl; + struct tcm_qla2xxx_nacl *nacl = container_of(se_nacl, + struct tcm_qla2xxx_nacl, se_node_acl); + struct qla_tgt_sess *qlat_sess = p; + uint16_t loop_id = qlat_sess->loop_id; + unsigned long flags; + unsigned char be_sid[3]; + + be_sid[0] = qlat_sess->s_id.b.domain; + be_sid[1] = qlat_sess->s_id.b.area; + be_sid[2] = qlat_sess->s_id.b.al_pa; + + /* + * And now setup se_nacl and session pointers into HW lport internal + * mappings for fabric S_ID and LOOP_ID. + */ + spin_lock_irqsave(&ha->tgt.sess_lock, flags); + tcm_qla2xxx_set_sess_by_s_id(lport, se_nacl, nacl, + se_sess, qlat_sess, be_sid); + tcm_qla2xxx_set_sess_by_loop_id(lport, se_nacl, nacl, + se_sess, qlat_sess, loop_id); + spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); + + return 0; +} + /* * Called via qlt_create_sess():ha->qla2x_tmpl->check_initiator_node_acl() * to locate struct se_node_acl @@ -1413,20 +1446,13 @@ static void tcm_qla2xxx_free_session(struct qla_tgt_sess *sess) static int tcm_qla2xxx_check_initiator_node_acl( scsi_qla_host_t *vha, unsigned char *fc_wwpn, - void *qla_tgt_sess, - uint8_t *s_id, - uint16_t loop_id) + struct qla_tgt_sess *qlat_sess) { struct qla_hw_data *ha = vha->hw; struct tcm_qla2xxx_lport *lport; struct tcm_qla2xxx_tpg *tpg; - struct tcm_qla2xxx_nacl *nacl; - struct se_portal_group *se_tpg; - struct se_node_acl *se_nacl; struct se_session *se_sess; - struct qla_tgt_sess *sess = qla_tgt_sess; unsigned char port_name[36]; - unsigned long flags; int num_tags = (ha->cur_fw_xcb_count) ? ha->cur_fw_xcb_count : TCM_QLA2XXX_DEFAULT_TAGS; @@ -1444,15 +1470,6 @@ static int tcm_qla2xxx_check_initiator_node_acl( pr_err("Unable to lcoate struct tcm_qla2xxx_lport->tpg_1\n"); return -EINVAL; } - se_tpg = &tpg->se_tpg; - - se_sess = transport_init_session_tags(num_tags, - sizeof(struct qla_tgt_cmd), - TARGET_PROT_ALL); - if (IS_ERR(se_sess)) { - pr_err("Unable to initialize struct se_session\n"); - return PTR_ERR(se_sess); - } /* * Format the FCP Initiator port_name into colon seperated values to * match the format by tcm_qla2xxx explict ConfigFS NodeACLs. @@ -1463,28 +1480,12 @@ static int tcm_qla2xxx_check_initiator_node_acl( * Locate our struct se_node_acl either from an explict NodeACL created * via ConfigFS, or via running in TPG demo mode. */ - se_sess->se_node_acl = core_tpg_check_initiator_node_acl(se_tpg, - port_name); - if (!se_sess->se_node_acl) { - transport_free_session(se_sess); - return -EINVAL; - } - se_nacl = se_sess->se_node_acl; - nacl = container_of(se_nacl, struct tcm_qla2xxx_nacl, se_node_acl); - /* - * And now setup the new se_nacl and session pointers into our HW lport - * mappings for fabric S_ID and LOOP_ID. - */ - spin_lock_irqsave(&ha->tgt.sess_lock, flags); - tcm_qla2xxx_set_sess_by_s_id(lport, se_nacl, nacl, se_sess, - qla_tgt_sess, s_id); - tcm_qla2xxx_set_sess_by_loop_id(lport, se_nacl, nacl, se_sess, - qla_tgt_sess, loop_id); - spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); - /* - * Finally register the new FC Nexus with TCM - */ - transport_register_session(se_nacl->se_tpg, se_nacl, se_sess, sess); + se_sess = target_alloc_session(&tpg->se_tpg, num_tags, + sizeof(struct qla_tgt_cmd), + TARGET_PROT_ALL, port_name, + qlat_sess, tcm_qla2xxx_session_cb); + if (IS_ERR(se_sess)) + return PTR_ERR(se_sess); return 0; } diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index fa6b2c4eb7a2b1..5405242cdabc57 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -271,7 +271,7 @@ int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd, { char *sense = NULL; int result; - + if (sshdr) { sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO); if (!sense) @@ -374,7 +374,7 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev) spin_unlock_irqrestore(shost->host_lock, flags); scsi_kick_queue(sdev->request_queue); spin_lock_irqsave(shost->host_lock, flags); - + scsi_device_put(sdev); } out: @@ -676,6 +676,7 @@ static void scsi_release_buffers(struct scsi_cmnd *cmd) if (scsi_prot_sg_count(cmd)) scsi_free_sgtable(cmd->prot_sdb, false); } +EXPORT_SYMBOL(scsi_release_buffers); static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd) { @@ -1089,7 +1090,7 @@ static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb) req->mq_ctx != NULL))) return BLKPREP_DEFER; - /* + /* * Next, walk the list, and fill in the addresses and sizes of * each segment. */ @@ -1835,7 +1836,7 @@ static void scsi_request_fn(struct request_queue *q) if (!scsi_host_queue_ready(q, shost, sdev)) goto host_not_ready; - + if (sdev->simple_tags) cmd->flags |= SCMD_TAGGED; else @@ -2386,7 +2387,7 @@ scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage, real_buffer[1] = data->medium_type; real_buffer[2] = data->device_specific; real_buffer[3] = data->block_descriptor_length; - + cmd[0] = MODE_SELECT; cmd[4] = len; @@ -2470,7 +2471,7 @@ scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage, if (scsi_sense_valid(sshdr)) { if ((sshdr->sense_key == ILLEGAL_REQUEST) && (sshdr->asc == 0x20) && (sshdr->ascq == 0)) { - /* + /* * Invalid command operation code */ sdev->use_10_for_ms = 0; @@ -2562,7 +2563,7 @@ EXPORT_SYMBOL(scsi_test_unit_ready); * @sdev: scsi device to change the state of. * @state: state to change to. * - * Returns zero if unsuccessful or an error if the requested + * Returns zero if unsuccessful or an error if the requested * transition is illegal. */ int @@ -2582,7 +2583,7 @@ scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state) goto illegal; } break; - + case SDEV_RUNNING: switch (oldstate) { case SDEV_CREATED: @@ -2859,7 +2860,7 @@ EXPORT_SYMBOL_GPL(sdev_evt_send_simple); * (which must be a legal transition). When the device is in this * state, only special requests will be accepted, all others will * be deferred. Since special requests may also be requeued requests, - * a successful return doesn't guarantee the device will be + * a successful return doesn't guarantee the device will be * totally quiescent. * * Must be called with user context, may sleep. @@ -2940,7 +2941,7 @@ EXPORT_SYMBOL(scsi_target_resume); * * Returns zero if successful or error if not * - * Notes: + * Notes: * This routine transitions the device to the SDEV_BLOCK state * (which must be a legal transition). When the device is in this * state, all commands are deferred until the scsi lld reenables @@ -2961,10 +2962,10 @@ scsi_internal_device_block(struct scsi_device *sdev) return err; } - /* + /* * The device has transitioned to SDEV_BLOCK. Stop the * block layer from calling the midlayer with this device's - * request queue. + * request queue. */ if (q->mq_ops) { blk_mq_stop_hw_queues(q); @@ -2977,7 +2978,7 @@ scsi_internal_device_block(struct scsi_device *sdev) return 0; } EXPORT_SYMBOL_GPL(scsi_internal_device_block); - + /** * scsi_internal_device_unblock - resume a device after a block request * @sdev: device to resume @@ -2989,7 +2990,7 @@ EXPORT_SYMBOL_GPL(scsi_internal_device_block); * * Returns zero if successful or error if not. * - * Notes: + * Notes: * This routine transitions the device to the SDEV_RUNNING state * or to one of the offline states (which must be a legal transition) * allowing the midlayer to goose the queue for this device. @@ -2998,7 +2999,7 @@ int scsi_internal_device_unblock(struct scsi_device *sdev, enum scsi_device_state new_state) { - struct request_queue *q = sdev->request_queue; + struct request_queue *q = sdev->request_queue; unsigned long flags; /* diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h index 27b4d0a6a01dbc..d2fba62f24b66f 100644 --- a/drivers/scsi/scsi_priv.h +++ b/drivers/scsi/scsi_priv.h @@ -35,7 +35,7 @@ extern void scsi_destroy_command_freelist(struct Scsi_Host *shost); void scsi_log_send(struct scsi_cmnd *cmd); void scsi_log_completion(struct scsi_cmnd *cmd, int disposition); #else -static inline void scsi_log_send(struct scsi_cmnd *cmd) +static inline void scsi_log_send(struct scsi_cmnd *cmd) { }; static inline void scsi_log_completion(struct scsi_cmnd *cmd, int disposition) { }; @@ -180,7 +180,7 @@ static inline void scsi_dh_release_device(struct scsi_device *sdev) { } #endif static inline void scsi_dh_remove_device(struct scsi_device *sdev) { } -/* +/* * internal scsi timeout functions: for use by mid-layer and transport * classes. */ diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c index e3cd3ece44121c..2bc7a88b3043be 100644 --- a/drivers/scsi/scsi_transport_srp.c +++ b/drivers/scsi/scsi_transport_srp.c @@ -129,6 +129,7 @@ show_srp_rport_id(struct device *dev, struct device_attribute *attr, char *buf) { struct srp_rport *rport = transport_class_to_srp_rport(dev); + return sprintf(buf, SRP_PID_FMT "\n", SRP_PID(rport)); } @@ -835,12 +836,14 @@ static int srp_tsk_mgmt_response(struct Scsi_Host *shost, u64 nexus, u64 tm_id, int result) { struct srp_internal *i = to_srp_internal(shost->transportt); + return i->f->tsk_mgmt_response(shost, nexus, tm_id, result); } static int srp_it_nexus_response(struct Scsi_Host *shost, u64 nexus, int result) { struct srp_internal *i = to_srp_internal(shost->transportt); + return i->f->it_nexus_response(shost, nexus, result); } diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c index d41a5c300e31a6..e0ffb03aab52e7 100644 --- a/drivers/target/loopback/tcm_loop.c +++ b/drivers/target/loopback/tcm_loop.c @@ -806,54 +806,35 @@ static int tcm_loop_make_nexus( struct tcm_loop_tpg *tl_tpg, const char *name) { - struct se_portal_group *se_tpg; struct tcm_loop_hba *tl_hba = tl_tpg->tl_hba; struct tcm_loop_nexus *tl_nexus; - int ret = -ENOMEM; + int ret; if (tl_tpg->tl_nexus) { pr_debug("tl_tpg->tl_nexus already exists\n"); return -EEXIST; } - se_tpg = &tl_tpg->tl_se_tpg; tl_nexus = kzalloc(sizeof(struct tcm_loop_nexus), GFP_KERNEL); if (!tl_nexus) { pr_err("Unable to allocate struct tcm_loop_nexus\n"); return -ENOMEM; } - /* - * Initialize the struct se_session pointer - */ - tl_nexus->se_sess = transport_init_session( - TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS); + + tl_nexus->se_sess = target_alloc_session(&tl_tpg->tl_se_tpg, 0, 0, + TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS, + name, tl_nexus, NULL); if (IS_ERR(tl_nexus->se_sess)) { ret = PTR_ERR(tl_nexus->se_sess); - goto out; - } - /* - * Since we are running in 'demo mode' this call with generate a - * struct se_node_acl for the tcm_loop struct se_portal_group with the SCSI - * Initiator port name of the passed configfs group 'name'. - */ - tl_nexus->se_sess->se_node_acl = core_tpg_check_initiator_node_acl( - se_tpg, (unsigned char *)name); - if (!tl_nexus->se_sess->se_node_acl) { - transport_free_session(tl_nexus->se_sess); - goto out; + kfree(tl_nexus); + return ret; } - /* Now, register the I_T Nexus as active. */ - transport_register_session(se_tpg, tl_nexus->se_sess->se_node_acl, - tl_nexus->se_sess, tl_nexus); + tl_tpg->tl_nexus = tl_nexus; pr_debug("TCM_Loop_ConfigFS: Established I_T Nexus to emulated" " %s Initiator Port: %s\n", tcm_loop_dump_proto_id(tl_hba), name); return 0; - -out: - kfree(tl_nexus); - return ret; } static int tcm_loop_drop_nexus( diff --git a/drivers/target/sbp/sbp_target.c b/drivers/target/sbp/sbp_target.c index 3072f1aca8ec5a..a04b0605f8d021 100644 --- a/drivers/target/sbp/sbp_target.c +++ b/drivers/target/sbp/sbp_target.c @@ -196,45 +196,30 @@ static struct sbp_session *sbp_session_create( struct sbp_session *sess; int ret; char guid_str[17]; - struct se_node_acl *se_nacl; + + snprintf(guid_str, sizeof(guid_str), "%016llx", guid); sess = kmalloc(sizeof(*sess), GFP_KERNEL); if (!sess) { pr_err("failed to allocate session descriptor\n"); return ERR_PTR(-ENOMEM); } + spin_lock_init(&sess->lock); + INIT_LIST_HEAD(&sess->login_list); + INIT_DELAYED_WORK(&sess->maint_work, session_maintenance_work); + sess->guid = guid; - sess->se_sess = transport_init_session(TARGET_PROT_NORMAL); + sess->se_sess = target_alloc_session(&tpg->se_tpg, 128, + sizeof(struct sbp_target_request), + TARGET_PROT_NORMAL, guid_str, + sess, NULL); if (IS_ERR(sess->se_sess)) { pr_err("failed to init se_session\n"); - ret = PTR_ERR(sess->se_sess); kfree(sess); return ERR_PTR(ret); } - snprintf(guid_str, sizeof(guid_str), "%016llx", guid); - - se_nacl = core_tpg_check_initiator_node_acl(&tpg->se_tpg, guid_str); - if (!se_nacl) { - pr_warn("Node ACL not found for %s\n", guid_str); - - transport_free_session(sess->se_sess); - kfree(sess); - - return ERR_PTR(-EPERM); - } - - sess->se_sess->se_node_acl = se_nacl; - - spin_lock_init(&sess->lock); - INIT_LIST_HEAD(&sess->login_list); - INIT_DELAYED_WORK(&sess->maint_work, session_maintenance_work); - - sess->guid = guid; - - transport_register_session(&tpg->se_tpg, se_nacl, sess->se_sess, sess); - return sess; } @@ -908,7 +893,6 @@ static void tgt_agent_process_work(struct work_struct *work) STATUS_BLOCK_SBP_STATUS( SBP_STATUS_REQ_TYPE_NOTSUPP)); sbp_send_status(req); - sbp_free_request(req); return; case 3: /* Dummy ORB */ req->status.status |= cpu_to_be32( @@ -919,7 +903,6 @@ static void tgt_agent_process_work(struct work_struct *work) STATUS_BLOCK_SBP_STATUS( SBP_STATUS_DUMMY_ORB_COMPLETE)); sbp_send_status(req); - sbp_free_request(req); return; default: BUG(); @@ -938,6 +921,24 @@ static inline bool tgt_agent_check_active(struct sbp_target_agent *agent) return active; } +static struct sbp_target_request *sbp_mgt_get_req(struct sbp_session *sess, + struct fw_card *card, u64 next_orb) +{ + struct se_session *se_sess = sess->se_sess; + struct sbp_target_request *req; + int tag; + + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, GFP_ATOMIC); + if (tag < 0) + return ERR_PTR(-ENOMEM); + + req = &((struct sbp_target_request *)se_sess->sess_cmd_map)[tag]; + req->se_cmd.map_tag = tag; + req->se_cmd.tag = next_orb; + + return req; +} + static void tgt_agent_fetch_work(struct work_struct *work) { struct sbp_target_agent *agent = @@ -949,8 +950,8 @@ static void tgt_agent_fetch_work(struct work_struct *work) u64 next_orb = agent->orb_pointer; while (next_orb && tgt_agent_check_active(agent)) { - req = kzalloc(sizeof(*req), GFP_KERNEL); - if (!req) { + req = sbp_mgt_get_req(sess, sess->card, next_orb); + if (IS_ERR(req)) { spin_lock_bh(&agent->lock); agent->state = AGENT_STATE_DEAD; spin_unlock_bh(&agent->lock); @@ -985,7 +986,6 @@ static void tgt_agent_fetch_work(struct work_struct *work) spin_unlock_bh(&agent->lock); sbp_send_status(req); - sbp_free_request(req); return; } @@ -1232,7 +1232,7 @@ static void sbp_handle_command(struct sbp_target_request *req) req->se_cmd.tag = req->orb_pointer; if (target_submit_cmd(&req->se_cmd, sess->se_sess, req->cmd_buf, req->sense_buf, unpacked_lun, data_length, - TCM_SIMPLE_TAG, data_dir, 0)) + TCM_SIMPLE_TAG, data_dir, TARGET_SCF_ACK_KREF)) goto err; return; @@ -1244,7 +1244,6 @@ static void sbp_handle_command(struct sbp_target_request *req) STATUS_BLOCK_LEN(1) | STATUS_BLOCK_SBP_STATUS(SBP_STATUS_UNSPECIFIED_ERROR)); sbp_send_status(req); - sbp_free_request(req); } /* @@ -1343,22 +1342,29 @@ static int sbp_rw_data(struct sbp_target_request *req) static int sbp_send_status(struct sbp_target_request *req) { - int ret, length; + int rc, ret = 0, length; struct sbp_login_descriptor *login = req->login; length = (((be32_to_cpu(req->status.status) >> 24) & 0x07) + 1) * 4; - ret = sbp_run_request_transaction(req, TCODE_WRITE_BLOCK_REQUEST, + rc = sbp_run_request_transaction(req, TCODE_WRITE_BLOCK_REQUEST, login->status_fifo_addr, &req->status, length); - if (ret != RCODE_COMPLETE) { - pr_debug("sbp_send_status: write failed: 0x%x\n", ret); - return -EIO; + if (rc != RCODE_COMPLETE) { + pr_debug("sbp_send_status: write failed: 0x%x\n", rc); + ret = -EIO; + goto put_ref; } pr_debug("sbp_send_status: status write complete for ORB: 0x%llx\n", req->orb_pointer); - - return 0; + /* + * Drop the extra ACK_KREF reference taken by target_submit_cmd() + * ahead of sbp_check_stop_free() -> transport_generic_free_cmd() + * final se_cmd->cmd_kref put. + */ +put_ref: + target_put_sess_cmd(&req->se_cmd); + return ret; } static void sbp_sense_mangle(struct sbp_target_request *req) @@ -1447,9 +1453,13 @@ static int sbp_send_sense(struct sbp_target_request *req) static void sbp_free_request(struct sbp_target_request *req) { + struct se_cmd *se_cmd = &req->se_cmd; + struct se_session *se_sess = se_cmd->se_sess; + kfree(req->pg_tbl); kfree(req->cmd_buf); - kfree(req); + + percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); } static void sbp_mgt_agent_process(struct work_struct *work) @@ -1609,8 +1619,12 @@ static void sbp_mgt_agent_rw(struct fw_card *card, rcode = RCODE_CONFLICT_ERROR; goto out; } - + // XXX: +#if 0 + req = sbp_mgt_get_req(agent->login->sess, card); +#else req = kzalloc(sizeof(*req), GFP_ATOMIC); +#endif if (!req) { rcode = RCODE_CONFLICT_ERROR; goto out; @@ -1815,8 +1829,7 @@ static int sbp_check_stop_free(struct se_cmd *se_cmd) struct sbp_target_request *req = container_of(se_cmd, struct sbp_target_request, se_cmd); - transport_generic_free_cmd(&req->se_cmd, 0); - return 1; + return transport_generic_free_cmd(&req->se_cmd, 0); } static int sbp_count_se_tpg_luns(struct se_portal_group *tpg) diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c index da457e25717a60..a4046ca6e60da8 100644 --- a/drivers/target/target_core_device.c +++ b/drivers/target/target_core_device.c @@ -86,7 +86,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun) se_cmd->lun_ref_active = true; if ((se_cmd->data_direction == DMA_TO_DEVICE) && - (deve->lun_flags & TRANSPORT_LUNFLAGS_READ_ONLY)) { + deve->lun_access_ro) { pr_err("TARGET_CORE[%s]: Detected WRITE_PROTECTED LUN" " Access for 0x%08llx\n", se_cmd->se_tfo->get_fabric_name(), @@ -199,7 +199,7 @@ bool target_lun_is_rdonly(struct se_cmd *cmd) rcu_read_lock(); deve = target_nacl_find_deve(se_sess->se_node_acl, cmd->orig_fe_lun); - ret = (deve && deve->lun_flags & TRANSPORT_LUNFLAGS_READ_ONLY); + ret = deve && deve->lun_access_ro; rcu_read_unlock(); return ret; @@ -258,22 +258,15 @@ void core_free_device_list_for_node( void core_update_device_list_access( u64 mapped_lun, - u32 lun_access, + bool lun_access_ro, struct se_node_acl *nacl) { struct se_dev_entry *deve; mutex_lock(&nacl->lun_entry_mutex); deve = target_nacl_find_deve(nacl, mapped_lun); - if (deve) { - if (lun_access & TRANSPORT_LUNFLAGS_READ_WRITE) { - deve->lun_flags &= ~TRANSPORT_LUNFLAGS_READ_ONLY; - deve->lun_flags |= TRANSPORT_LUNFLAGS_READ_WRITE; - } else { - deve->lun_flags &= ~TRANSPORT_LUNFLAGS_READ_WRITE; - deve->lun_flags |= TRANSPORT_LUNFLAGS_READ_ONLY; - } - } + if (deve) + deve->lun_access_ro = lun_access_ro; mutex_unlock(&nacl->lun_entry_mutex); } @@ -319,7 +312,7 @@ int core_enable_device_list_for_node( struct se_lun *lun, struct se_lun_acl *lun_acl, u64 mapped_lun, - u32 lun_access, + bool lun_access_ro, struct se_node_acl *nacl, struct se_portal_group *tpg) { @@ -340,11 +333,7 @@ int core_enable_device_list_for_node( kref_init(&new->pr_kref); init_completion(&new->pr_comp); - if (lun_access & TRANSPORT_LUNFLAGS_READ_WRITE) - new->lun_flags |= TRANSPORT_LUNFLAGS_READ_WRITE; - else - new->lun_flags |= TRANSPORT_LUNFLAGS_READ_ONLY; - + new->lun_access_ro = lun_access_ro; new->creation_time = get_jiffies_64(); new->attach_count++; @@ -433,7 +422,7 @@ void core_disable_device_list_for_node( hlist_del_rcu(&orig->link); clear_bit(DEF_PR_REG_ACTIVE, &orig->deve_flags); - orig->lun_flags = 0; + orig->lun_access_ro = false; orig->creation_time = 0; orig->attach_count--; /* @@ -558,8 +547,7 @@ int core_dev_add_lun( { int rc; - rc = core_tpg_add_lun(tpg, lun, - TRANSPORT_LUNFLAGS_READ_WRITE, dev); + rc = core_tpg_add_lun(tpg, lun, false, dev); if (rc < 0) return rc; @@ -635,7 +623,7 @@ int core_dev_add_initiator_node_lun_acl( struct se_portal_group *tpg, struct se_lun_acl *lacl, struct se_lun *lun, - u32 lun_access) + bool lun_access_ro) { struct se_node_acl *nacl = lacl->se_lun_nacl; /* @@ -647,20 +635,19 @@ int core_dev_add_initiator_node_lun_acl( if (!nacl) return -EINVAL; - if ((lun->lun_access & TRANSPORT_LUNFLAGS_READ_ONLY) && - (lun_access & TRANSPORT_LUNFLAGS_READ_WRITE)) - lun_access = TRANSPORT_LUNFLAGS_READ_ONLY; + if (lun->lun_access_ro) + lun_access_ro = true; lacl->se_lun = lun; if (core_enable_device_list_for_node(lun, lacl, lacl->mapped_lun, - lun_access, nacl, tpg) < 0) + lun_access_ro, nacl, tpg) < 0) return -EINVAL; pr_debug("%s_TPG[%hu]_LUN[%llu->%llu] - Added %s ACL for " " InitiatorNode: %s\n", tpg->se_tpg_tfo->get_fabric_name(), tpg->se_tpg_tfo->tpg_get_tag(tpg), lun->unpacked_lun, lacl->mapped_lun, - (lun_access & TRANSPORT_LUNFLAGS_READ_WRITE) ? "RW" : "RO", + lun_access_ro ? "RO" : "RW", nacl->initiatorname); /* * Check to see if there are any existing persistent reservation APTPL diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c index f916d18ccb487c..8cc68be712301d 100644 --- a/drivers/target/target_core_fabric_configfs.c +++ b/drivers/target/target_core_fabric_configfs.c @@ -78,7 +78,7 @@ static int target_fabric_mappedlun_link( struct se_lun_acl, se_lun_group); struct se_portal_group *se_tpg; struct config_item *nacl_ci, *tpg_ci, *tpg_ci_s, *wwn_ci, *wwn_ci_s; - int lun_access; + bool lun_access_ro; if (lun->lun_link_magic != SE_LUN_LINK_MAGIC) { pr_err("Bad lun->lun_link_magic, not a valid lun_ci pointer:" @@ -115,19 +115,18 @@ static int target_fabric_mappedlun_link( } /* * If this struct se_node_acl was dynamically generated with - * tpg_1/attrib/generate_node_acls=1, use the existing deve->lun_flags, - * which be will write protected (READ-ONLY) when + * tpg_1/attrib/generate_node_acls=1, use the existing + * deve->lun_access_ro value, which will be true when * tpg_1/attrib/demo_mode_write_protect=1 */ rcu_read_lock(); deve = target_nacl_find_deve(lacl->se_lun_nacl, lacl->mapped_lun); if (deve) - lun_access = deve->lun_flags; + lun_access_ro = deve->lun_access_ro; else - lun_access = + lun_access_ro = (se_tpg->se_tpg_tfo->tpg_check_prod_mode_write_protect( - se_tpg)) ? TRANSPORT_LUNFLAGS_READ_ONLY : - TRANSPORT_LUNFLAGS_READ_WRITE; + se_tpg)) ? true : false; rcu_read_unlock(); /* * Determine the actual mapped LUN value user wants.. @@ -135,7 +134,7 @@ static int target_fabric_mappedlun_link( * This value is what the SCSI Initiator actually sees the * $FABRIC/$WWPN/$TPGT/lun/lun_* as on their SCSI Initiator Ports. */ - return core_dev_add_initiator_node_lun_acl(se_tpg, lacl, lun, lun_access); + return core_dev_add_initiator_node_lun_acl(se_tpg, lacl, lun, lun_access_ro); } static int target_fabric_mappedlun_unlink( @@ -167,8 +166,7 @@ static ssize_t target_fabric_mappedlun_write_protect_show( rcu_read_lock(); deve = target_nacl_find_deve(se_nacl, lacl->mapped_lun); if (deve) { - len = sprintf(page, "%d\n", - (deve->lun_flags & TRANSPORT_LUNFLAGS_READ_ONLY) ? 1 : 0); + len = sprintf(page, "%d\n", deve->lun_access_ro); } rcu_read_unlock(); @@ -181,25 +179,23 @@ static ssize_t target_fabric_mappedlun_write_protect_store( struct se_lun_acl *lacl = item_to_lun_acl(item); struct se_node_acl *se_nacl = lacl->se_lun_nacl; struct se_portal_group *se_tpg = se_nacl->se_tpg; - unsigned long op; + unsigned long wp; int ret; - ret = kstrtoul(page, 0, &op); + ret = kstrtoul(page, 0, &wp); if (ret) return ret; - if ((op != 1) && (op != 0)) + if ((wp != 1) && (wp != 0)) return -EINVAL; - core_update_device_list_access(lacl->mapped_lun, (op) ? - TRANSPORT_LUNFLAGS_READ_ONLY : - TRANSPORT_LUNFLAGS_READ_WRITE, - lacl->se_lun_nacl); + /* wp=1 means lun_access_ro=true */ + core_update_device_list_access(lacl->mapped_lun, wp, lacl->se_lun_nacl); pr_debug("%s_ConfigFS: Changed Initiator ACL: %s" " Mapped LUN: %llu Write Protect bit to %s\n", se_tpg->se_tpg_tfo->get_fabric_name(), - se_nacl->initiatorname, lacl->mapped_lun, (op) ? "ON" : "OFF"); + se_nacl->initiatorname, lacl->mapped_lun, (wp) ? "ON" : "OFF"); return count; diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index abe4eb997a8422..ce754f15b7a9e4 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -282,11 +282,28 @@ static void iblock_complete_cmd(struct se_cmd *cmd) if (!atomic_dec_and_test(&ibr->pending)) return; - - if (atomic_read(&ibr->ib_bio_err_cnt)) - status = SAM_STAT_CHECK_CONDITION; - else + /* + * Propigate use these two bio completion values from raw block + * drivers to signal up BUSY and TASK_SET_FULL status to the + * host side initiator. The latter for Linux/iSCSI initiators + * means the Linux/SCSI LLD will begin to reduce it's internal + * per session queue_depth. + */ + if (atomic_read(&ibr->ib_bio_err_cnt)) { + switch (ibr->ib_bio_retry) { + case -EAGAIN: + status = SAM_STAT_BUSY; + break; + case -ENOMEM: + status = SAM_STAT_TASK_SET_FULL; + break; + default: + status = SAM_STAT_CHECK_CONDITION; + break; + } + } else { status = SAM_STAT_GOOD; + } target_complete_cmd(cmd, status); kfree(ibr); @@ -298,7 +315,15 @@ static void iblock_bio_done(struct bio *bio) struct iblock_req *ibr = cmd->priv; if (bio->bi_error) { - pr_err("bio error: %p, err: %d\n", bio, bio->bi_error); + pr_debug_ratelimited("test_bit(BIO_UPTODATE) failed for bio: %p," + " err: %d\n", bio, bio->bi_error); + /* + * Save the retryable status provided and translate into + * SAM status in iblock_complete_cmd(). + */ + if (bio->bi_error == -EAGAIN || bio->bi_error == -ENOMEM) { + ibr->ib_bio_retry = bio->bi_error; + } /* * Bump the ib_bio_err_cnt and release bio. */ @@ -412,9 +437,40 @@ iblock_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb) return 0; } +static sense_reason_t +iblock_execute_write_same_direct(struct block_device *bdev, struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + struct scatterlist *sg = &cmd->t_data_sg[0]; + struct page *page = NULL; + int ret; + + if (sg->offset) { + page = alloc_page(GFP_KERNEL); + if (!page) + return TCM_OUT_OF_RESOURCES; + sg_copy_to_buffer(sg, cmd->t_data_nents, page_address(page), + dev->dev_attrib.block_size); + } + + ret = blkdev_issue_write_same(bdev, + target_to_linux_sector(dev, cmd->t_task_lba), + target_to_linux_sector(dev, + sbc_get_write_same_sectors(cmd)), + GFP_KERNEL, page ? page : sg_page(sg)); + if (page) + __free_page(page); + if (ret) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + + target_complete_cmd(cmd, GOOD); + return 0; +} + static sense_reason_t iblock_execute_write_same(struct se_cmd *cmd) { + struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd; struct iblock_req *ibr; struct scatterlist *sg; struct bio *bio; @@ -439,6 +495,9 @@ iblock_execute_write_same(struct se_cmd *cmd) return TCM_INVALID_CDB_FIELD; } + if (bdev_write_same(bdev)) + return iblock_execute_write_same_direct(bdev, cmd); + ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL); if (!ibr) goto fail; @@ -643,8 +702,7 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, struct scatterlist *sg; u32 sg_num = sgl_nents; unsigned bio_cnt; - int rw = 0; - int i; + int i, rw = 0; if (data_direction == DMA_TO_DEVICE) { struct iblock_dev *ib_dev = IBLOCK_DEV(dev); diff --git a/drivers/target/target_core_iblock.h b/drivers/target/target_core_iblock.h index 01c2afd815008d..ff3cfdd7b469ee 100644 --- a/drivers/target/target_core_iblock.h +++ b/drivers/target/target_core_iblock.h @@ -9,6 +9,7 @@ struct iblock_req { atomic_t pending; atomic_t ib_bio_err_cnt; + int ib_bio_retry; } ____cacheline_aligned; #define IBDF_HAS_UDEV_PATH 0x01 diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h index db4412fe6b8a3f..040cf5202e548b 100644 --- a/drivers/target/target_core_internal.h +++ b/drivers/target/target_core_internal.h @@ -60,10 +60,10 @@ struct se_dev_entry *core_get_se_deve_from_rtpi(struct se_node_acl *, u16); void target_pr_kref_release(struct kref *); void core_free_device_list_for_node(struct se_node_acl *, struct se_portal_group *); -void core_update_device_list_access(u64, u32, struct se_node_acl *); +void core_update_device_list_access(u64, bool, struct se_node_acl *); struct se_dev_entry *target_nacl_find_deve(struct se_node_acl *, u64); int core_enable_device_list_for_node(struct se_lun *, struct se_lun_acl *, - u64, u32, struct se_node_acl *, struct se_portal_group *); + u64, bool, struct se_node_acl *, struct se_portal_group *); void core_disable_device_list_for_node(struct se_lun *, struct se_dev_entry *, struct se_node_acl *, struct se_portal_group *); void core_clear_lun_from_tpg(struct se_lun *, struct se_portal_group *); @@ -73,7 +73,7 @@ void core_dev_del_lun(struct se_portal_group *, struct se_lun *); struct se_lun_acl *core_dev_init_initiator_node_lun_acl(struct se_portal_group *, struct se_node_acl *, u64, int *); int core_dev_add_initiator_node_lun_acl(struct se_portal_group *, - struct se_lun_acl *, struct se_lun *lun, u32); + struct se_lun_acl *, struct se_lun *lun, bool); int core_dev_del_initiator_node_lun_acl(struct se_lun *, struct se_lun_acl *); void core_dev_free_initiator_node_lun_acl(struct se_portal_group *, @@ -119,7 +119,7 @@ void core_tpg_add_node_to_devs(struct se_node_acl *, struct se_portal_group *, void core_tpg_wait_for_nacl_pr_ref(struct se_node_acl *); struct se_lun *core_tpg_alloc_lun(struct se_portal_group *, u64); int core_tpg_add_lun(struct se_portal_group *, struct se_lun *, - u32, struct se_device *); + bool, struct se_device *); void core_tpg_remove_lun(struct se_portal_group *, struct se_lun *); struct se_node_acl *core_tpg_add_initiator_node_acl(struct se_portal_group *tpg, const char *initiatorname); diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c index 0aa47babd16c61..891d441caa6aa5 100644 --- a/drivers/target/target_core_spc.c +++ b/drivers/target/target_core_spc.c @@ -997,7 +997,6 @@ static sense_reason_t spc_emulate_modesense(struct se_cmd *cmd) int length = 0; int ret; int i; - bool read_only = target_lun_is_rdonly(cmd);; memset(buf, 0, SE_MODE_PAGE_BUF); @@ -1008,7 +1007,7 @@ static sense_reason_t spc_emulate_modesense(struct se_cmd *cmd) length = ten ? 3 : 2; /* DEVICE-SPECIFIC PARAMETER */ - if ((cmd->se_lun->lun_access & TRANSPORT_LUNFLAGS_READ_ONLY) || read_only) + if (cmd->se_lun->lun_access_ro || target_lun_is_rdonly(cmd)) spc_modesense_write_protect(&buf[length], type); /* @@ -1212,6 +1211,7 @@ sense_reason_t spc_emulate_report_luns(struct se_cmd *cmd) u32 lun_count = 0, offset = 8; __be32 len; + pr_debug("spc_emulate_report_luns\n"); buf = transport_kmap_data_sg(cmd); if (cmd->data_length && !buf) return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; @@ -1261,6 +1261,8 @@ sense_reason_t spc_emulate_report_luns(struct se_cmd *cmd) if (buf) { len = cpu_to_be32(lun_count * 8); + pr_debug("spc_emulate_report: len:%x luncount:%x\n", + be32_to_cpu(len), lun_count); memcpy(buf, &len, min_t(int, sizeof len, cmd->data_length)); transport_kunmap_data_sg(cmd); } diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c index 3608b1b5ecf799..ddf046080dc3ae 100644 --- a/drivers/target/target_core_tpg.c +++ b/drivers/target/target_core_tpg.c @@ -121,7 +121,7 @@ void core_tpg_add_node_to_devs( struct se_portal_group *tpg, struct se_lun *lun_orig) { - u32 lun_access = 0; + bool lun_access_ro = true; struct se_lun *lun; struct se_device *dev; @@ -137,27 +137,26 @@ void core_tpg_add_node_to_devs( * demo_mode_write_protect is ON, or READ_ONLY; */ if (!tpg->se_tpg_tfo->tpg_check_demo_mode_write_protect(tpg)) { - lun_access = TRANSPORT_LUNFLAGS_READ_WRITE; + lun_access_ro = false; } else { /* * Allow only optical drives to issue R/W in default RO * demo mode. */ if (dev->transport->get_device_type(dev) == TYPE_DISK) - lun_access = TRANSPORT_LUNFLAGS_READ_ONLY; + lun_access_ro = true; else - lun_access = TRANSPORT_LUNFLAGS_READ_WRITE; + lun_access_ro = false; } pr_debug("TARGET_CORE[%s]->TPG[%u]_LUN[%llu] - Adding %s" " access for LUN in Demo Mode\n", tpg->se_tpg_tfo->get_fabric_name(), tpg->se_tpg_tfo->tpg_get_tag(tpg), lun->unpacked_lun, - (lun_access == TRANSPORT_LUNFLAGS_READ_WRITE) ? - "READ-WRITE" : "READ-ONLY"); + lun_access_ro ? "READ-ONLY" : "READ-WRITE"); core_enable_device_list_for_node(lun, NULL, lun->unpacked_lun, - lun_access, acl, tpg); + lun_access_ro, acl, tpg); /* * Check to see if there are any existing persistent reservation * APTPL pre-registrations that need to be enabled for this dynamic @@ -522,7 +521,7 @@ int core_tpg_register( return PTR_ERR(se_tpg->tpg_virt_lun0); ret = core_tpg_add_lun(se_tpg, se_tpg->tpg_virt_lun0, - TRANSPORT_LUNFLAGS_READ_ONLY, g_lun0_dev); + true, g_lun0_dev); if (ret < 0) { kfree(se_tpg->tpg_virt_lun0); return ret; @@ -616,7 +615,7 @@ struct se_lun *core_tpg_alloc_lun( int core_tpg_add_lun( struct se_portal_group *tpg, struct se_lun *lun, - u32 lun_access, + bool lun_access_ro, struct se_device *dev) { int ret; @@ -644,9 +643,9 @@ int core_tpg_add_lun( spin_unlock(&dev->se_port_lock); if (dev->dev_flags & DF_READ_ONLY) - lun->lun_access = TRANSPORT_LUNFLAGS_READ_ONLY; + lun->lun_access_ro = true; else - lun->lun_access = lun_access; + lun->lun_access_ro = lun_access_ro; if (!(dev->se_hba->hba_flags & HBA_FLAGS_INTERNAL_USE)) hlist_add_head_rcu(&lun->link, &tpg->tpg_lun_hlist); mutex_unlock(&tpg->tpg_lun_mutex); diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index f5ad9e063b65ed..25de959247f36a 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -731,11 +731,20 @@ static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) { struct se_device *dev = cmd->se_dev; - int success = scsi_status == GOOD; + int success; unsigned long flags; cmd->scsi_status = scsi_status; - + switch (cmd->scsi_status) { + case SAM_STAT_GOOD: + case SAM_STAT_BUSY: + case SAM_STAT_TASK_SET_FULL: + success = 1; + break; + default: + success = 0; + break; + } spin_lock_irqsave(&cmd->t_state_lock, flags); cmd->transport_state &= ~CMD_T_BUSY; @@ -1471,6 +1480,11 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess sense_reason_t rc; int ret; + pr_debug("target: se_cmd(%p), se_sess(%p), cdb: %x, sense: %x" + " unpacked lun: %llx, data_length: %x, task_attr: %x" + " data_dir: %x, flags: %x\n", se_cmd, se_sess, cdb[0], sense[0], + unpacked_lun, data_length, task_attr, data_dir, flags); + se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); @@ -1997,6 +2011,9 @@ static void transport_complete_qf(struct se_cmd *cmd) switch (cmd->data_direction) { case DMA_FROM_DEVICE: + if (cmd->scsi_status) + goto queue_status; + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_data_in(cmd); break; @@ -2007,6 +2024,7 @@ static void transport_complete_qf(struct se_cmd *cmd) } /* Fall through for DMA_TO_DEVICE */ case DMA_NONE: +queue_status: trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_status(cmd); break; @@ -2128,6 +2146,9 @@ static void target_complete_ok_work(struct work_struct *work) queue_rsp: switch (cmd->data_direction) { case DMA_FROM_DEVICE: + if (cmd->scsi_status) + goto queue_status; + atomic_long_add(cmd->data_length, &cmd->se_lun->lun_stats.tx_data_octets); /* @@ -2167,6 +2188,7 @@ static void target_complete_ok_work(struct work_struct *work) } /* Fall through for DMA_TO_DEVICE */ case DMA_NONE: +queue_status: trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_status(cmd); if (ret == -EAGAIN || ret == -ENOMEM) diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 94f5154ac78859..62bf4fe5704a92 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -63,8 +64,11 @@ #define TCMU_TIME_OUT (30 * MSEC_PER_SEC) +#define DATA_BLOCK_BITS 256 +#define DATA_BLOCK_SIZE 4096 + #define CMDR_SIZE (16 * 4096) -#define DATA_SIZE (257 * 4096) +#define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE) #define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE) @@ -93,12 +97,11 @@ struct tcmu_dev { u32 cmdr_size; u32 cmdr_last_cleaned; /* Offset of data ring from start of mb */ + /* Must add data_off and mb_addr to get the address */ size_t data_off; size_t data_size; - /* Ring head + tail values. */ - /* Must add data_off and mb_addr to get the address */ - size_t data_head; - size_t data_tail; + + DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); wait_queue_head_t wait_cmdr; /* TODO should this be a mutex? */ @@ -122,9 +125,9 @@ struct tcmu_cmd { uint16_t cmd_id; - /* Can't use se_cmd->data_length when cleaning up expired cmds, because if + /* Can't use se_cmd when cleaning up expired cmds, because if cmd has been completed then accessing se_cmd is off limits */ - size_t data_length; + DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); unsigned long deadline; @@ -168,13 +171,6 @@ static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd) tcmu_cmd->se_cmd = se_cmd; tcmu_cmd->tcmu_dev = udev; - tcmu_cmd->data_length = se_cmd->data_length; - - if (se_cmd->se_cmd_flags & SCF_BIDI) { - BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents)); - tcmu_cmd->data_length += se_cmd->t_bidi_data_sg->length; - } - tcmu_cmd->deadline = jiffies + msecs_to_jiffies(TCMU_TIME_OUT); idr_preload(GFP_KERNEL); @@ -231,105 +227,126 @@ static inline size_t head_to_end(size_t head, size_t size) return size - head; } +static inline void new_iov(struct iovec **iov, int *iov_cnt, + struct tcmu_dev *udev) +{ + struct iovec *iovec; + + if (*iov_cnt != 0) + (*iov)++; + (*iov_cnt)++; + + iovec = *iov; + memset(iovec, 0, sizeof(struct iovec)); +} + #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size) +/* offset is relative to mb_addr */ +static inline size_t get_block_offset(struct tcmu_dev *dev, + int block, int remaining) +{ + return dev->data_off + block * DATA_BLOCK_SIZE + + DATA_BLOCK_SIZE - remaining; +} + +static inline size_t iov_tail(struct tcmu_dev *udev, struct iovec *iov) +{ + return (size_t)iov->iov_base + iov->iov_len; +} + static void alloc_and_scatter_data_area(struct tcmu_dev *udev, struct scatterlist *data_sg, unsigned int data_nents, struct iovec **iov, int *iov_cnt, bool copy_data) { - int i; + int i, block; + int block_remaining = 0; void *from, *to; - size_t copy_bytes; + size_t copy_bytes, to_offset; struct scatterlist *sg; for_each_sg(data_sg, sg, data_nents, i) { - copy_bytes = min_t(size_t, sg->length, - head_to_end(udev->data_head, udev->data_size)); + int sg_remaining = sg->length; from = kmap_atomic(sg_page(sg)) + sg->offset; - to = (void *) udev->mb_addr + udev->data_off + udev->data_head; - - if (copy_data) { - memcpy(to, from, copy_bytes); - tcmu_flush_dcache_range(to, copy_bytes); - } - - /* Even iov_base is relative to mb_addr */ - (*iov)->iov_len = copy_bytes; - (*iov)->iov_base = (void __user *) udev->data_off + - udev->data_head; - (*iov_cnt)++; - (*iov)++; - - UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size); - - /* Uh oh, we wrapped the buffer. Must split sg across 2 iovs. */ - if (sg->length != copy_bytes) { - void *from_skip = from + copy_bytes; - - copy_bytes = sg->length - copy_bytes; - - (*iov)->iov_len = copy_bytes; - (*iov)->iov_base = (void __user *) udev->data_off + - udev->data_head; - + while (sg_remaining > 0) { + if (block_remaining == 0) { + block = find_first_zero_bit(udev->data_bitmap, + DATA_BLOCK_BITS); + block_remaining = DATA_BLOCK_SIZE; + set_bit(block, udev->data_bitmap); + } + copy_bytes = min_t(size_t, sg_remaining, + block_remaining); + to_offset = get_block_offset(udev, block, + block_remaining); + to = (void *)udev->mb_addr + to_offset; + if (*iov_cnt != 0 && + to_offset == iov_tail(udev, *iov)) { + (*iov)->iov_len += copy_bytes; + } else { + new_iov(iov, iov_cnt, udev); + (*iov)->iov_base = (void __user *) to_offset; + (*iov)->iov_len = copy_bytes; + } if (copy_data) { - to = (void *) udev->mb_addr + - udev->data_off + udev->data_head; - memcpy(to, from_skip, copy_bytes); + memcpy(to, from + sg->length - sg_remaining, + copy_bytes); tcmu_flush_dcache_range(to, copy_bytes); } - - (*iov_cnt)++; - (*iov)++; - - UPDATE_HEAD(udev->data_head, - copy_bytes, udev->data_size); + sg_remaining -= copy_bytes; + block_remaining -= copy_bytes; } - kunmap_atomic(from - sg->offset); } } -static void gather_and_free_data_area(struct tcmu_dev *udev, - struct scatterlist *data_sg, unsigned int data_nents) +static void free_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd) { - int i; + bitmap_xor(udev->data_bitmap, udev->data_bitmap, cmd->data_bitmap, + DATA_BLOCK_BITS); +} + +static void gather_data_area(struct tcmu_dev *udev, unsigned long *cmd_bitmap, + struct scatterlist *data_sg, unsigned int data_nents) +{ + int i, block; + int block_remaining = 0; void *from, *to; - size_t copy_bytes; + size_t copy_bytes, from_offset; struct scatterlist *sg; - /* It'd be easier to look at entry's iovec again, but UAM */ for_each_sg(data_sg, sg, data_nents, i) { - copy_bytes = min_t(size_t, sg->length, - head_to_end(udev->data_tail, udev->data_size)); - + int sg_remaining = sg->length; to = kmap_atomic(sg_page(sg)) + sg->offset; - WARN_ON(sg->length + sg->offset > PAGE_SIZE); - from = (void *) udev->mb_addr + - udev->data_off + udev->data_tail; - tcmu_flush_dcache_range(from, copy_bytes); - memcpy(to, from, copy_bytes); - - UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size); - - /* Uh oh, wrapped the data buffer for this sg's data */ - if (sg->length != copy_bytes) { - void *to_skip = to + copy_bytes; - - from = (void *) udev->mb_addr + - udev->data_off + udev->data_tail; - WARN_ON(udev->data_tail); - copy_bytes = sg->length - copy_bytes; + while (sg_remaining > 0) { + if (block_remaining == 0) { + block = find_first_bit(cmd_bitmap, + DATA_BLOCK_BITS); + block_remaining = DATA_BLOCK_SIZE; + clear_bit(block, cmd_bitmap); + } + copy_bytes = min_t(size_t, sg_remaining, + block_remaining); + from_offset = get_block_offset(udev, block, + block_remaining); + from = (void *) udev->mb_addr + from_offset; tcmu_flush_dcache_range(from, copy_bytes); - memcpy(to_skip, from, copy_bytes); + memcpy(to + sg->length - sg_remaining, from, + copy_bytes); - UPDATE_HEAD(udev->data_tail, - copy_bytes, udev->data_size); + sg_remaining -= copy_bytes; + block_remaining -= copy_bytes; } kunmap_atomic(to - sg->offset); } } +static inline size_t spc_bitmap_free(unsigned long *bitmap) +{ + return DATA_BLOCK_SIZE * (DATA_BLOCK_BITS - + bitmap_weight(bitmap, DATA_BLOCK_BITS)); +} + /* * We can't queue a command until we have space available on the cmd ring *and* * space available on the data ring. @@ -339,9 +356,8 @@ static void gather_and_free_data_area(struct tcmu_dev *udev, static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed) { struct tcmu_mailbox *mb = udev->mb_addr; - size_t space; + size_t space, cmd_needed; u32 cmd_head; - size_t cmd_needed; tcmu_flush_dcache_range(mb, sizeof(*mb)); @@ -363,10 +379,10 @@ static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t d return false; } - space = spc_free(udev->data_head, udev->data_tail, udev->data_size); + space = spc_bitmap_free(udev->data_bitmap); if (space < data_needed) { - pr_debug("no data space: %zu %zu %zu\n", udev->data_head, - udev->data_tail, udev->data_size); + pr_debug("no data space: only %zu available, but ask for %zu\n", + space, data_needed); return false; } @@ -385,6 +401,8 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) uint32_t cmd_head; uint64_t cdb_off; bool copy_to_data_area; + size_t data_length; + DECLARE_BITMAP(old_bitmap, DATA_BLOCK_BITS); if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) return -EINVAL; @@ -393,12 +411,12 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) * Must be a certain minimum size for response sense info, but * also may be larger if the iov array is large. * - * iovs = sgl_nents+1, for end-of-ring case, plus another 1 - * b/c size == offsetof one-past-element. + * We prepare way too many iovs for potential uses here, because it's + * expensive to tell how many regions are freed in the bitmap */ base_command_size = max(offsetof(struct tcmu_cmd_entry, - req.iov[se_cmd->t_bidi_data_nents + - se_cmd->t_data_nents + 2]), + req.iov[se_cmd->t_bidi_data_nents + + se_cmd->t_data_nents]), sizeof(struct tcmu_cmd_entry)); command_size = base_command_size + round_up(scsi_command_size(se_cmd->t_task_cdb), TCMU_OP_ALIGN_SIZE); @@ -409,13 +427,18 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) mb = udev->mb_addr; cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ + data_length = se_cmd->data_length; + if (se_cmd->se_cmd_flags & SCF_BIDI) { + BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents)); + data_length += se_cmd->t_bidi_data_sg->length; + } if ((command_size > (udev->cmdr_size / 2)) - || tcmu_cmd->data_length > (udev->data_size - 1)) + || data_length > udev->data_size) pr_warn("TCMU: Request of size %zu/%zu may be too big for %u/%zu " - "cmd/data ring buffers\n", command_size, tcmu_cmd->data_length, + "cmd/data ring buffers\n", command_size, data_length, udev->cmdr_size, udev->data_size); - while (!is_ring_space_avail(udev, command_size, tcmu_cmd->data_length)) { + while (!is_ring_space_avail(udev, command_size, data_length)) { int ret; DEFINE_WAIT(__wait); @@ -462,6 +485,8 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) entry->hdr.kflags = 0; entry->hdr.uflags = 0; + bitmap_copy(old_bitmap, udev->data_bitmap, DATA_BLOCK_BITS); + /* * Fix up iovecs, and handle if allocation in data ring wrapped. */ @@ -480,6 +505,10 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) se_cmd->t_bidi_data_nents, &iov, &iov_cnt, false); entry->req.iov_bidi_cnt = iov_cnt; + /* cmd's data_bitmap is what changed in process */ + bitmap_xor(tcmu_cmd->data_bitmap, old_bitmap, udev->data_bitmap, + DATA_BLOCK_BITS); + /* All offsets relative to mb_addr, not start of entry! */ cdb_off = CMDR_OFF + cmd_head + base_command_size; memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb)); @@ -530,35 +559,42 @@ static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry * struct tcmu_dev *udev = cmd->tcmu_dev; if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { - /* cmd has been completed already from timeout, just reclaim data - ring space */ - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + /* + * cmd has been completed already from timeout, just reclaim + * data ring space and free cmd + */ + free_data_area(udev, cmd); + + kmem_cache_free(tcmu_cmd_cache, cmd); return; } if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) { - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n", cmd->se_cmd); entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION; } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) { memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer, se_cmd->scsi_sense_length); - - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); } else if (se_cmd->se_cmd_flags & SCF_BIDI) { - /* Discard data_out buffer */ - UPDATE_HEAD(udev->data_tail, - (size_t)se_cmd->t_data_sg->length, udev->data_size); + DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS); - /* Get Data-In buffer */ - gather_and_free_data_area(udev, + /* Get Data-In buffer before clean up */ + bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS); + gather_data_area(udev, bitmap, se_cmd->t_bidi_data_sg, se_cmd->t_bidi_data_nents); + free_data_area(udev, cmd); } else if (se_cmd->data_direction == DMA_FROM_DEVICE) { - gather_and_free_data_area(udev, + DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS); + + bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS); + gather_data_area(udev, bitmap, se_cmd->t_data_sg, se_cmd->t_data_nents); + free_data_area(udev, cmd); } else if (se_cmd->data_direction == DMA_TO_DEVICE) { - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); } else if (se_cmd->data_direction != DMA_NONE) { pr_warn("TCMU: data direction was %d!\n", se_cmd->data_direction); @@ -894,11 +930,13 @@ static int tcmu_configure_device(struct se_device *dev) mb = udev->mb_addr; mb->version = TCMU_MAILBOX_VERSION; + mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC; mb->cmdr_off = CMDR_OFF; mb->cmdr_size = udev->cmdr_size; WARN_ON(!PAGE_ALIGNED(udev->data_off)); WARN_ON(udev->data_size % PAGE_SIZE); + WARN_ON(udev->data_size % DATA_BLOCK_SIZE); info->version = __stringify(TCMU_MAILBOX_VERSION); @@ -942,12 +980,12 @@ static int tcmu_configure_device(struct se_device *dev) return ret; } -static int tcmu_check_pending_cmd(int id, void *p, void *data) +static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd) { - struct tcmu_cmd *cmd = p; - - if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) + if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { + kmem_cache_free(tcmu_cmd_cache, cmd); return 0; + } return -EINVAL; } @@ -962,6 +1000,8 @@ static void tcmu_dev_call_rcu(struct rcu_head *p) static void tcmu_free_device(struct se_device *dev) { struct tcmu_dev *udev = TCMU_DEV(dev); + struct tcmu_cmd *cmd; + bool all_expired = true; int i; del_timer_sync(&udev->timeout); @@ -970,10 +1010,13 @@ static void tcmu_free_device(struct se_device *dev) /* Upper layer should drain all requests before calling this */ spin_lock_irq(&udev->commands_lock); - i = idr_for_each(&udev->commands, tcmu_check_pending_cmd, NULL); + idr_for_each_entry(&udev->commands, cmd, i) { + if (tcmu_check_and_free_pending_cmd(cmd) != 0) + all_expired = false; + } idr_destroy(&udev->commands); spin_unlock_irq(&udev->commands_lock); - WARN_ON(i); + WARN_ON(!all_expired); /* Device was configured */ if (udev->uio_info.uio_dev) { diff --git a/drivers/target/tcm_fc/tfc_cmd.c b/drivers/target/tcm_fc/tfc_cmd.c index 064d6dfb5b6d93..ea551b0c9f7768 100644 --- a/drivers/target/tcm_fc/tfc_cmd.c +++ b/drivers/target/tcm_fc/tfc_cmd.c @@ -107,8 +107,7 @@ void ft_release_cmd(struct se_cmd *se_cmd) int ft_check_stop_free(struct se_cmd *se_cmd) { - transport_generic_free_cmd(se_cmd, 0); - return 1; + return transport_generic_free_cmd(se_cmd, 0); } /* @@ -179,6 +178,12 @@ int ft_queue_status(struct se_cmd *se_cmd) return -ENOMEM; } lport->tt.exch_done(cmd->seq); + /* + * Drop the extra ACK_KREF reference taken by target_submit_cmd() + * ahead of ft_check_stop_free() -> transport_generic_free_cmd() + * final se_cmd->cmd_kref put. + */ + target_put_sess_cmd(&cmd->se_cmd); return 0; } @@ -422,6 +427,12 @@ void ft_queue_tm_resp(struct se_cmd *se_cmd) pr_debug("tmr fn %d resp %d fcp code %d\n", tmr->function, tmr->response, code); ft_send_resp_code(cmd, code); + /* + * Drop the extra ACK_KREF reference taken by target_submit_tmr() + * ahead of ft_check_stop_free() -> transport_generic_free_cmd() + * final se_cmd->cmd_kref put. + */ + target_put_sess_cmd(&cmd->se_cmd); } void ft_aborted_task(struct se_cmd *se_cmd) @@ -560,7 +571,8 @@ static void ft_send_work(struct work_struct *work) */ if (target_submit_cmd(&cmd->se_cmd, cmd->sess->se_sess, fcp->fc_cdb, &cmd->ft_sense_buffer[0], scsilun_to_int(&fcp->fc_lun), - ntohl(fcp->fc_dl), task_attr, data_dir, 0)) + ntohl(fcp->fc_dl), task_attr, data_dir, + TARGET_SCF_ACK_KREF)) goto err; pr_debug("r_ctl %x alloc target_submit_cmd\n", fh->fh_r_ctl); diff --git a/drivers/target/tcm_fc/tfc_sess.c b/drivers/target/tcm_fc/tfc_sess.c index e19f4c58c6fa56..d0c3e1894c6142 100644 --- a/drivers/target/tcm_fc/tfc_sess.c +++ b/drivers/target/tcm_fc/tfc_sess.c @@ -186,6 +186,20 @@ static struct ft_sess *ft_sess_get(struct fc_lport *lport, u32 port_id) return NULL; } +static int ft_sess_alloc_cb(struct se_portal_group *se_tpg, + struct se_session *se_sess, void *p) +{ + struct ft_sess *sess = p; + struct ft_tport *tport = sess->tport; + struct hlist_head *head = &tport->hash[ft_sess_hash(sess->port_id)]; + + pr_debug("port_id %x sess %p\n", sess->port_id, sess); + hlist_add_head_rcu(&sess->hash, head); + tport->sess_count++; + + return 0; +} + /* * Allocate session and enter it in the hash for the local port. * Caller holds ft_lport_lock. @@ -194,7 +208,6 @@ static struct ft_sess *ft_sess_create(struct ft_tport *tport, u32 port_id, struct fc_rport_priv *rdata) { struct se_portal_group *se_tpg = &tport->tpg->se_tpg; - struct se_node_acl *se_acl; struct ft_sess *sess; struct hlist_head *head; unsigned char initiatorname[TRANSPORT_IQN_LEN]; @@ -210,31 +223,18 @@ static struct ft_sess *ft_sess_create(struct ft_tport *tport, u32 port_id, if (!sess) return NULL; - sess->se_sess = transport_init_session_tags(TCM_FC_DEFAULT_TAGS, - sizeof(struct ft_cmd), - TARGET_PROT_NORMAL); - if (IS_ERR(sess->se_sess)) { - kfree(sess); - return NULL; - } + kref_init(&sess->kref); /* ref for table entry */ + sess->tport = tport; + sess->port_id = port_id; - se_acl = core_tpg_get_initiator_node_acl(se_tpg, &initiatorname[0]); - if (!se_acl) { - transport_free_session(sess->se_sess); + sess->se_sess = target_alloc_session(se_tpg, TCM_FC_DEFAULT_TAGS, + sizeof(struct ft_cmd), + TARGET_PROT_NORMAL, &initiatorname[0], + sess, ft_sess_alloc_cb); + if (IS_ERR(sess->se_sess)) { kfree(sess); return NULL; } - sess->se_sess->se_node_acl = se_acl; - sess->tport = tport; - sess->port_id = port_id; - kref_init(&sess->kref); /* ref for table entry */ - hlist_add_head_rcu(&sess->hash, head); - tport->sess_count++; - - pr_debug("port_id %x sess %p\n", port_id, sess); - - transport_register_session(&tport->tpg->se_tpg, se_acl, - sess->se_sess, sess); return sess; } diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index bad007b5a190c1..e352a31dcaa3e9 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -41,13 +41,6 @@ static inline struct f_uas *to_f_uas(struct usb_function *f) return container_of(f, struct f_uas, function); } -static void usbg_cmd_release(struct kref *); - -static inline void usbg_cleanup_cmd(struct usbg_cmd *cmd) -{ - kref_put(&cmd->ref, usbg_cmd_release); -} - /* Start bot.c code */ static int bot_enqueue_cmd_cbw(struct f_uas *fu) @@ -68,7 +61,7 @@ static void bot_status_complete(struct usb_ep *ep, struct usb_request *req) struct usbg_cmd *cmd = req->context; struct f_uas *fu = cmd->fu; - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); if (req->status < 0) { pr_err("ERR %s(%d)\n", __func__, __LINE__); return; @@ -605,7 +598,7 @@ static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req) break; case UASP_QUEUE_COMMAND: - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); usb_ep_queue(fu->ep_cmd, fu->cmd.req, GFP_ATOMIC); break; @@ -615,7 +608,7 @@ static void uasp_status_data_cmpl(struct usb_ep *ep, struct usb_request *req) return; cleanup: - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); } static int uasp_send_status_response(struct usbg_cmd *cmd) @@ -977,7 +970,7 @@ static void usbg_data_write_cmpl(struct usb_ep *ep, struct usb_request *req) return; cleanup: - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); } static int usbg_prepare_w_request(struct usbg_cmd *cmd, struct usb_request *req) @@ -1046,7 +1039,7 @@ static void usbg_cmd_work(struct work_struct *work) struct se_cmd *se_cmd; struct tcm_usbg_nexus *tv_nexus; struct usbg_tpg *tpg; - int dir; + int dir, flags = (TARGET_SCF_UNKNOWN_SIZE | TARGET_SCF_ACK_KREF); se_cmd = &cmd->se_cmd; tpg = cmd->fu->tpg; @@ -1060,9 +1053,9 @@ static void usbg_cmd_work(struct work_struct *work) goto out; } - if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess, - cmd->cmd_buf, cmd->sense_iu.sense, cmd->unpacked_lun, - 0, cmd->prio_attr, dir, TARGET_SCF_UNKNOWN_SIZE) < 0) + if (target_submit_cmd(se_cmd, tv_nexus->tvn_se_sess, cmd->cmd_buf, + cmd->sense_iu.sense, cmd->unpacked_lun, 0, + cmd->prio_attr, dir, flags) < 0) goto out; return; @@ -1070,42 +1063,63 @@ static void usbg_cmd_work(struct work_struct *work) out: transport_send_check_condition_and_sense(se_cmd, TCM_UNSUPPORTED_SCSI_OPCODE, 1); - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); } +static struct usbg_cmd *usbg_get_cmd(struct f_uas *fu, + struct tcm_usbg_nexus *tv_nexus, u32 scsi_tag) +{ + struct se_session *se_sess = tv_nexus->tvn_se_sess; + struct usbg_cmd *cmd; + int tag; + + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, GFP_ATOMIC); + if (tag < 0) + return ERR_PTR(-ENOMEM); + + cmd = &((struct usbg_cmd *)se_sess->sess_cmd_map)[tag]; + cmd->se_cmd.map_tag = tag; + cmd->se_cmd.tag = cmd->tag = scsi_tag; + cmd->fu = fu; + + return cmd; +} + +static void usbg_release_cmd(struct se_cmd *); + static int usbg_submit_command(struct f_uas *fu, void *cmdbuf, unsigned int len) { struct command_iu *cmd_iu = cmdbuf; struct usbg_cmd *cmd; - struct usbg_tpg *tpg; - struct tcm_usbg_nexus *tv_nexus; + struct usbg_tpg *tpg = fu->tpg; + struct tcm_usbg_nexus *tv_nexus = tpg->tpg_nexus; u32 cmd_len; + u16 scsi_tag; if (cmd_iu->iu_id != IU_ID_COMMAND) { pr_err("Unsupported type %d\n", cmd_iu->iu_id); return -EINVAL; } - cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC); - if (!cmd) - return -ENOMEM; - - cmd->fu = fu; - - /* XXX until I figure out why I can't free in on complete */ - kref_init(&cmd->ref); - kref_get(&cmd->ref); + tv_nexus = tpg->tpg_nexus; + if (!tv_nexus) { + pr_err("Missing nexus, ignoring command\n"); + return -EINVAL; + } - tpg = fu->tpg; cmd_len = (cmd_iu->len & ~0x3) + 16; if (cmd_len > USBG_MAX_CMD) - goto err; + return -EINVAL; + scsi_tag = be16_to_cpup(&cmd_iu->tag); + cmd = usbg_get_cmd(fu, tv_nexus, scsi_tag); + if (IS_ERR(cmd)) { + pr_err("usbg_get_cmd failed\n"); + return -ENOMEM; + } memcpy(cmd->cmd_buf, cmd_iu->cdb, cmd_len); - cmd->tag = be16_to_cpup(&cmd_iu->tag); - cmd->se_cmd.tag = cmd->tag; if (fu->flags & USBG_USE_STREAMS) { if (cmd->tag > UASP_SS_EP_COMP_NUM_STREAMS) goto err; @@ -1117,12 +1131,6 @@ static int usbg_submit_command(struct f_uas *fu, cmd->stream = &fu->stream[0]; } - tv_nexus = tpg->tpg_nexus; - if (!tv_nexus) { - pr_err("Missing nexus, ignoring command\n"); - goto err; - } - switch (cmd_iu->prio_attr & 0x7) { case UAS_HEAD_TAG: cmd->prio_attr = TCM_HEAD_TAG; @@ -1148,7 +1156,7 @@ static int usbg_submit_command(struct f_uas *fu, return 0; err: - kfree(cmd); + usbg_release_cmd(&cmd->se_cmd); return -EINVAL; } @@ -1182,7 +1190,7 @@ static void bot_cmd_work(struct work_struct *work) out: transport_send_check_condition_and_sense(se_cmd, TCM_UNSUPPORTED_SCSI_OPCODE, 1); - usbg_cleanup_cmd(cmd); + transport_generic_free_cmd(&cmd->se_cmd, 0); } static int bot_submit_command(struct f_uas *fu, @@ -1190,7 +1198,7 @@ static int bot_submit_command(struct f_uas *fu, { struct bulk_cb_wrap *cbw = cmdbuf; struct usbg_cmd *cmd; - struct usbg_tpg *tpg; + struct usbg_tpg *tpg = fu->tpg; struct tcm_usbg_nexus *tv_nexus; u32 cmd_len; @@ -1207,28 +1215,20 @@ static int bot_submit_command(struct f_uas *fu, if (cmd_len < 1 || cmd_len > 16) return -EINVAL; - cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC); - if (!cmd) - return -ENOMEM; - - cmd->fu = fu; - - /* XXX until I figure out why I can't free in on complete */ - kref_init(&cmd->ref); - kref_get(&cmd->ref); - - tpg = fu->tpg; - - memcpy(cmd->cmd_buf, cbw->CDB, cmd_len); - - cmd->bot_tag = cbw->Tag; - tv_nexus = tpg->tpg_nexus; if (!tv_nexus) { pr_err("Missing nexus, ignoring command\n"); - goto err; + return -ENODEV; + } + + cmd = usbg_get_cmd(fu, tv_nexus, cbw->Tag); + if (IS_ERR(cmd)) { + pr_err("usbg_get_cmd failed\n"); + return -ENOMEM; } + memcpy(cmd->cmd_buf, cbw->CDB, cmd_len); + cmd->bot_tag = cbw->Tag; cmd->prio_attr = TCM_SIMPLE_TAG; cmd->unpacked_lun = cbw->Lun; cmd->is_read = cbw->Flags & US_BULK_FLAG_IN ? 1 : 0; @@ -1239,9 +1239,6 @@ static int bot_submit_command(struct f_uas *fu, queue_work(tpg->workqueue, &cmd->work); return 0; -err: - kfree(cmd); - return -EINVAL; } /* Start fabric.c code */ @@ -1282,20 +1279,14 @@ static u32 usbg_tpg_get_inst_index(struct se_portal_group *se_tpg) return 1; } -static void usbg_cmd_release(struct kref *ref) -{ - struct usbg_cmd *cmd = container_of(ref, struct usbg_cmd, - ref); - - transport_generic_free_cmd(&cmd->se_cmd, 0); -} - static void usbg_release_cmd(struct se_cmd *se_cmd) { struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd, se_cmd); + struct se_session *se_sess = se_cmd->se_sess; + kfree(cmd->data_buf); - kfree(cmd); + percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); } static int usbg_shutdown_session(struct se_session *se_sess) @@ -1581,53 +1572,35 @@ static ssize_t tcm_usbg_tpg_nexus_show(struct config_item *item, char *page) static int tcm_usbg_make_nexus(struct usbg_tpg *tpg, char *name) { - struct se_portal_group *se_tpg; struct tcm_usbg_nexus *tv_nexus; - int ret; + int ret = 0; mutex_lock(&tpg->tpg_mutex); if (tpg->tpg_nexus) { ret = -EEXIST; pr_debug("tpg->tpg_nexus already exists\n"); - goto err_unlock; + goto out_unlock; } - se_tpg = &tpg->se_tpg; - ret = -ENOMEM; tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL); - if (!tv_nexus) - goto err_unlock; - tv_nexus->tvn_se_sess = transport_init_session(TARGET_PROT_NORMAL); - if (IS_ERR(tv_nexus->tvn_se_sess)) - goto err_free; + if (!tv_nexus) { + ret = -ENOMEM; + goto out_unlock; + } - /* - * Since we are running in 'demo mode' this call with generate a - * struct se_node_acl for the tcm_vhost struct se_portal_group with - * the SCSI Initiator port name of the passed configfs group 'name'. - */ - tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl( - se_tpg, name); - if (!tv_nexus->tvn_se_sess->se_node_acl) { + tv_nexus->tvn_se_sess = target_alloc_session(&tpg->se_tpg, 128, + sizeof(struct usbg_cmd), + TARGET_PROT_NORMAL, name, + tv_nexus, NULL); + if (IS_ERR(tv_nexus->tvn_se_sess)) { #define MAKE_NEXUS_MSG "core_tpg_check_initiator_node_acl() failed for %s\n" pr_debug(MAKE_NEXUS_MSG, name); #undef MAKE_NEXUS_MSG - goto err_session; + ret = PTR_ERR(tv_nexus->tvn_se_sess); + kfree(tv_nexus); } - /* - * Now register the TCM vHost virtual I_T Nexus as active. - */ - transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl, - tv_nexus->tvn_se_sess, tv_nexus); - tpg->tpg_nexus = tv_nexus; - mutex_unlock(&tpg->tpg_mutex); - return 0; -err_session: - transport_free_session(tv_nexus->tvn_se_sess); -err_free: - kfree(tv_nexus); -err_unlock: +out_unlock: mutex_unlock(&tpg->tpg_mutex); return ret; } @@ -1735,11 +1708,7 @@ static void usbg_port_unlink(struct se_portal_group *se_tpg, static int usbg_check_stop_free(struct se_cmd *se_cmd) { - struct usbg_cmd *cmd = container_of(se_cmd, struct usbg_cmd, - se_cmd); - - kref_put(&cmd->ref, usbg_cmd_release); - return 1; + return target_put_sess_cmd(se_cmd); } static const struct target_core_fabric_ops usbg_ops = { diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 29cfc57d496e94..cd5f20f14d5ad8 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -1664,8 +1664,7 @@ static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg, mutex_unlock(&vhost_scsi_mutex); } -static void vhost_scsi_free_cmd_map_res(struct vhost_scsi_nexus *nexus, - struct se_session *se_sess) +static void vhost_scsi_free_cmd_map_res(struct se_session *se_sess) { struct vhost_scsi_cmd *tv_cmd; unsigned int i; @@ -1721,98 +1720,82 @@ static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = { NULL, }; -static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg, - const char *name) +static int vhost_scsi_nexus_cb(struct se_portal_group *se_tpg, + struct se_session *se_sess, void *p) { - struct se_portal_group *se_tpg; - struct se_session *se_sess; - struct vhost_scsi_nexus *tv_nexus; struct vhost_scsi_cmd *tv_cmd; unsigned int i; - mutex_lock(&tpg->tv_tpg_mutex); - if (tpg->tpg_nexus) { - mutex_unlock(&tpg->tv_tpg_mutex); - pr_debug("tpg->tpg_nexus already exists\n"); - return -EEXIST; - } - se_tpg = &tpg->se_tpg; - - tv_nexus = kzalloc(sizeof(struct vhost_scsi_nexus), GFP_KERNEL); - if (!tv_nexus) { - mutex_unlock(&tpg->tv_tpg_mutex); - pr_err("Unable to allocate struct vhost_scsi_nexus\n"); - return -ENOMEM; - } - /* - * Initialize the struct se_session pointer and setup tagpool - * for struct vhost_scsi_cmd descriptors - */ - tv_nexus->tvn_se_sess = transport_init_session_tags( - VHOST_SCSI_DEFAULT_TAGS, - sizeof(struct vhost_scsi_cmd), - TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS); - if (IS_ERR(tv_nexus->tvn_se_sess)) { - mutex_unlock(&tpg->tv_tpg_mutex); - kfree(tv_nexus); - return -ENOMEM; - } - se_sess = tv_nexus->tvn_se_sess; for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) { tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i]; tv_cmd->tvc_sgl = kzalloc(sizeof(struct scatterlist) * VHOST_SCSI_PREALLOC_SGLS, GFP_KERNEL); if (!tv_cmd->tvc_sgl) { - mutex_unlock(&tpg->tv_tpg_mutex); pr_err("Unable to allocate tv_cmd->tvc_sgl\n"); goto out; } tv_cmd->tvc_upages = kzalloc(sizeof(struct page *) * - VHOST_SCSI_PREALLOC_UPAGES, GFP_KERNEL); + VHOST_SCSI_PREALLOC_UPAGES, GFP_KERNEL); if (!tv_cmd->tvc_upages) { - mutex_unlock(&tpg->tv_tpg_mutex); pr_err("Unable to allocate tv_cmd->tvc_upages\n"); goto out; } tv_cmd->tvc_prot_sgl = kzalloc(sizeof(struct scatterlist) * - VHOST_SCSI_PREALLOC_PROT_SGLS, GFP_KERNEL); + VHOST_SCSI_PREALLOC_PROT_SGLS, GFP_KERNEL); if (!tv_cmd->tvc_prot_sgl) { - mutex_unlock(&tpg->tv_tpg_mutex); pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n"); goto out; } } + return 0; +out: + vhost_scsi_free_cmd_map_res(se_sess); + return -ENOMEM; +} + +static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg, + const char *name) +{ + struct se_portal_group *se_tpg; + struct vhost_scsi_nexus *tv_nexus; + + mutex_lock(&tpg->tv_tpg_mutex); + if (tpg->tpg_nexus) { + mutex_unlock(&tpg->tv_tpg_mutex); + pr_debug("tpg->tpg_nexus already exists\n"); + return -EEXIST; + } + se_tpg = &tpg->se_tpg; + + tv_nexus = kzalloc(sizeof(struct vhost_scsi_nexus), GFP_KERNEL); + if (!tv_nexus) { + mutex_unlock(&tpg->tv_tpg_mutex); + pr_err("Unable to allocate struct vhost_scsi_nexus\n"); + return -ENOMEM; + } /* * Since we are running in 'demo mode' this call with generate a * struct se_node_acl for the vhost_scsi struct se_portal_group with * the SCSI Initiator port name of the passed configfs group 'name'. */ - tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl( - se_tpg, (unsigned char *)name); - if (!tv_nexus->tvn_se_sess->se_node_acl) { + tv_nexus->tvn_se_sess = target_alloc_session(&tpg->se_tpg, + VHOST_SCSI_DEFAULT_TAGS, + sizeof(struct vhost_scsi_cmd), + TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS, + (unsigned char *)name, tv_nexus, + vhost_scsi_nexus_cb); + if (IS_ERR(tv_nexus->tvn_se_sess)) { mutex_unlock(&tpg->tv_tpg_mutex); - pr_debug("core_tpg_check_initiator_node_acl() failed" - " for %s\n", name); - goto out; + kfree(tv_nexus); + return -ENOMEM; } - /* - * Now register the TCM vhost virtual I_T Nexus as active. - */ - transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl, - tv_nexus->tvn_se_sess, tv_nexus); tpg->tpg_nexus = tv_nexus; mutex_unlock(&tpg->tv_tpg_mutex); return 0; - -out: - vhost_scsi_free_cmd_map_res(tv_nexus, se_sess); - transport_free_session(se_sess); - kfree(tv_nexus); - return -ENOMEM; } static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg) @@ -1853,7 +1836,7 @@ static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg) " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport), tv_nexus->tvn_se_sess->se_node_acl->initiatorname); - vhost_scsi_free_cmd_map_res(tv_nexus, se_sess); + vhost_scsi_free_cmd_map_res(se_sess); /* * Release the SCSI I_T Nexus to the emulated vhost Target Port */ diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index ad4eb1024d1ffb..c3f55a210866a0 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c @@ -190,7 +190,6 @@ module_param_named(max_buffer_pages, scsiback_max_buffer_pages, int, 0644); MODULE_PARM_DESC(max_buffer_pages, "Maximum number of free pages to keep in backend buffer"); -static struct kmem_cache *scsiback_cachep; static DEFINE_SPINLOCK(free_pages_lock); static int free_pages_num; static LIST_HEAD(scsiback_free_pages); @@ -321,11 +320,11 @@ static void scsiback_free_translation_entry(struct kref *kref) kfree(entry); } -static void scsiback_do_resp_with_sense(char *sense_buffer, int32_t result, - uint32_t resid, struct vscsibk_pend *pending_req) +static void scsiback_send_response(struct vscsibk_info *info, + char *sense_buffer, int32_t result, uint32_t resid, + uint16_t rqid) { struct vscsiif_response *ring_res; - struct vscsibk_info *info = pending_req->info; int notify; struct scsi_sense_hdr sshdr; unsigned long flags; @@ -337,7 +336,7 @@ static void scsiback_do_resp_with_sense(char *sense_buffer, int32_t result, info->ring.rsp_prod_pvt++; ring_res->rslt = result; - ring_res->rqid = pending_req->rqid; + ring_res->rqid = rqid; if (sense_buffer != NULL && scsi_normalize_sense(sense_buffer, VSCSIIF_SENSE_BUFFERSIZE, @@ -357,6 +356,13 @@ static void scsiback_do_resp_with_sense(char *sense_buffer, int32_t result, if (notify) notify_remote_via_irq(info->irq); +} + +static void scsiback_do_resp_with_sense(char *sense_buffer, int32_t result, + uint32_t resid, struct vscsibk_pend *pending_req) +{ + scsiback_send_response(pending_req->info, sense_buffer, result, + resid, pending_req->rqid); if (pending_req->v2p) kref_put(&pending_req->v2p->kref, @@ -380,6 +386,12 @@ static void scsiback_cmd_done(struct vscsibk_pend *pending_req) scsiback_fast_flush_area(pending_req); scsiback_do_resp_with_sense(sense_buffer, errors, resid, pending_req); scsiback_put(info); + /* + * Drop the extra KREF_ACK reference taken by target_submit_cmd_map_sgls() + * ahead of scsiback_check_stop_free() -> transport_generic_free_cmd() + * final se_cmd->cmd_kref put. + */ + target_put_sess_cmd(&pending_req->se_cmd); } static void scsiback_cmd_exec(struct vscsibk_pend *pending_req) @@ -388,16 +400,12 @@ static void scsiback_cmd_exec(struct vscsibk_pend *pending_req) struct se_session *sess = pending_req->v2p->tpg->tpg_nexus->tvn_se_sess; int rc; - memset(pending_req->sense_buffer, 0, VSCSIIF_SENSE_BUFFERSIZE); - - memset(se_cmd, 0, sizeof(*se_cmd)); - scsiback_get(pending_req->info); se_cmd->tag = pending_req->rqid; rc = target_submit_cmd_map_sgls(se_cmd, sess, pending_req->cmnd, pending_req->sense_buffer, pending_req->v2p->lun, pending_req->data_len, 0, - pending_req->sc_data_direction, 0, + pending_req->sc_data_direction, TARGET_SCF_ACK_KREF, pending_req->sgl, pending_req->n_sg, NULL, 0, NULL, 0); if (rc < 0) { @@ -586,45 +594,40 @@ static void scsiback_disconnect(struct vscsibk_info *info) static void scsiback_device_action(struct vscsibk_pend *pending_req, enum tcm_tmreq_table act, int tag) { - int rc, err = FAILED; struct scsiback_tpg *tpg = pending_req->v2p->tpg; + struct scsiback_nexus *nexus = tpg->tpg_nexus; struct se_cmd *se_cmd = &pending_req->se_cmd; struct scsiback_tmr *tmr; + u64 unpacked_lun = pending_req->v2p->lun; + int rc, err = FAILED; tmr = kzalloc(sizeof(struct scsiback_tmr), GFP_KERNEL); - if (!tmr) - goto out; + if (!tmr) { + target_put_sess_cmd(se_cmd); + goto err; + } init_waitqueue_head(&tmr->tmr_wait); - transport_init_se_cmd(se_cmd, tpg->se_tpg.se_tpg_tfo, - tpg->tpg_nexus->tvn_se_sess, 0, DMA_NONE, TCM_SIMPLE_TAG, - &pending_req->sense_buffer[0]); - - rc = core_tmr_alloc_req(se_cmd, tmr, act, GFP_KERNEL); - if (rc < 0) - goto out; - - se_cmd->se_tmr_req->ref_task_tag = tag; - - if (transport_lookup_tmr_lun(se_cmd, pending_req->v2p->lun) < 0) - goto out; + rc = target_submit_tmr(&pending_req->se_cmd, nexus->tvn_se_sess, + &pending_req->sense_buffer[0], + unpacked_lun, tmr, act, GFP_KERNEL, + tag, TARGET_SCF_ACK_KREF); + if (rc) + goto err; - transport_generic_handle_tmr(se_cmd); wait_event(tmr->tmr_wait, atomic_read(&tmr->tmr_complete)); err = (se_cmd->se_tmr_req->response == TMR_FUNCTION_COMPLETE) ? SUCCESS : FAILED; -out: - if (tmr) { - transport_generic_free_cmd(&pending_req->se_cmd, 1); + scsiback_do_resp_with_sense(NULL, err, 0, pending_req); + transport_generic_free_cmd(&pending_req->se_cmd, 1); + return; +err: + if (tmr) kfree(tmr); - } - scsiback_do_resp_with_sense(NULL, err, 0, pending_req); - - kmem_cache_free(scsiback_cachep, pending_req); } /* @@ -653,15 +656,53 @@ static struct v2p_entry *scsiback_do_translation(struct vscsibk_info *info, return entry; } -static int prepare_pending_reqs(struct vscsibk_info *info, - struct vscsiif_request *ring_req, - struct vscsibk_pend *pending_req) +static struct vscsibk_pend *scsiback_get_pend_req(struct vscsiif_back_ring *ring, + struct v2p_entry *v2p) { + struct scsiback_tpg *tpg = v2p->tpg; + struct scsiback_nexus *nexus = tpg->tpg_nexus; + struct se_session *se_sess = nexus->tvn_se_sess; + struct vscsibk_pend *req; + int tag, i; + + tag = percpu_ida_alloc(&se_sess->sess_tag_pool, TASK_RUNNING); + if (tag < 0) { + pr_err("Unable to obtain tag for vscsiif_request\n"); + return ERR_PTR(-ENOMEM); + } + + req = &((struct vscsibk_pend *)se_sess->sess_cmd_map)[tag]; + memset(req, 0, sizeof(*req)); + req->se_cmd.map_tag = tag; + + for (i = 0; i < VSCSI_MAX_GRANTS; i++) + req->grant_handles[i] = SCSIBACK_INVALID_HANDLE; + + return req; +} + +static struct vscsibk_pend *prepare_pending_reqs(struct vscsibk_info *info, + struct vscsiif_back_ring *ring, + struct vscsiif_request *ring_req) +{ + struct vscsibk_pend *pending_req; struct v2p_entry *v2p; struct ids_tuple vir; - pending_req->rqid = ring_req->rqid; - pending_req->info = info; + /* request range check from frontend */ + if ((ring_req->sc_data_direction != DMA_BIDIRECTIONAL) && + (ring_req->sc_data_direction != DMA_TO_DEVICE) && + (ring_req->sc_data_direction != DMA_FROM_DEVICE) && + (ring_req->sc_data_direction != DMA_NONE)) { + pr_debug("invalid parameter data_dir = %d\n", + ring_req->sc_data_direction); + return ERR_PTR(-EINVAL); + } + if (ring_req->cmd_len > VSCSIIF_MAX_COMMAND_SIZE) { + pr_debug("invalid parameter cmd_len = %d\n", + ring_req->cmd_len); + return ERR_PTR(-EINVAL); + } vir.chn = ring_req->channel; vir.tgt = ring_req->id; @@ -669,33 +710,24 @@ static int prepare_pending_reqs(struct vscsibk_info *info, v2p = scsiback_do_translation(info, &vir); if (!v2p) { - pending_req->v2p = NULL; pr_debug("the v2p of (chn:%d, tgt:%d, lun:%d) doesn't exist.\n", - vir.chn, vir.tgt, vir.lun); - return -ENODEV; + vir.chn, vir.tgt, vir.lun); + return ERR_PTR(-ENODEV); } - pending_req->v2p = v2p; - /* request range check from frontend */ - pending_req->sc_data_direction = ring_req->sc_data_direction; - if ((pending_req->sc_data_direction != DMA_BIDIRECTIONAL) && - (pending_req->sc_data_direction != DMA_TO_DEVICE) && - (pending_req->sc_data_direction != DMA_FROM_DEVICE) && - (pending_req->sc_data_direction != DMA_NONE)) { - pr_debug("invalid parameter data_dir = %d\n", - pending_req->sc_data_direction); - return -EINVAL; + pending_req = scsiback_get_pend_req(ring, v2p); + if (IS_ERR(pending_req)) { + kref_put(&v2p->kref, scsiback_free_translation_entry); + return ERR_PTR(-ENOMEM); } - + pending_req->rqid = ring_req->rqid; + pending_req->info = info; + pending_req->v2p = v2p; + pending_req->sc_data_direction = ring_req->sc_data_direction; pending_req->cmd_len = ring_req->cmd_len; - if (pending_req->cmd_len > VSCSIIF_MAX_COMMAND_SIZE) { - pr_debug("invalid parameter cmd_len = %d\n", - pending_req->cmd_len); - return -EINVAL; - } memcpy(pending_req->cmnd, ring_req->cmnd, pending_req->cmd_len); - return 0; + return pending_req; } static int scsiback_do_cmd_fn(struct vscsibk_info *info) @@ -704,7 +736,7 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) struct vscsiif_request ring_req; struct vscsibk_pend *pending_req; RING_IDX rc, rp; - int err, more_to_do; + int more_to_do; uint32_t result; rc = ring->req_cons; @@ -722,16 +754,13 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) while ((rc != rp)) { if (RING_REQUEST_CONS_OVERFLOW(ring, rc)) break; - pending_req = kmem_cache_alloc(scsiback_cachep, GFP_KERNEL); - if (!pending_req) - return 1; RING_COPY_REQUEST(ring, rc, &ring_req); ring->req_cons = ++rc; - err = prepare_pending_reqs(info, &ring_req, pending_req); - if (err) { - switch (err) { + pending_req = prepare_pending_reqs(info, ring, &ring_req); + if (IS_ERR(pending_req)) { + switch (PTR_ERR(pending_req)) { case -ENODEV: result = DID_NO_CONNECT; break; @@ -739,9 +768,8 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) result = DRIVER_ERROR; break; } - scsiback_do_resp_with_sense(NULL, result << 24, 0, - pending_req); - kmem_cache_free(scsiback_cachep, pending_req); + scsiback_send_response(info, NULL, result << 24, 0, + ring_req.rqid); return 1; } @@ -750,8 +778,8 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) if (scsiback_gnttab_data_map(&ring_req, pending_req)) { scsiback_fast_flush_area(pending_req); scsiback_do_resp_with_sense(NULL, - DRIVER_ERROR << 24, 0, pending_req); - kmem_cache_free(scsiback_cachep, pending_req); + DRIVER_ERROR << 24, 0, pending_req); + transport_generic_free_cmd(&pending_req->se_cmd, 0); } else { scsiback_cmd_exec(pending_req); } @@ -765,9 +793,9 @@ static int scsiback_do_cmd_fn(struct vscsibk_info *info) break; default: pr_err_ratelimited("invalid request\n"); - scsiback_do_resp_with_sense(NULL, DRIVER_ERROR << 24, - 0, pending_req); - kmem_cache_free(scsiback_cachep, pending_req); + scsiback_do_resp_with_sense(NULL, DRIVER_ERROR << 24, 0, + pending_req); + transport_generic_free_cmd(&pending_req->se_cmd, 0); break; } @@ -1341,24 +1369,20 @@ static u32 scsiback_tpg_get_inst_index(struct se_portal_group *se_tpg) static int scsiback_check_stop_free(struct se_cmd *se_cmd) { - /* - * Do not release struct se_cmd's containing a valid TMR pointer. - * These will be released directly in scsiback_device_action() - * with transport_generic_free_cmd(). - */ - if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) - return 0; - - transport_generic_free_cmd(se_cmd, 0); - return 1; + return transport_generic_free_cmd(se_cmd, 0); } static void scsiback_release_cmd(struct se_cmd *se_cmd) { - struct vscsibk_pend *pending_req = container_of(se_cmd, - struct vscsibk_pend, se_cmd); + struct se_session *se_sess = se_cmd->se_sess; + struct se_tmr_req *se_tmr = se_cmd->se_tmr_req; + + if (se_tmr && se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) { + struct scsiback_tmr *tmr = se_tmr->fabric_tmr_ptr; + kfree(tmr); + } - kmem_cache_free(scsiback_cachep, pending_req); + percpu_ida_free(&se_sess->sess_tag_pool, se_cmd->map_tag); } static int scsiback_shutdown_session(struct se_session *se_sess) @@ -1485,58 +1509,36 @@ static struct configfs_attribute *scsiback_param_attrs[] = { static int scsiback_make_nexus(struct scsiback_tpg *tpg, const char *name) { - struct se_portal_group *se_tpg; - struct se_session *se_sess; struct scsiback_nexus *tv_nexus; + int ret = 0; mutex_lock(&tpg->tv_tpg_mutex); if (tpg->tpg_nexus) { - mutex_unlock(&tpg->tv_tpg_mutex); pr_debug("tpg->tpg_nexus already exists\n"); - return -EEXIST; + ret = -EEXIST; + goto out_unlock; } - se_tpg = &tpg->se_tpg; tv_nexus = kzalloc(sizeof(struct scsiback_nexus), GFP_KERNEL); if (!tv_nexus) { - mutex_unlock(&tpg->tv_tpg_mutex); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } - /* - * Initialize the struct se_session pointer - */ - tv_nexus->tvn_se_sess = transport_init_session(TARGET_PROT_NORMAL); + + tv_nexus->tvn_se_sess = target_alloc_session(&tpg->se_tpg, 128, + sizeof(struct vscsibk_pend), + TARGET_PROT_NORMAL, name, + tv_nexus, NULL); if (IS_ERR(tv_nexus->tvn_se_sess)) { - mutex_unlock(&tpg->tv_tpg_mutex); kfree(tv_nexus); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } - se_sess = tv_nexus->tvn_se_sess; - /* - * Since we are running in 'demo mode' this call with generate a - * struct se_node_acl for the scsiback struct se_portal_group with - * the SCSI Initiator port name of the passed configfs group 'name'. - */ - tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl( - se_tpg, (unsigned char *)name); - if (!tv_nexus->tvn_se_sess->se_node_acl) { - mutex_unlock(&tpg->tv_tpg_mutex); - pr_debug("core_tpg_check_initiator_node_acl() failed for %s\n", - name); - goto out; - } - /* Now register the TCM pvscsi virtual I_T Nexus as active. */ - transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl, - tv_nexus->tvn_se_sess, tv_nexus); - tpg->tpg_nexus = tv_nexus; + tpg->tpg_nexus = tv_nexus; +out_unlock: mutex_unlock(&tpg->tv_tpg_mutex); - return 0; - -out: - transport_free_session(se_sess); - kfree(tv_nexus); - return -ENOMEM; + return ret; } static int scsiback_drop_nexus(struct scsiback_tpg *tpg) @@ -1854,16 +1856,6 @@ static struct xenbus_driver scsiback_driver = { .otherend_changed = scsiback_frontend_changed }; -static void scsiback_init_pend(void *p) -{ - struct vscsibk_pend *pend = p; - int i; - - memset(pend, 0, sizeof(*pend)); - for (i = 0; i < VSCSI_MAX_GRANTS; i++) - pend->grant_handles[i] = SCSIBACK_INVALID_HANDLE; -} - static int __init scsiback_init(void) { int ret; @@ -1874,14 +1866,9 @@ static int __init scsiback_init(void) pr_debug("xen-pvscsi: fabric module %s on %s/%s on "UTS_RELEASE"\n", VSCSI_VERSION, utsname()->sysname, utsname()->machine); - scsiback_cachep = kmem_cache_create("vscsiif_cache", - sizeof(struct vscsibk_pend), 0, 0, scsiback_init_pend); - if (!scsiback_cachep) - return -ENOMEM; - ret = xenbus_register_backend(&scsiback_driver); if (ret) - goto out_cache_destroy; + goto out; ret = target_register_template(&scsiback_ops); if (ret) @@ -1891,8 +1878,7 @@ static int __init scsiback_init(void) out_unregister_xenbus: xenbus_unregister_driver(&scsiback_driver); -out_cache_destroy: - kmem_cache_destroy(scsiback_cachep); +out: pr_err("%s: error %d\n", __func__, ret); return ret; } @@ -1908,7 +1894,6 @@ static void __exit scsiback_exit(void) } target_unregister_template(&scsiback_ops); xenbus_unregister_driver(&scsiback_driver); - kmem_cache_destroy(scsiback_cachep); } module_init(scsiback_init); diff --git a/include/scsi/libsrp.h b/include/scsi/libsrp.h new file mode 100644 index 00000000000000..38ec0fe0cc87c0 --- /dev/null +++ b/include/scsi/libsrp.h @@ -0,0 +1,89 @@ +#ifndef __LIBSRP_H__ +#define __LIBSRP_H__ + +#include +#include +#include +#include +#include +#include + +enum srp_task_attributes { + SRP_SIMPLE_TASK = 0, + SRP_HEAD_TASK = 1, + SRP_ORDERED_TASK = 2, + SRP_ACA_TASK = 4 +}; + +enum iue_flags { + V_DIOVER, + V_WRITE, + V_LINKED, + V_FLYING, +}; + +struct srp_buf { + dma_addr_t dma; + void *buf; +}; + +struct srp_queue { + void *pool; + void *items; + struct kfifo queue; + spinlock_t lock; +}; + +struct srp_target { + struct Scsi_Host *shost; + struct se_device *tgt; + struct device *dev; + + spinlock_t lock; + struct list_head cmd_queue; + + size_t srp_iu_size; + struct srp_queue iu_queue; + size_t rx_ring_size; + struct srp_buf **rx_ring; + + void *ldata; +}; + +struct iu_entry { + struct srp_target *target; + + struct list_head ilist; + dma_addr_t remote_token; + unsigned long flags; + + struct srp_buf *sbuf; +}; + +typedef int (srp_rdma_t)(struct scsi_cmnd *, struct scatterlist *, int, + struct srp_direct_buf *, int, + enum dma_data_direction, unsigned int); +extern int srp_target_alloc(struct srp_target *, struct device *, + size_t, size_t); +extern void srp_target_free(struct srp_target *); + +extern struct iu_entry *srp_iu_get(struct srp_target *); +extern void srp_iu_put(struct iu_entry *); + +extern int srp_transfer_data(struct scsi_cmnd *, struct srp_cmd *, + srp_rdma_t, int, int); + + +static inline struct srp_target *host_to_srp_target(struct Scsi_Host *host) +{ + return (struct srp_target *) host->hostdata; +} + +static inline int srp_cmd_direction(struct srp_cmd *cmd) +{ + return (cmd->buf_fmt >> 4) ? DMA_TO_DEVICE : DMA_FROM_DEVICE; +} + +extern u64 srp_data_length(struct srp_cmd *cmd, enum dma_data_direction dir); + +#endif diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index 9fc1aecfc81369..be83b7a734421b 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -44,7 +44,7 @@ struct scsi_pointer { struct scatterlist *buffer; /* which buffer */ int buffers_residual; /* how many buffers left */ - dma_addr_t dma_handle; + dma_addr_t dma_handle; volatile int Status; volatile int Message; @@ -99,32 +99,38 @@ struct scsi_cmnd { struct scsi_data_buffer *prot_sdb; unsigned underflow; /* Return error if less than - this amount is transferred */ + * this amount is transferred + */ unsigned transfersize; /* How much we are guaranteed to - transfer with each SCSI transfer - (ie, between disconnect / - reconnects. Probably == sector - size */ + * transfer with each SCSI transfer + * (ie, between disconnect / + * reconnects. Probably == sector + * size + */ - struct request *request; /* The command we are - working on */ + /* The command we are working on*/ + struct request *request; -#define SCSI_SENSE_BUFFERSIZE 96 +#define SCSI_SENSE_BUFFERSIZE 96 unsigned char *sense_buffer; /* obtained by REQUEST SENSE when * CHECK CONDITION is received on original - * command (auto-sense) */ + * command (auto-sense) + */ /* Low-level done function - can be used by low-level driver to point - * to completion function. Not used by mid/upper level code. */ - void (*scsi_done) (struct scsi_cmnd *); + * to completion function. Not used by mid/upper level code. + */ + void (*scsi_done)(struct scsi_cmnd *); /* - * The following fields can be written to by the host specific code. - * Everything else should be left alone. + * The following fields can be written to by the host specific code. + * Everything else should be left alone. */ - struct scsi_pointer SCp; /* Scratchpad used by some host adapters */ + struct scsi_pointer SCp; /* Scratchpad used by some + * host adapters + */ unsigned char *host_scribble; /* The host adapter is allowed to * call scsi_malloc and get some memory @@ -132,7 +138,8 @@ struct scsi_cmnd { * is also expected to call scsi_free * to release this memory. (The memory * obtained by scsi_malloc is guaranteed - * to be at an address < 16Mb). */ + * to be at an address < 16Mb). + */ int result; /* Status code from lower level driver */ int flags; /* Command flags */ @@ -164,6 +171,7 @@ extern void *scsi_kmap_atomic_sg(struct scatterlist *sg, int sg_count, extern void scsi_kunmap_atomic_sg(void *virt); extern int scsi_init_io(struct scsi_cmnd *cmd); +extern void scsi_release_buffers(struct scsi_cmnd *cmd); extern int scsi_dma_map(struct scsi_cmnd *cmd); extern void scsi_dma_unmap(struct scsi_cmnd *cmd); @@ -279,7 +287,8 @@ enum scsi_prot_target_type { SCSI_PROT_DIF_TYPE3, }; -static inline void scsi_set_prot_type(struct scsi_cmnd *scmd, unsigned char type) +static inline void scsi_set_prot_type(struct scsi_cmnd *scmd, + unsigned char type) { scmd->prot_type = type; } diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index fcfa3d7f5e7e38..5db0807e877f53 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -83,7 +83,7 @@ struct scsi_host_template { #ifdef CONFIG_COMPAT - /* + /* * Compat handler. Handle 32bit ABI. * When unknown ioctl is passed return -ENOIOCTLCMD. * @@ -126,6 +126,27 @@ struct scsi_host_template { */ int (* queuecommand)(struct Scsi_Host *, struct scsi_cmnd *); + /* + * The transfer functions are used to queue a scsi command to + * the LLD. When the driver is finished processing the command + * the done callback is invoked. + * + * This is called to inform the LLD to transfer + * scsi_bufflen(cmd) bytes. scsi_sg_count(cmd) speciefies the + * number of scatterlist entried in the command and + * scsi_sglist(cmd) returns the scatterlist. + * + * return values: see queuecommand + * + * If the LLD accepts the cmd, it should set the result to an + * appropriate value when completed before calling the done function. + * + * STATUS: REQUIRED FOR TARGET DRIVERS + */ + /* TODO: rename */ + int (* transfer_response)(struct scsi_cmnd *, + void (*done)(struct scsi_cmnd *)); + /* * This is an error handling strategy routine. You don't need to * define one of these if you don't want to - there is a default @@ -185,7 +206,7 @@ struct scsi_host_template { * this function, it *must* perform the task of setting the queue * depth on the device. All other tasks are optional and depend * on what the driver supports and various implementation details. - * + * * Things currently recommended to be handled at this time include: * * 1. Setting the device queue depth. Proper setting of this is @@ -214,7 +235,7 @@ struct scsi_host_template { * has ceased the mid layer calls this point so that the low level * driver may completely detach itself from the scsi device and vice * versa. The low level driver is responsible for freeing any memory - * it allocated in the slave_alloc or slave_configure calls. + * it allocated in the slave_alloc or slave_configure calls. * * Status: OPTIONAL */ @@ -456,7 +477,7 @@ struct scsi_host_template { /* * Default value for the blocking. If the queue is empty, * host_blocked counts down in the request_fn until it restarts - * host operations as zero is reached. + * host operations as zero is reached. * * FIXME: This should probably be a value in the template */ @@ -545,7 +566,7 @@ struct Scsi_Host { */ struct list_head __devices; struct list_head __targets; - + struct scsi_host_cmd_pool *cmd_pool; spinlock_t free_list_lock; struct list_head free_list; /* backup store of cmd structs */ @@ -579,7 +600,7 @@ struct Scsi_Host { unsigned int host_failed; /* commands that failed. protected by host_lock */ unsigned int host_eh_scheduled; /* EH scheduled without command */ - + unsigned int host_no; /* Used for IOCTL_GET_IDLUN, /proc/scsi et al. */ /* next two fields are used to bound the time spent in error handling */ @@ -630,12 +651,12 @@ struct Scsi_Host { * is nr_hw_queues * can_queue. */ unsigned nr_hw_queues; - /* + /* * Used to assign serial numbers to the cmds. * Protected by the host lock. */ unsigned long cmd_serial_number; - + unsigned active_mode:2; unsigned unchecked_isa_dma:1; unsigned use_clustering:1; @@ -645,7 +666,7 @@ struct Scsi_Host { * time being. */ unsigned host_self_blocked:1; - + /* * Host uses correct SCSI ordering not PC ordering. The bit is * set for the minority of drivers whose authors actually read @@ -706,7 +727,7 @@ struct Scsi_Host { unsigned char n_io_port; unsigned char dma_channel; unsigned int irq; - + enum scsi_host_state shost_state; diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h index e8c8c08bf575f4..fca03b993bec08 100644 --- a/include/target/target_core_base.h +++ b/include/target/target_core_base.h @@ -144,12 +144,6 @@ enum se_cmd_flags_table { SCF_USE_CPUID = 0x00800000, }; -/* struct se_dev_entry->lun_flags and struct se_lun->lun_access */ -enum transport_lunflags_table { - TRANSPORT_LUNFLAGS_READ_ONLY = 0x01, - TRANSPORT_LUNFLAGS_READ_WRITE = 0x02, -}; - /* * Used by transport_send_check_condition_and_sense() * to signal which ASC/ASCQ sense payload should be built. @@ -634,11 +628,10 @@ struct se_lun_acl { }; struct se_dev_entry { - /* See transport_lunflags_table */ u64 mapped_lun; u64 pr_res_key; u64 creation_time; - u32 lun_flags; + bool lun_access_ro; u32 attach_count; atomic_long_t total_cmds; atomic_long_t read_bytes; @@ -712,7 +705,7 @@ struct se_lun { u64 unpacked_lun; #define SE_LUN_LINK_MAGIC 0xffff7771 u32 lun_link_magic; - u32 lun_access; + bool lun_access_ro; u32 lun_index; /* RELATIVE TARGET PORT IDENTIFER */ diff --git a/include/uapi/linux/target_core_user.h b/include/uapi/linux/target_core_user.h index 95c6521d8a95ff..c506cddb8165cf 100644 --- a/include/uapi/linux/target_core_user.h +++ b/include/uapi/linux/target_core_user.h @@ -41,6 +41,7 @@ #define TCMU_MAILBOX_VERSION 2 #define ALIGN_SIZE 64 /* Should be enough for most CPUs */ +#define TCMU_MAILBOX_FLAG_CAP_OOOC (1 << 0) /* Out-of-order completions */ struct tcmu_mailbox { __u16 version; diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 48958d3cec9e38..e080746e1a6b52 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -594,8 +594,7 @@ static int ignore_undef_symbol(struct elf_info *info, const char *symname) if (strncmp(symname, "_restgpr0_", sizeof("_restgpr0_") - 1) == 0 || strncmp(symname, "_savegpr0_", sizeof("_savegpr0_") - 1) == 0 || strncmp(symname, "_restvr_", sizeof("_restvr_") - 1) == 0 || - strncmp(symname, "_savevr_", sizeof("_savevr_") - 1) == 0 || - strcmp(symname, ".TOC.") == 0) + strncmp(symname, "_savevr_", sizeof("_savevr_") - 1) == 0) return 1; /* Do not ignore this symbol */ return 0;