Source-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[src/trunk]: src/sys/dev/iscsi Remove throttling code, instead signal scsipi ...



details:   https://anonhg.NetBSD.org/src/rev/36b7f02f20df
branches:  trunk
changeset: 816049:36b7f02f20df
user:      mlelstv <mlelstv%NetBSD.org@localhost>
date:      Wed Jun 15 04:30:30 2016 +0000

description:
Remove throttling code, instead signal scsipi layer to reduce the openings
and retry the command. Start with a small openings number and let scsipi
request to grow it up to the current send window.

Adjust ccb and pdu counts to avoid ressource shortages. These are still
very ad-hoc numbers, but seem to be sufficient for a Gigabit link.

Use separate condvar for PDU pool and add counter to help debugging.

Revert setting PDU disposition to UNUSED before freeing. free_pdu
uses this as a flag to detect already returned PDUs.

Add reference counter for open commands to defer unmapping a session
that would lead to crashes in scsipi.

Move session cleanup to cleanup thread.

Use get_sernum to retrieve current serial number where possible and
make it check for immediate commands itself.

Adjust debug output.

diffstat:

 sys/dev/iscsi/iscsi.h         |    3 +-
 sys/dev/iscsi/iscsi_globals.h |   49 ++---
 sys/dev/iscsi/iscsi_ioctl.c   |  323 +++++++++++++++++++++++++----------------
 sys/dev/iscsi/iscsi_main.c    |   78 ++++++++-
 sys/dev/iscsi/iscsi_rcv.c     |   88 ++--------
 sys/dev/iscsi/iscsi_send.c    |   91 ++++++-----
 sys/dev/iscsi/iscsi_utils.c   |   82 ++++-----
 7 files changed, 394 insertions(+), 320 deletions(-)

diffs (truncated from 1464 to 300 lines):

diff -r 98184908a538 -r 36b7f02f20df sys/dev/iscsi/iscsi.h
--- a/sys/dev/iscsi/iscsi.h     Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi.h     Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: iscsi.h,v 1.3 2011/11/19 16:41:56 agc Exp $    */
+/*     $NetBSD: iscsi.h,v 1.4 2016/06/15 04:30:30 mlelstv Exp $        */
 
 /*-
  * Copyright (c) 2004,2006,2011 The NetBSD Foundation, Inc.
@@ -182,6 +182,7 @@
 #define ISCSI_STATUS_LOGOUT_CID_NOT_FOUND    45        /* Logout error: CID not found */
 #define ISCSI_STATUS_LOGOUT_RECOVERY_NS             46 /* Logout error: Recovery not supported */
 #define ISCSI_STATUS_LOGOUT_ERROR           47 /* Logout error: Unknown reason */
+#define ISCSI_STATUS_QUEUE_FULL              48 /* iSCSI send window exhausted */
 
 #define ISCSID_STATUS_SUCCESS                0         /* Indicates success. */
 #define ISCSID_STATUS_LIST_EMPTY             1001      /* The requested list is empty. */
diff -r 98184908a538 -r 36b7f02f20df sys/dev/iscsi/iscsi_globals.h
--- a/sys/dev/iscsi/iscsi_globals.h     Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi_globals.h     Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: iscsi_globals.h,v 1.20 2016/06/15 03:51:55 mlelstv Exp $       */
+/*     $NetBSD: iscsi_globals.h,v 1.21 2016/06/15 04:30:30 mlelstv Exp $       */
 
 /*-
  * Copyright (c) 2004,2005,2006,2011 The NetBSD Foundation, Inc.
@@ -71,33 +71,25 @@
 #define VERSION_STRING         "NetBSD iSCSI Software Initiator 20110407"
 
 /*
-Various checks are made that the expected cmd Serial Number is less than
-the actual command serial number. The extremely paranoid amongst us
-believe that a malicious iSCSI server could set this artificially low
-and effectively DoS a naive initiator. For this (possibly ludicrous)
-reason, I have added the two definitions below (agc, 2011/04/09). The
-throttling definition enables a check that the CmdSN is less than the
-ExpCmdSN in iscsi_send.c, and is enabled by default. The second definition
-effectively says "don't bother testing these values", and is used right
-now only in iscsi_send.c.
- */
-#define ISCSI_THROTTLING_ENABLED       1
-#define ISCSI_SERVER_TRUSTED           0
-
-/*
    NOTE: CCBS_PER_SESSION must not exceed 256 due to the way the ITT
    is constructed (it has the CCB index in its lower 8 bits). If it should ever
    be necessary to increase the number beyond that (which isn't expected),
    the corresponding ITT generation and extraction code must be rewritten.
 */
-#define CCBS_PER_SESSION      64       /* ToDo: Reasonable number?? */
+#define CCBS_PER_SESSION      32       /* ToDo: Reasonable number?? */
+/*
+   NOTE: CCBS_FOR_SCSPI limits the number of outstanding commands for
+   SCSI commands, leaving some CCBs for keepalive and logout attempts,
+   which are needed for each connection.
+*/
+#define CCBS_FOR_SCSIPI       16       /* ToDo: Reasonable number?? */
 /*
    NOTE: PDUS_PER_CONNECTION is a number that could potentially impact
    performance if set too low, as a single command may use up a lot of PDUs for
    high values of First/MaxBurstLength and small values of
    MaxRecvDataSegmentLength of the target.
 */
-#define PDUS_PER_CONNECTION   128      /* ToDo: Reasonable number?? */
+#define PDUS_PER_CONNECTION   64       /* ToDo: Reasonable number?? */
 
 /* max outstanding serial nums before we give up on the connection */
 #define SERNUM_BUFFER_LENGTH  (CCBS_PER_SESSION / 2)   /* ToDo: Reasonable?? */
@@ -106,7 +98,7 @@
 #define DEFAULT_MaxRecvDataSegmentLength     (64*1024)
 
 /* Command timeout (reset on received PDU associated with the command's CCB) */
-#define COMMAND_TIMEOUT                (7 * hz) /* ToDo: Reasonable? (7 seconds) */
+#define COMMAND_TIMEOUT                (60 * hz) /* ToDo: Reasonable? (60 seconds) */
 #define MAX_CCB_TIMEOUTS       3               /* Max number of tries to resend or SNACK */
 #define MAX_CCB_TRIES          9       /* Max number of total tries to recover */
 
@@ -131,12 +123,10 @@
 #define CCBF_COMPLETE   0x0001 /* received status */
 #define CCBF_RESENT     0x0002 /* ccb was resent */
 #define CCBF_SENDTARGET 0x0004 /* SendTargets text request, not negotiation */
-#define CCBF_WAITING    0x0008 /* CCB is waiting for MaxCmdSN, wake it up */
 #define CCBF_GOT_RSP    0x0010 /* Got at least one response to this request */
 #define CCBF_REASSIGN   0x0020 /* Command can be reassigned */
 #define CCBF_OTHERCONN  0x0040 /* a logout for a different connection */
 #define CCBF_WAITQUEUE  0x0080 /* CCB is on waiting queue */
-#define CCBF_THROTTLING 0x0100 /* CCB is on throttling queue */
 
 /* ---------------------------  Global Types  ------------------------------- */
 
@@ -322,6 +312,7 @@
 
        kmutex_t                        lock;
        kcondvar_t                      conn_cv;
+       kcondvar_t                      pdu_cv;
        kcondvar_t                      ccb_cv;
        kcondvar_t                      idle_cv;
 
@@ -375,6 +366,7 @@
        int                             recover; /* recovery count */
                /* (reset on first successful data transfer) */
        volatile unsigned               usecount; /* number of active CCBs */
+       volatile unsigned               pducount; /* number of active PDUs */
 
        bool                            destroy; /* conn will be destroyed */
        bool                            in_session;
@@ -417,6 +409,8 @@
        device_t                child_dev;
        /* the child we're associated with - (NULL if not mapped) */
 
+       int                     refcount;       /* session in use by scsipi */
+
        /* local stuff */
        TAILQ_ENTRY(session_s)  sessions;       /* the list of sessions */
 
@@ -425,8 +419,8 @@
        kcondvar_t              ccb_cv;
 
        ccb_list_t              ccb_pool;       /* The free CCB pool */
-       ccb_list_t              ccbs_throttled;
-                               /* CCBs waiting for MaxCmdSN to increase */
+
+       int                     send_window;
 
        uint16_t                id;     /* session ID (unique within driver) */
        uint16_t                TSIH;   /* Target assigned session ID */
@@ -638,7 +632,7 @@
 /* in iscsi_ioctl.c */
 
 void iscsi_init_cleanup(void);
-void iscsi_destroy_cleanup(void);
+int iscsi_destroy_cleanup(void);
 void iscsi_notify_cleanup(void);
 
 
@@ -652,7 +646,7 @@
 
 void kill_connection(connection_t *, uint32_t, int, bool);
 void kill_session(session_t *, uint32_t, int, bool);
-void kill_all_sessions(void);
+int kill_all_sessions(void);
 void handle_connection_error(connection_t *, uint32_t, int);
 void add_connection_cleanup(connection_t *);
 
@@ -664,7 +658,8 @@
 
 session_t *find_session(uint32_t);
 connection_t *find_connection(session_t *, uint32_t);
-
+int ref_session(session_t *);
+void unref_session(session_t *);
 
 /* in iscsi_main.c */
 
@@ -725,7 +720,6 @@
 ccb_t *get_ccb(connection_t *, bool);
 void free_ccb(ccb_t *);
 void suspend_ccb(ccb_t *, bool);
-void throttle_ccb(ccb_t *, bool);
 void wake_ccb(ccb_t *, uint32_t);
 
 void create_pdus(connection_t *);
@@ -736,8 +730,9 @@
 int add_sernum(sernum_buffer_t *, uint32_t);
 uint32_t ack_sernum(sernum_buffer_t *, uint32_t);
 
-uint32_t get_sernum(session_t *, bool);
+uint32_t get_sernum(session_t *, pdu_t *);
 int sernum_in_window(session_t *);
+int window_size(session_t *, int);
 
 /* in iscsi_text.c */
 
diff -r 98184908a538 -r 36b7f02f20df sys/dev/iscsi/iscsi_ioctl.c
--- a/sys/dev/iscsi/iscsi_ioctl.c       Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi_ioctl.c       Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/*     $NetBSD: iscsi_ioctl.c,v 1.21 2016/06/05 15:04:31 mlelstv Exp $ */
+/*     $NetBSD: iscsi_ioctl.c,v 1.22 2016/06/15 04:30:30 mlelstv Exp $ */
 
 /*-
  * Copyright (c) 2004,2005,2006,2011 The NetBSD Foundation, Inc.
@@ -430,6 +430,50 @@
        return curr;
 }
 
+/*
+ * ref_session:
+ *    Reference a session
+ *
+ *    Session cannot be release until reference count reaches zero
+ *
+ *    Returns: 1 if reference counter would overflow
+ */
+
+int
+ref_session(session_t *session)
+{
+       int rc = 1;
+
+       mutex_enter(&iscsi_cleanup_mtx);
+       KASSERT(session != NULL);
+       if (session->refcount <= CCBS_PER_SESSION) {
+               session->refcount++;
+               rc = 0;
+       }
+       mutex_exit(&iscsi_cleanup_mtx);
+
+       return rc;
+}
+
+/*
+ * unref_session:
+ *    Unreference a session
+ *
+ *    Release session reference, trigger cleanup
+ */
+
+void
+unref_session(session_t *session)
+{
+
+       mutex_enter(&iscsi_cleanup_mtx);
+       KASSERT(session != NULL);
+       KASSERT(session->refcount > 0);
+       if (--session->refcount == 0)
+               cv_broadcast(&session->sess_cv);
+       mutex_exit(&iscsi_cleanup_mtx);
+}
+
 
 /*
  * kill_connection:
@@ -543,8 +587,7 @@
 void
 kill_session(session_t *session, uint32_t status, int logout, bool recover)
 {
-       connection_t *curr;
-       ccb_t *ccb;
+       connection_t *conn;
 
        DEB(1, ("ISCSI: kill_session %d, status %d, logout %d, recover %d\n",
                        session->id, status, logout, recover));
@@ -552,6 +595,7 @@
        mutex_enter(&iscsi_cleanup_mtx);
        if (session->terminating) {
                mutex_exit(&iscsi_cleanup_mtx);
+
                DEB(5, ("Session is being killed with status %d\n",session->terminating));
                return;
        }
@@ -560,15 +604,16 @@
         * don't do anything if session isn't established yet, termination will be
         * handled elsewhere
         */
-       if (session->sessions.tqe_next == NULL &&
-           session->sessions.tqe_prev == NULL) {
+       if (session->sessions.tqe_next == NULL && session->sessions.tqe_prev == NULL) {
                mutex_exit(&iscsi_cleanup_mtx);
+
+               DEB(5, ("Session is being killed which is not yet established\n"));
                return;
        }
-       session->terminating = status;
-       mutex_exit(&iscsi_cleanup_mtx);
 
        if (recover) {
+               mutex_exit(&iscsi_cleanup_mtx);
+
                /*
                 * Only recover when there's just one active connection left.
                 * Otherwise we get in all sorts of timing problems, and it doesn't
@@ -576,41 +621,32 @@
                 * requested that we kill a multipathed session.
                 */
                if (session->active_connections == 1) {
-                       curr = assign_connection(session, FALSE);
-                       if (curr != NULL)
-                               kill_connection(curr, status, logout, TRUE);
+                       conn = assign_connection(session, FALSE);
+                       if (conn != NULL)
+                               kill_connection(conn, status, logout, TRUE);
                }
-               /* don't allow the session to disappear when the target */
-               /* requested the logout */
-               session->terminating = ISCSI_STATUS_SUCCESS;
                return;
        }
 
-       /* remove from session list */
-       mutex_enter(&iscsi_cleanup_mtx);
+       if (session->refcount > 0) {
+               mutex_exit(&iscsi_cleanup_mtx);
+
+               DEB(5, ("Session is being killed while in use (refcnt = %d)\n",
+                       session->refcount));
+               return;
+       }
+
+       /* Remove session from global list */



Home | Main Index | Thread Index | Old Index