Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/dev/iscsi Remove throttling code, instead signal scsipi ...
details: https://anonhg.NetBSD.org/src/rev/0db5ce0111b3
branches: trunk
changeset: 345907:0db5ce0111b3
user: mlelstv <mlelstv%NetBSD.org@localhost>
date: Wed Jun 15 04:30:30 2016 +0000
description:
Remove throttling code, instead signal scsipi layer to reduce the openings
and retry the command. Start with a small openings number and let scsipi
request to grow it up to the current send window.
Adjust ccb and pdu counts to avoid ressource shortages. These are still
very ad-hoc numbers, but seem to be sufficient for a Gigabit link.
Use separate condvar for PDU pool and add counter to help debugging.
Revert setting PDU disposition to UNUSED before freeing. free_pdu
uses this as a flag to detect already returned PDUs.
Add reference counter for open commands to defer unmapping a session
that would lead to crashes in scsipi.
Move session cleanup to cleanup thread.
Use get_sernum to retrieve current serial number where possible and
make it check for immediate commands itself.
Adjust debug output.
diffstat:
sys/dev/iscsi/iscsi.h | 3 +-
sys/dev/iscsi/iscsi_globals.h | 49 ++---
sys/dev/iscsi/iscsi_ioctl.c | 323 +++++++++++++++++++++++++----------------
sys/dev/iscsi/iscsi_main.c | 78 ++++++++-
sys/dev/iscsi/iscsi_rcv.c | 88 ++--------
sys/dev/iscsi/iscsi_send.c | 91 ++++++-----
sys/dev/iscsi/iscsi_utils.c | 82 ++++-----
7 files changed, 394 insertions(+), 320 deletions(-)
diffs (truncated from 1464 to 300 lines):
diff -r 1d6c3d49c022 -r 0db5ce0111b3 sys/dev/iscsi/iscsi.h
--- a/sys/dev/iscsi/iscsi.h Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi.h Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: iscsi.h,v 1.3 2011/11/19 16:41:56 agc Exp $ */
+/* $NetBSD: iscsi.h,v 1.4 2016/06/15 04:30:30 mlelstv Exp $ */
/*-
* Copyright (c) 2004,2006,2011 The NetBSD Foundation, Inc.
@@ -182,6 +182,7 @@
#define ISCSI_STATUS_LOGOUT_CID_NOT_FOUND 45 /* Logout error: CID not found */
#define ISCSI_STATUS_LOGOUT_RECOVERY_NS 46 /* Logout error: Recovery not supported */
#define ISCSI_STATUS_LOGOUT_ERROR 47 /* Logout error: Unknown reason */
+#define ISCSI_STATUS_QUEUE_FULL 48 /* iSCSI send window exhausted */
#define ISCSID_STATUS_SUCCESS 0 /* Indicates success. */
#define ISCSID_STATUS_LIST_EMPTY 1001 /* The requested list is empty. */
diff -r 1d6c3d49c022 -r 0db5ce0111b3 sys/dev/iscsi/iscsi_globals.h
--- a/sys/dev/iscsi/iscsi_globals.h Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi_globals.h Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: iscsi_globals.h,v 1.20 2016/06/15 03:51:55 mlelstv Exp $ */
+/* $NetBSD: iscsi_globals.h,v 1.21 2016/06/15 04:30:30 mlelstv Exp $ */
/*-
* Copyright (c) 2004,2005,2006,2011 The NetBSD Foundation, Inc.
@@ -71,33 +71,25 @@
#define VERSION_STRING "NetBSD iSCSI Software Initiator 20110407"
/*
-Various checks are made that the expected cmd Serial Number is less than
-the actual command serial number. The extremely paranoid amongst us
-believe that a malicious iSCSI server could set this artificially low
-and effectively DoS a naive initiator. For this (possibly ludicrous)
-reason, I have added the two definitions below (agc, 2011/04/09). The
-throttling definition enables a check that the CmdSN is less than the
-ExpCmdSN in iscsi_send.c, and is enabled by default. The second definition
-effectively says "don't bother testing these values", and is used right
-now only in iscsi_send.c.
- */
-#define ISCSI_THROTTLING_ENABLED 1
-#define ISCSI_SERVER_TRUSTED 0
-
-/*
NOTE: CCBS_PER_SESSION must not exceed 256 due to the way the ITT
is constructed (it has the CCB index in its lower 8 bits). If it should ever
be necessary to increase the number beyond that (which isn't expected),
the corresponding ITT generation and extraction code must be rewritten.
*/
-#define CCBS_PER_SESSION 64 /* ToDo: Reasonable number?? */
+#define CCBS_PER_SESSION 32 /* ToDo: Reasonable number?? */
+/*
+ NOTE: CCBS_FOR_SCSPI limits the number of outstanding commands for
+ SCSI commands, leaving some CCBs for keepalive and logout attempts,
+ which are needed for each connection.
+*/
+#define CCBS_FOR_SCSIPI 16 /* ToDo: Reasonable number?? */
/*
NOTE: PDUS_PER_CONNECTION is a number that could potentially impact
performance if set too low, as a single command may use up a lot of PDUs for
high values of First/MaxBurstLength and small values of
MaxRecvDataSegmentLength of the target.
*/
-#define PDUS_PER_CONNECTION 128 /* ToDo: Reasonable number?? */
+#define PDUS_PER_CONNECTION 64 /* ToDo: Reasonable number?? */
/* max outstanding serial nums before we give up on the connection */
#define SERNUM_BUFFER_LENGTH (CCBS_PER_SESSION / 2) /* ToDo: Reasonable?? */
@@ -106,7 +98,7 @@
#define DEFAULT_MaxRecvDataSegmentLength (64*1024)
/* Command timeout (reset on received PDU associated with the command's CCB) */
-#define COMMAND_TIMEOUT (7 * hz) /* ToDo: Reasonable? (7 seconds) */
+#define COMMAND_TIMEOUT (60 * hz) /* ToDo: Reasonable? (60 seconds) */
#define MAX_CCB_TIMEOUTS 3 /* Max number of tries to resend or SNACK */
#define MAX_CCB_TRIES 9 /* Max number of total tries to recover */
@@ -131,12 +123,10 @@
#define CCBF_COMPLETE 0x0001 /* received status */
#define CCBF_RESENT 0x0002 /* ccb was resent */
#define CCBF_SENDTARGET 0x0004 /* SendTargets text request, not negotiation */
-#define CCBF_WAITING 0x0008 /* CCB is waiting for MaxCmdSN, wake it up */
#define CCBF_GOT_RSP 0x0010 /* Got at least one response to this request */
#define CCBF_REASSIGN 0x0020 /* Command can be reassigned */
#define CCBF_OTHERCONN 0x0040 /* a logout for a different connection */
#define CCBF_WAITQUEUE 0x0080 /* CCB is on waiting queue */
-#define CCBF_THROTTLING 0x0100 /* CCB is on throttling queue */
/* --------------------------- Global Types ------------------------------- */
@@ -322,6 +312,7 @@
kmutex_t lock;
kcondvar_t conn_cv;
+ kcondvar_t pdu_cv;
kcondvar_t ccb_cv;
kcondvar_t idle_cv;
@@ -375,6 +366,7 @@
int recover; /* recovery count */
/* (reset on first successful data transfer) */
volatile unsigned usecount; /* number of active CCBs */
+ volatile unsigned pducount; /* number of active PDUs */
bool destroy; /* conn will be destroyed */
bool in_session;
@@ -417,6 +409,8 @@
device_t child_dev;
/* the child we're associated with - (NULL if not mapped) */
+ int refcount; /* session in use by scsipi */
+
/* local stuff */
TAILQ_ENTRY(session_s) sessions; /* the list of sessions */
@@ -425,8 +419,8 @@
kcondvar_t ccb_cv;
ccb_list_t ccb_pool; /* The free CCB pool */
- ccb_list_t ccbs_throttled;
- /* CCBs waiting for MaxCmdSN to increase */
+
+ int send_window;
uint16_t id; /* session ID (unique within driver) */
uint16_t TSIH; /* Target assigned session ID */
@@ -638,7 +632,7 @@
/* in iscsi_ioctl.c */
void iscsi_init_cleanup(void);
-void iscsi_destroy_cleanup(void);
+int iscsi_destroy_cleanup(void);
void iscsi_notify_cleanup(void);
@@ -652,7 +646,7 @@
void kill_connection(connection_t *, uint32_t, int, bool);
void kill_session(session_t *, uint32_t, int, bool);
-void kill_all_sessions(void);
+int kill_all_sessions(void);
void handle_connection_error(connection_t *, uint32_t, int);
void add_connection_cleanup(connection_t *);
@@ -664,7 +658,8 @@
session_t *find_session(uint32_t);
connection_t *find_connection(session_t *, uint32_t);
-
+int ref_session(session_t *);
+void unref_session(session_t *);
/* in iscsi_main.c */
@@ -725,7 +720,6 @@
ccb_t *get_ccb(connection_t *, bool);
void free_ccb(ccb_t *);
void suspend_ccb(ccb_t *, bool);
-void throttle_ccb(ccb_t *, bool);
void wake_ccb(ccb_t *, uint32_t);
void create_pdus(connection_t *);
@@ -736,8 +730,9 @@
int add_sernum(sernum_buffer_t *, uint32_t);
uint32_t ack_sernum(sernum_buffer_t *, uint32_t);
-uint32_t get_sernum(session_t *, bool);
+uint32_t get_sernum(session_t *, pdu_t *);
int sernum_in_window(session_t *);
+int window_size(session_t *, int);
/* in iscsi_text.c */
diff -r 1d6c3d49c022 -r 0db5ce0111b3 sys/dev/iscsi/iscsi_ioctl.c
--- a/sys/dev/iscsi/iscsi_ioctl.c Wed Jun 15 03:51:55 2016 +0000
+++ b/sys/dev/iscsi/iscsi_ioctl.c Wed Jun 15 04:30:30 2016 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: iscsi_ioctl.c,v 1.21 2016/06/05 15:04:31 mlelstv Exp $ */
+/* $NetBSD: iscsi_ioctl.c,v 1.22 2016/06/15 04:30:30 mlelstv Exp $ */
/*-
* Copyright (c) 2004,2005,2006,2011 The NetBSD Foundation, Inc.
@@ -430,6 +430,50 @@
return curr;
}
+/*
+ * ref_session:
+ * Reference a session
+ *
+ * Session cannot be release until reference count reaches zero
+ *
+ * Returns: 1 if reference counter would overflow
+ */
+
+int
+ref_session(session_t *session)
+{
+ int rc = 1;
+
+ mutex_enter(&iscsi_cleanup_mtx);
+ KASSERT(session != NULL);
+ if (session->refcount <= CCBS_PER_SESSION) {
+ session->refcount++;
+ rc = 0;
+ }
+ mutex_exit(&iscsi_cleanup_mtx);
+
+ return rc;
+}
+
+/*
+ * unref_session:
+ * Unreference a session
+ *
+ * Release session reference, trigger cleanup
+ */
+
+void
+unref_session(session_t *session)
+{
+
+ mutex_enter(&iscsi_cleanup_mtx);
+ KASSERT(session != NULL);
+ KASSERT(session->refcount > 0);
+ if (--session->refcount == 0)
+ cv_broadcast(&session->sess_cv);
+ mutex_exit(&iscsi_cleanup_mtx);
+}
+
/*
* kill_connection:
@@ -543,8 +587,7 @@
void
kill_session(session_t *session, uint32_t status, int logout, bool recover)
{
- connection_t *curr;
- ccb_t *ccb;
+ connection_t *conn;
DEB(1, ("ISCSI: kill_session %d, status %d, logout %d, recover %d\n",
session->id, status, logout, recover));
@@ -552,6 +595,7 @@
mutex_enter(&iscsi_cleanup_mtx);
if (session->terminating) {
mutex_exit(&iscsi_cleanup_mtx);
+
DEB(5, ("Session is being killed with status %d\n",session->terminating));
return;
}
@@ -560,15 +604,16 @@
* don't do anything if session isn't established yet, termination will be
* handled elsewhere
*/
- if (session->sessions.tqe_next == NULL &&
- session->sessions.tqe_prev == NULL) {
+ if (session->sessions.tqe_next == NULL && session->sessions.tqe_prev == NULL) {
mutex_exit(&iscsi_cleanup_mtx);
+
+ DEB(5, ("Session is being killed which is not yet established\n"));
return;
}
- session->terminating = status;
- mutex_exit(&iscsi_cleanup_mtx);
if (recover) {
+ mutex_exit(&iscsi_cleanup_mtx);
+
/*
* Only recover when there's just one active connection left.
* Otherwise we get in all sorts of timing problems, and it doesn't
@@ -576,41 +621,32 @@
* requested that we kill a multipathed session.
*/
if (session->active_connections == 1) {
- curr = assign_connection(session, FALSE);
- if (curr != NULL)
- kill_connection(curr, status, logout, TRUE);
+ conn = assign_connection(session, FALSE);
+ if (conn != NULL)
+ kill_connection(conn, status, logout, TRUE);
}
- /* don't allow the session to disappear when the target */
- /* requested the logout */
- session->terminating = ISCSI_STATUS_SUCCESS;
return;
}
- /* remove from session list */
- mutex_enter(&iscsi_cleanup_mtx);
+ if (session->refcount > 0) {
+ mutex_exit(&iscsi_cleanup_mtx);
+
+ DEB(5, ("Session is being killed while in use (refcnt = %d)\n",
+ session->refcount));
+ return;
+ }
+
+ /* Remove session from global list */
Home |
Main Index |
Thread Index |
Old Index