Bayard's initial drop, needs finishing, or at least testing.

@@ -19,11 +19,13 @@
  * CDDL HEADER END
  */
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright (c) 2012 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2016 by Delphix. All rights reserved.
+ * Copyright (c) 2017, Joyent, Inc.
  */
 
 /*
  * IPsec Security Policy Database.
  *

@@ -411,21 +413,21 @@
         for (i = 0; i < ipss->ipsec_spd_hashsize; i++) {
                 ASSERT(ipss->ipsec_sel_hash[i].hash_head == NULL);
                 mutex_destroy(&(ipss->ipsec_sel_hash[i].hash_lock));
         }
 
-        mutex_enter(&ipss->ipsec_alg_lock);
+        rw_enter(&ipss->ipsec_alg_lock, RW_WRITER);
         for (algtype = 0; algtype < IPSEC_NALGTYPES; algtype ++) {
                 int nalgs = ipss->ipsec_nalgs[algtype];
 
                 for (i = 0; i < nalgs; i++) {
                         if (ipss->ipsec_alglists[algtype][i] != NULL)
                                 ipsec_alg_unreg(algtype, i, ns);
                 }
         }
-        mutex_exit(&ipss->ipsec_alg_lock);
-        mutex_destroy(&ipss->ipsec_alg_lock);
+        rw_exit(&ipss->ipsec_alg_lock);
+        rw_destroy(&ipss->ipsec_alg_lock);
 
         ipsid_gc(ns);
         ipsid_fini(ns);
 
         (void) ipsec_free_tables(ipss);

@@ -653,11 +655,11 @@
 
         for (i = 0; i < ipss->ipsec_spd_hashsize; i++)
                 mutex_init(&(ipss->ipsec_sel_hash[i].hash_lock),
                     NULL, MUTEX_DEFAULT, NULL);
 
-        mutex_init(&ipss->ipsec_alg_lock, NULL, MUTEX_DEFAULT, NULL);
+        rw_init(&ipss->ipsec_alg_lock, NULL, RW_DEFAULT, NULL);
         for (i = 0; i < IPSEC_NALGTYPES; i++) {
                 ipss->ipsec_nalgs[i] = 0;
         }
 
         ip_drop_init(ipss);

@@ -716,11 +718,11 @@
         uint_t i;
         uint_t count = ipss->ipsec_nalgs[at];
         ASSERT(ai != NULL);
         ASSERT(algid == ai->alg_id);
 
-        ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
+        ASSERT(RW_WRITE_HELD(&ipss->ipsec_alg_lock));
 
         holder = algid;
 
         for (i = 0; i < count - 1; i++) {
                 ipsec_alginfo_t *alt;

@@ -753,11 +755,11 @@
         boolean_t copyback = B_FALSE;
         int i;
         ipsec_stack_t   *ipss = ns->netstack_ipsec;
         int newcount = ipss->ipsec_nalgs[at];
 
-        ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
+        ASSERT(RW_WRITE_HELD(&ipss->ipsec_alg_lock));
 
         for (i = 0; i <= newcount; i++) {
                 if (copyback) {
                         ipss->ipsec_sortlist[at][i-1] =
                             ipss->ipsec_sortlist[at][i];

@@ -774,11 +776,11 @@
 void
 ipsec_alg_reg(ipsec_algtype_t algtype, ipsec_alginfo_t *alg, netstack_t *ns)
 {
         ipsec_stack_t   *ipss = ns->netstack_ipsec;
 
-        ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
+        ASSERT(RW_WRITE_HELD(&ipss->ipsec_alg_lock));
 
         ASSERT(ipss->ipsec_alglists[algtype][alg->alg_id] == NULL);
         ipsec_alg_fix_min_max(alg, algtype, ns);
         ipss->ipsec_alglists[algtype][alg->alg_id] = alg;
 

@@ -793,11 +795,11 @@
 void
 ipsec_alg_unreg(ipsec_algtype_t algtype, uint8_t algid, netstack_t *ns)
 {
         ipsec_stack_t   *ipss = ns->netstack_ipsec;
 
-        ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
+        ASSERT(RW_WRITE_HELD(&ipss->ipsec_alg_lock));
 
         ASSERT(ipss->ipsec_alglists[algtype][algid] != NULL);
         ipsec_alg_free(ipss->ipsec_alglists[algtype][algid]);
         ipss->ipsec_alglists[algtype][algid] = NULL;
 

@@ -4685,12 +4687,13 @@
                 mutex_destroy(&bucket->ipsif_lock);
         }
 }
 
 /*
- * Update the minimum and maximum supported key sizes for the
- * specified algorithm. Must be called while holding the algorithms lock.
+ * Update the minimum and maximum supported key sizes for the specified
+ * algorithm, which is either a member of a netstack alg array or about to be,
+ * and therefore must be called holding ipsec_alg_lock for write.
  */
 void
 ipsec_alg_fix_min_max(ipsec_alginfo_t *alg, ipsec_algtype_t alg_type,
     netstack_t *ns)
 {

@@ -4701,11 +4704,11 @@
         uint_t nmech_infos;
         int crypto_rc, i;
         crypto_mech_usage_t mask;
         ipsec_stack_t   *ipss = ns->netstack_ipsec;
 
-        ASSERT(MUTEX_HELD(&ipss->ipsec_alg_lock));
+        ASSERT(RW_WRITE_HELD(&ipss->ipsec_alg_lock));
 
         /*
          * Compute the min, max, and default key sizes (in number of
          * increments to the default key size in bits) as defined
          * by the algorithm mappings. This range of key sizes is used

@@ -5054,11 +5057,11 @@
         /*
          * Walk the list of currently defined IPsec algorithm. Update
          * the algorithm valid flag and trigger an update of the
          * SAs that depend on that algorithm.
          */
-        mutex_enter(&ipss->ipsec_alg_lock);
+        rw_enter(&ipss->ipsec_alg_lock, RW_WRITER);
         for (algtype = 0; algtype < IPSEC_NALGTYPES; algtype++) {
                 for (algidx = 0; algidx < ipss->ipsec_nalgs[algtype];
                     algidx++) {
 
                         algid = ipss->ipsec_sortlist[algtype][algidx];

@@ -5117,11 +5120,11 @@
                                 sadb_alg_update(algtype, alg->alg_id,
                                     prov_change->ec_change ==
                                     CRYPTO_MECH_ADDED, ns);
                 }
         }
-        mutex_exit(&ipss->ipsec_alg_lock);
+        rw_exit(&ipss->ipsec_alg_lock);
         crypto_free_mech_list(mechs, mech_count);
 
         if (alg_changed) {
                 /*
                  * An algorithm has changed, i.e. it became valid or