4912 soft_giant_mutex in pkcs11_softtoken stays held after fork

@@ -19,10 +19,12 @@
  * CDDL HEADER END
  */
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ *
+ * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
  */
 
 #include <strings.h>
 #include <errno.h>
 #include <cryptoutil.h>

@@ -126,16 +128,22 @@
 obj_to_be_freed_list_t obj_delay_freed;
 ses_to_be_freed_list_t ses_delay_freed;
 
 /* protects softtoken_initialized and access to C_Initialize/C_Finalize */
 pthread_mutex_t soft_giant_mutex = PTHREAD_MUTEX_INITIALIZER;
+/*
+ * ONLY SET TO TRUE BY softtoken_fork_prepare().
+ * ONLY SET TO FALSE BY softtoken_fork_after().
+ */
+boolean_t fork_prepared = B_FALSE;
 
 static CK_RV finalize_common(boolean_t force, CK_VOID_PTR pReserved);
 static void softtoken_init();
 static void softtoken_fini();
 static void softtoken_fork_prepare();
 static void softtoken_fork_after();
+static void softtoken_fork_after_droplocks(void);
 
 CK_RV
 C_Initialize(CK_VOID_PTR pInitArgs)
 {
 

@@ -409,20 +417,45 @@
  * is done if softtoken is ever unloaded without a C_Finalize() call.
  */
 static void
 softtoken_fini()
 {
+        int rc;
+
+        rc = pthread_mutex_trylock(&soft_giant_mutex);
+        /*
+         * Check to see if it was acquired by softtoken_fork_prepare().
+         */
+        if (rc == EBUSY && fork_prepared)
+                rc = 0;
+
+        if (rc != 0) {
+                /*
+                 * I don't know WHAT just happened, but it's pretty bad.  Grab
+                 * the lock for real, even if it means deadlocking here.
+                 */
         (void) pthread_mutex_lock(&soft_giant_mutex);
+        } /* Else we acquired the lock and life is good. */
 
-        /* if we're not initilized, do not attempt to finalize */
-        if (!softtoken_initialized) {
-                (void) pthread_mutex_unlock(&soft_giant_mutex);
-                return;
+        /* Only finalize if we're initialized. */
+        if (softtoken_initialized) {
+                /*
+                 * We also have to DROP all of the other locks that
+                 * softtoken_fork_prepare did IF it was prepared, because
+                 * finalize_common() is going to pick them right back up.
+                 */
+                if (fork_prepared)
+                        softtoken_fork_after_droplocks();
+                (void) finalize_common(B_TRUE, NULL_PTR);
+                /* ASSERT(!softtoken_initialized); */
         }
 
-        (void) finalize_common(B_TRUE, NULL_PTR);
-
+        /*
+         * If we don't want to drop the lock, we assume softtoken_fork_after()
+         * will get called.
+         */
+        if (!fork_prepared)
         (void) pthread_mutex_unlock(&soft_giant_mutex);
 }
 
 CK_RV
 C_GetInfo(CK_INFO_PTR pInfo)

@@ -494,11 +527,11 @@
  * 6. all soft_session_list mutexes via soft_acquire_all_session_mutexes()
  * 7. obj_delay_freed.obj_to_be_free_mutex;
  * 8. ses_delay_freed.ses_to_be_free_mutex
  */
 void
-softtoken_fork_prepare()
+softtoken_fork_prepare(void)
 {
         (void) pthread_mutex_lock(&soft_giant_mutex);
         if (softtoken_initialized) {
                 (void) pthread_mutex_lock(&soft_sessionlist_mutex);
                 (void) pthread_mutex_lock(&soft_slot.slot_mutex);

@@ -508,27 +541,33 @@
                 (void) pthread_mutex_lock(
                     &obj_delay_freed.obj_to_be_free_mutex);
                 (void) pthread_mutex_lock(
                     &ses_delay_freed.ses_to_be_free_mutex);
         }
+        fork_prepared = B_TRUE;
 }
 
-/*
- * Release in opposite order to softtoken_fork_prepare().
- * Function is used for parent and child.
- */
-void
-softtoken_fork_after()
+static void
+softtoken_fork_after_droplocks(void)
 {
-        if (softtoken_initialized) {
-                (void) pthread_mutex_unlock(
-                    &ses_delay_freed.ses_to_be_free_mutex);
-                (void) pthread_mutex_unlock(
-                    &obj_delay_freed.obj_to_be_free_mutex);
+        /* ASSERT(softtoken_initialized); */
+        (void) pthread_mutex_unlock(&ses_delay_freed.ses_to_be_free_mutex);
+        (void) pthread_mutex_unlock(&obj_delay_freed.obj_to_be_free_mutex);
                 soft_release_all_session_mutexes(soft_session_list);
                 soft_release_all_session_mutexes(&token_session);
                 (void) pthread_mutex_unlock(&soft_slot.keystore_mutex);
                 (void) pthread_mutex_unlock(&soft_slot.slot_mutex);
                 (void) pthread_mutex_unlock(&soft_sessionlist_mutex);
-        }
+}
+
+/*
+ * Release in opposite order to softtoken_fork_prepare().
+ * Function is used for parent and child.
+ */
+void
+softtoken_fork_after(void)
+{
+        if (softtoken_initialized)
+                softtoken_fork_after_droplocks();
+        fork_prepared = B_FALSE;
         (void) pthread_mutex_unlock(&soft_giant_mutex);
 }