4912 soft_giant_mutex in pkcs11_softtoken stays held after fork

*** 19,28 ****
--- 19,30 ----
   * CDDL HEADER END
   */
  /*
   * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
+  *
+  * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved.
   */
  
  #include <strings.h>
  #include <errno.h>
  #include <cryptoutil.h>
*** 126,141 ****
--- 128,149 ----
  obj_to_be_freed_list_t obj_delay_freed;
  ses_to_be_freed_list_t ses_delay_freed;
  
  /* protects softtoken_initialized and access to C_Initialize/C_Finalize */
  pthread_mutex_t soft_giant_mutex = PTHREAD_MUTEX_INITIALIZER;
+ /*
+  * ONLY SET TO TRUE BY softtoken_fork_prepare().
+  * ONLY SET TO FALSE BY softtoken_fork_after().
+  */
+ boolean_t fork_prepared = B_FALSE;
  
  static CK_RV finalize_common(boolean_t force, CK_VOID_PTR pReserved);
  static void softtoken_init();
  static void softtoken_fini();
  static void softtoken_fork_prepare();
  static void softtoken_fork_after();
+ static void softtoken_fork_after_droplocks(void);
  
  CK_RV
  C_Initialize(CK_VOID_PTR pInitArgs)
  {
  
*** 409,428 ****
   * is done if softtoken is ever unloaded without a C_Finalize() call.
   */
  static void
  softtoken_fini()
  {
          (void) pthread_mutex_lock(&soft_giant_mutex);
  
!         /* if we're not initilized, do not attempt to finalize */
!         if (!softtoken_initialized) {
!                 (void) pthread_mutex_unlock(&soft_giant_mutex);
!                 return;
          }
  
!         (void) finalize_common(B_TRUE, NULL_PTR);
! 
          (void) pthread_mutex_unlock(&soft_giant_mutex);
  }
  
  CK_RV
  C_GetInfo(CK_INFO_PTR pInfo)
--- 417,461 ----
   * is done if softtoken is ever unloaded without a C_Finalize() call.
   */
  static void
  softtoken_fini()
  {
+         int rc;
+ 
+         rc = pthread_mutex_trylock(&soft_giant_mutex);
+         /*
+          * Check to see if it was acquired by softtoken_fork_prepare().
+          */
+         if (rc == EBUSY && fork_prepared)
+                 rc = 0;
+ 
+         if (rc != 0) {
+                 /*
+                  * I don't know WHAT just happened, but it's pretty bad.  Grab
+                  * the lock for real, even if it means deadlocking here.
+                  */
                  (void) pthread_mutex_lock(&soft_giant_mutex);
+         } /* Else we acquired the lock and life is good. */
  
!         /* Only finalize if we're initialized. */
!         if (softtoken_initialized) {
!                 /*
!                  * We also have to DROP all of the other locks that
!                  * softtoken_fork_prepare did IF it was prepared, because
!                  * finalize_common() is going to pick them right back up.
!                  */
!                 if (fork_prepared)
!                         softtoken_fork_after_droplocks();
!                 (void) finalize_common(B_TRUE, NULL_PTR);
!                 /* ASSERT(!softtoken_initialized); */
          }
  
!         /*
!          * If we don't want to drop the lock, we assume softtoken_fork_after()
!          * will get called.
!          */
!         if (!fork_prepared)
                  (void) pthread_mutex_unlock(&soft_giant_mutex);
  }
  
  CK_RV
  C_GetInfo(CK_INFO_PTR pInfo)
*** 494,504 ****
   * 6. all soft_session_list mutexes via soft_acquire_all_session_mutexes()
   * 7. obj_delay_freed.obj_to_be_free_mutex;
   * 8. ses_delay_freed.ses_to_be_free_mutex
   */
  void
! softtoken_fork_prepare()
  {
          (void) pthread_mutex_lock(&soft_giant_mutex);
          if (softtoken_initialized) {
                  (void) pthread_mutex_lock(&soft_sessionlist_mutex);
                  (void) pthread_mutex_lock(&soft_slot.slot_mutex);
--- 527,537 ----
   * 6. all soft_session_list mutexes via soft_acquire_all_session_mutexes()
   * 7. obj_delay_freed.obj_to_be_free_mutex;
   * 8. ses_delay_freed.ses_to_be_free_mutex
   */
  void
! softtoken_fork_prepare(void)
  {
          (void) pthread_mutex_lock(&soft_giant_mutex);
          if (softtoken_initialized) {
                  (void) pthread_mutex_lock(&soft_sessionlist_mutex);
                  (void) pthread_mutex_lock(&soft_slot.slot_mutex);
*** 508,534 ****
                  (void) pthread_mutex_lock(
                      &obj_delay_freed.obj_to_be_free_mutex);
                  (void) pthread_mutex_lock(
                      &ses_delay_freed.ses_to_be_free_mutex);
          }
  }
  
! /*
!  * Release in opposite order to softtoken_fork_prepare().
!  * Function is used for parent and child.
!  */
! void
! softtoken_fork_after()
  {
!         if (softtoken_initialized) {
!                 (void) pthread_mutex_unlock(
!                     &ses_delay_freed.ses_to_be_free_mutex);
!                 (void) pthread_mutex_unlock(
!                     &obj_delay_freed.obj_to_be_free_mutex);
                  soft_release_all_session_mutexes(soft_session_list);
                  soft_release_all_session_mutexes(&token_session);
                  (void) pthread_mutex_unlock(&soft_slot.keystore_mutex);
                  (void) pthread_mutex_unlock(&soft_slot.slot_mutex);
                  (void) pthread_mutex_unlock(&soft_sessionlist_mutex);
!         }
          (void) pthread_mutex_unlock(&soft_giant_mutex);
  }
--- 541,573 ----
                  (void) pthread_mutex_lock(
                      &obj_delay_freed.obj_to_be_free_mutex);
                  (void) pthread_mutex_lock(
                      &ses_delay_freed.ses_to_be_free_mutex);
          }
+         fork_prepared = B_TRUE;
  }
  
! static void
! softtoken_fork_after_droplocks(void)
  {
!         /* ASSERT(softtoken_initialized); */
!         (void) pthread_mutex_unlock(&ses_delay_freed.ses_to_be_free_mutex);
!         (void) pthread_mutex_unlock(&obj_delay_freed.obj_to_be_free_mutex);
          soft_release_all_session_mutexes(soft_session_list);
          soft_release_all_session_mutexes(&token_session);
          (void) pthread_mutex_unlock(&soft_slot.keystore_mutex);
          (void) pthread_mutex_unlock(&soft_slot.slot_mutex);
          (void) pthread_mutex_unlock(&soft_sessionlist_mutex);
! }
! 
! /*
!  * Release in opposite order to softtoken_fork_prepare().
!  * Function is used for parent and child.
!  */
! void
! softtoken_fork_after(void)
! {
!         if (softtoken_initialized)
!                 softtoken_fork_after_droplocks();
!         fork_prepared = B_FALSE;
          (void) pthread_mutex_unlock(&soft_giant_mutex);
  }