NEX-16554 alloca() in libscf causes an iSCSI application to crash
Reviewed by: Yuri Pankon <yuri.pankov@nexenta.com>
Reviewed by: Roman Strashkin <roman.strashkin@nexenta.com>
Reviewed by: Evan Layton <evan.layton@nexenta.com>

@@ -36,11 +36,10 @@
 
 #include "repcache_protocol.h"
 #include "scf_type.h"
 
 #include <assert.h>
-#include <alloca.h>
 #include <door.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <fnmatch.h>
 #include <libuutil.h>

@@ -4080,12 +4079,16 @@
                 assert(TX_SIZE(size) == size);
                 total += size;
         }
 
         request_size = REP_PROTOCOL_TRANSACTION_COMMIT_SIZE(total);
-        request = alloca(request_size);
-        (void) memset(request, '\0', request_size);
+        request = calloc(1, request_size);
+        if (request == NULL) {
+                (void) pthread_mutex_unlock(&h->rh_lock);
+                return (scf_set_error(SCF_ERROR_NO_MEMORY));
+        }
+
         request->rpr_request = REP_PROTOCOL_PROPERTYGRP_TX_COMMIT;
         request->rpr_entityid = tran->tran_pg.rd_d.rd_entity;
         request->rpr_size = request_size;
         cmd = (uintptr_t)request->rpr_cmd;
 

@@ -4095,11 +4098,12 @@
         for (cur = uu_list_first(tran->tran_props); cur != NULL;
             cur = uu_list_next(tran->tran_props, cur)) {
                 size = commit_process(cur, (void *)cmd);
                 if (size == BAD_SIZE) {
                         (void) pthread_mutex_unlock(&h->rh_lock);
-                        return (scf_set_error(SCF_ERROR_INTERNAL));
+                        r = (scf_set_error(SCF_ERROR_INTERNAL));
+                        goto out;
                 }
                 cmd += size;
                 new_total += size;
         }
         assert(new_total == total);

@@ -4113,16 +4117,21 @@
         }
 
         if (response.rpr_response != REP_PROTOCOL_SUCCESS &&
             response.rpr_response != REP_PROTOCOL_FAIL_NOT_LATEST) {
                 (void) pthread_mutex_unlock(&h->rh_lock);
-                return (scf_set_error(proto_error(response.rpr_response)));
+                r = (scf_set_error(proto_error(response.rpr_response)));
+                goto out;
         }
 
         tran->tran_state = TRAN_STATE_COMMITTED;
         (void) pthread_mutex_unlock(&h->rh_lock);
-        return (response.rpr_response == REP_PROTOCOL_SUCCESS);
+        r = (response.rpr_response == REP_PROTOCOL_SUCCESS);
+
+out:
+        free(request);
+        return (r);
 }
 
 static void
 transaction_reset(scf_transaction_t *tran)
 {