Print this page
NEX-20260 NFS hung in transitional state when RSF marks it maintenance
NEX-20423 NFSv4 state database entry locking is not always used around reference count.
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
NEX-19996 exi_id_get_next() calls should be WRITER locked
NEX-20014 NFS v4 state lock mutex exited before entered (on error path)
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
Reviewed by: Joyce McIntosh <joyce.mcintosh@nexenta.com>
NEX-16452 NFS server in a zone state database needs to be per zone
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
NEX-15279 support NFS server in zone
NEX-15520 online NFS shares cause zoneadm halt to hang in nfs_export_zone_fini
Portions contributed by: Dan Kruchinin dan.kruchinin@nexenta.com
Portions contributed by: Stepan Zastupov stepan.zastupov@gmail.com
Reviewed by: Joyce McIntosh <joyce.mcintosh@nexenta.com>
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>

Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/fs/nfs/nfs4_state.c
          +++ new/usr/src/uts/common/fs/nfs/nfs4_state.c
↓ open down ↓ 10 lines elided ↑ open up ↑
  11   11   * and limitations under the License.
  12   12   *
  13   13   * When distributing Covered Code, include this CDDL HEADER in each
  14   14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15   15   * If applicable, add the following below this CDDL HEADER, with the
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
       21 +
  21   22  /*
  22   23   * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  23      - * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  24   24   */
  25   25  
       26 +/*
       27 + * Copyright 2018 Nexenta Systems, Inc.
       28 + */
       29 +
  26   30  #include <sys/systm.h>
  27   31  #include <sys/kmem.h>
  28   32  #include <sys/cmn_err.h>
  29   33  #include <sys/atomic.h>
  30   34  #include <sys/clconf.h>
  31   35  #include <sys/cladm.h>
  32   36  #include <sys/flock.h>
  33   37  #include <nfs/export.h>
  34   38  #include <nfs/nfs.h>
  35   39  #include <nfs/nfs4.h>
  36   40  #include <nfs/nfssys.h>
  37   41  #include <nfs/lm.h>
  38   42  #include <sys/pathname.h>
  39   43  #include <sys/sdt.h>
  40   44  #include <sys/nvpair.h>
  41   45  
  42   46  extern u_longlong_t nfs4_srv_caller_id;
  43   47  
  44      -extern time_t rfs4_start_time;
  45   48  extern uint_t nfs4_srv_vkey;
  46   49  
  47   50  stateid4 special0 = {
  48   51          0,
  49   52          { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
  50   53  };
  51   54  
  52   55  stateid4 special1 = {
  53   56          0xffffffff,
  54   57          {
↓ open down ↓ 10 lines elided ↑ open up ↑
  65   68  /* For embedding the cluster nodeid into our clientid */
  66   69  #define CLUSTER_NODEID_SHIFT    24
  67   70  #define CLUSTER_MAX_NODEID      255
  68   71  
  69   72  #ifdef DEBUG
  70   73  int rfs4_debug;
  71   74  #endif
  72   75  
  73   76  static uint32_t rfs4_database_debug = 0x00;
  74   77  
  75      -static void rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf);
       78 +/* CSTYLED */
       79 +static void rfs4_ss_clid_write(nfs4_srv_t *nsrv4, rfs4_client_t *cp, char *leaf);
  76   80  static void rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dir, char *leaf);
  77   81  static void rfs4_dss_clear_oldstate(rfs4_servinst_t *sip);
  78   82  static void rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip);
  79   83  
  80   84  /*
  81   85   * Couple of simple init/destroy functions for a general waiter
  82   86   */
  83   87  void
  84   88  rfs4_sw_init(rfs4_state_wait_t *swp)
  85   89  {
↓ open down ↓ 28 lines elided ↑ open up ↑
 114  118  rfs4_sw_exit(rfs4_state_wait_t *swp)
 115  119  {
 116  120          mutex_enter(swp->sw_cv_lock);
 117  121          ASSERT(swp->sw_active == TRUE);
 118  122          swp->sw_active = FALSE;
 119  123          if (swp->sw_wait_count != 0)
 120  124                  cv_broadcast(swp->sw_cv);
 121  125          mutex_exit(swp->sw_cv_lock);
 122  126  }
 123  127  
 124      -/*
 125      - * CPR callback id -- not related to v4 callbacks
 126      - */
 127      -static callb_id_t cpr_id = 0;
 128      -
 129  128  static void
 130  129  deep_lock_copy(LOCK4res *dres, LOCK4res *sres)
 131  130  {
 132  131          lock_owner4 *slo = &sres->LOCK4res_u.denied.owner;
 133  132          lock_owner4 *dlo = &dres->LOCK4res_u.denied.owner;
 134  133  
 135  134          if (sres->status == NFS4ERR_DENIED) {
 136  135                  dlo->owner_val = kmem_alloc(slo->owner_len, KM_SLEEP);
 137  136                  bcopy(slo->owner_val, dlo->owner_val, slo->owner_len);
 138  137          }
 139  138  }
 140  139  
      140 +/*
      141 + * CPR callback id -- not related to v4 callbacks
      142 + */
      143 +static callb_id_t cpr_id = 0;
      144 +
 141  145  static void
 142  146  deep_lock_free(LOCK4res *res)
 143  147  {
 144  148          lock_owner4 *lo = &res->LOCK4res_u.denied.owner;
 145  149  
 146  150          if (res->status == NFS4ERR_DENIED)
 147  151                  kmem_free(lo->owner_val, lo->owner_len);
 148  152  }
 149  153  
 150  154  static void
↓ open down ↓ 115 lines elided ↑ open up ↑
 266  270   */
 267  271  
 268  272  #ifdef DEBUG
 269  273  #define TABSIZE 17
 270  274  #else
 271  275  #define TABSIZE 2047
 272  276  #endif
 273  277  
 274  278  #define ADDRHASH(key) ((unsigned long)(key) >> 3)
 275  279  
 276      -/* Used to serialize create/destroy of rfs4_server_state database */
 277      -kmutex_t        rfs4_state_lock;
 278      -static rfs4_database_t *rfs4_server_state = NULL;
 279      -
 280      -/* Used to serialize lookups of clientids */
 281      -static  krwlock_t       rfs4_findclient_lock;
 282      -
 283      -/*
 284      - * For now this "table" is exposed so that the CPR callback
 285      - * function can tromp through it..
 286      - */
 287      -rfs4_table_t *rfs4_client_tab;
 288      -
 289      -static rfs4_index_t *rfs4_clientid_idx;
 290      -static rfs4_index_t *rfs4_nfsclnt_idx;
 291      -static rfs4_table_t *rfs4_clntip_tab;
 292      -static rfs4_index_t *rfs4_clntip_idx;
 293      -static rfs4_table_t *rfs4_openowner_tab;
 294      -static rfs4_index_t *rfs4_openowner_idx;
 295      -static rfs4_table_t *rfs4_state_tab;
 296      -static rfs4_index_t *rfs4_state_idx;
 297      -static rfs4_index_t *rfs4_state_owner_file_idx;
 298      -static rfs4_index_t *rfs4_state_file_idx;
 299      -static rfs4_table_t *rfs4_lo_state_tab;
 300      -static rfs4_index_t *rfs4_lo_state_idx;
 301      -static rfs4_index_t *rfs4_lo_state_owner_idx;
 302      -static rfs4_table_t *rfs4_lockowner_tab;
 303      -static rfs4_index_t *rfs4_lockowner_idx;
 304      -static rfs4_index_t *rfs4_lockowner_pid_idx;
 305      -static rfs4_table_t *rfs4_file_tab;
 306      -static rfs4_index_t *rfs4_file_idx;
 307      -static rfs4_table_t *rfs4_deleg_state_tab;
 308      -static rfs4_index_t *rfs4_deleg_idx;
 309      -static rfs4_index_t *rfs4_deleg_state_idx;
 310      -
 311  280  #define MAXTABSZ 1024*1024
 312  281  
 313  282  /* The values below are rfs4_lease_time units */
 314  283  
 315  284  #ifdef DEBUG
 316  285  #define CLIENT_CACHE_TIME 1
 317  286  #define OPENOWNER_CACHE_TIME 1
 318  287  #define STATE_CACHE_TIME 1
 319  288  #define LO_STATE_CACHE_TIME 1
 320  289  #define LOCKOWNER_CACHE_TIME 1
↓ open down ↓ 2 lines elided ↑ open up ↑
 323  292  #else
 324  293  #define CLIENT_CACHE_TIME 10
 325  294  #define OPENOWNER_CACHE_TIME 5
 326  295  #define STATE_CACHE_TIME 1
 327  296  #define LO_STATE_CACHE_TIME 1
 328  297  #define LOCKOWNER_CACHE_TIME 3
 329  298  #define FILE_CACHE_TIME 40
 330  299  #define DELEG_STATE_CACHE_TIME 1
 331  300  #endif
 332  301  
      302 +/*
      303 + * NFSv4 server state databases
      304 + *
      305 + * Initilized when the module is loaded and used by NFSv4 state tables.
      306 + * These kmem_cache databases are global, the tables that make use of these
      307 + * are per zone.
      308 + */
      309 +kmem_cache_t *rfs4_client_mem_cache;
      310 +kmem_cache_t *rfs4_clntIP_mem_cache;
      311 +kmem_cache_t *rfs4_openown_mem_cache;
      312 +kmem_cache_t *rfs4_openstID_mem_cache;
      313 +kmem_cache_t *rfs4_lockstID_mem_cache;
      314 +kmem_cache_t *rfs4_lockown_mem_cache;
      315 +kmem_cache_t *rfs4_file_mem_cache;
      316 +kmem_cache_t *rfs4_delegstID_mem_cache;
 333  317  
 334      -static time_t rfs4_client_cache_time = 0;
 335      -static time_t rfs4_clntip_cache_time = 0;
 336      -static time_t rfs4_openowner_cache_time = 0;
 337      -static time_t rfs4_state_cache_time = 0;
 338      -static time_t rfs4_lo_state_cache_time = 0;
 339      -static time_t rfs4_lockowner_cache_time = 0;
 340      -static time_t rfs4_file_cache_time = 0;
 341      -static time_t rfs4_deleg_state_cache_time = 0;
 342      -
      318 +/*
      319 + * NFSv4 state table functions
      320 + */
 343  321  static bool_t rfs4_client_create(rfs4_entry_t, void *);
 344  322  static void rfs4_dss_remove_cpleaf(rfs4_client_t *);
 345  323  static void rfs4_dss_remove_leaf(rfs4_servinst_t *, char *, char *);
 346  324  static void rfs4_client_destroy(rfs4_entry_t);
 347  325  static bool_t rfs4_client_expiry(rfs4_entry_t);
 348  326  static uint32_t clientid_hash(void *);
 349  327  static bool_t clientid_compare(rfs4_entry_t, void *);
 350  328  static void *clientid_mkkey(rfs4_entry_t);
 351  329  static uint32_t nfsclnt_hash(void *);
 352  330  static bool_t nfsclnt_compare(rfs4_entry_t, void *);
↓ open down ↓ 345 lines elided ↑ open up ↑
 698  676          }
 699  677  
 700  678  out:
 701  679          (void) VOP_CLOSE(dvp, FREAD, 1, (offset_t)0, CRED(), NULL);
 702  680          VN_RELE(dvp);
 703  681          if (dirt)
 704  682                  kmem_free((caddr_t)dirt, RFS4_SS_DIRSIZE);
 705  683  }
 706  684  
 707  685  static void
 708      -rfs4_ss_init(void)
      686 +rfs4_ss_init(nfs4_srv_t *nsrv4)
 709  687  {
 710  688          int npaths = 1;
 711  689          char *default_dss_path = NFS4_DSS_VAR_DIR;
 712  690  
 713  691          /* read the default stable storage state */
 714      -        rfs4_dss_readstate(npaths, &default_dss_path);
      692 +        rfs4_dss_readstate(nsrv4, npaths, &default_dss_path);
 715  693  
 716  694          rfs4_ss_enabled = 1;
 717  695  }
 718  696  
 719  697  static void
 720      -rfs4_ss_fini(void)
      698 +rfs4_ss_fini(nfs4_srv_t *nsrv4)
 721  699  {
 722  700          rfs4_servinst_t *sip;
 723  701  
 724      -        mutex_enter(&rfs4_servinst_lock);
 725      -        sip = rfs4_cur_servinst;
      702 +        mutex_enter(&nsrv4->servinst_lock);
      703 +        sip = nsrv4->nfs4_cur_servinst;
 726  704          while (sip != NULL) {
 727  705                  rfs4_dss_clear_oldstate(sip);
 728  706                  sip = sip->next;
 729  707          }
 730      -        mutex_exit(&rfs4_servinst_lock);
      708 +        mutex_exit(&nsrv4->servinst_lock);
 731  709  }
 732  710  
 733  711  /*
 734  712   * Remove all oldstate files referenced by this servinst.
 735  713   */
 736  714  static void
 737  715  rfs4_dss_clear_oldstate(rfs4_servinst_t *sip)
 738  716  {
 739  717          rfs4_oldstate_t *os_head, *osp;
 740  718  
↓ open down ↓ 23 lines elided ↑ open up ↑
 764  742                  osp = os_next;
 765  743          }
 766  744  
 767  745          rw_exit(&sip->oldstate_lock);
 768  746  }
 769  747  
 770  748  /*
 771  749   * Form the state and oldstate paths, and read in the stable storage files.
 772  750   */
 773  751  void
 774      -rfs4_dss_readstate(int npaths, char **paths)
      752 +rfs4_dss_readstate(nfs4_srv_t *nsrv4, int npaths, char **paths)
 775  753  {
 776  754          int i;
 777  755          char *state, *oldstate;
 778  756  
 779  757          state = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 780  758          oldstate = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 781  759  
 782  760          for (i = 0; i < npaths; i++) {
 783  761                  char *path = paths[i];
 784  762  
↓ open down ↓ 3 lines elided ↑ open up ↑
 788  766                  /*
 789  767                   * Populate the current server instance's oldstate list.
 790  768                   *
 791  769                   * 1. Read stable storage data from old state directory,
 792  770                   *    leaving its contents alone.
 793  771                   *
 794  772                   * 2. Read stable storage data from state directory,
 795  773                   *    and move the latter's contents to old state
 796  774                   *    directory.
 797  775                   */
 798      -                rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, oldstate, NULL);
 799      -                rfs4_ss_oldstate(rfs4_cur_servinst->oldstate, state, oldstate);
      776 +                /* CSTYLED */
      777 +                rfs4_ss_oldstate(nsrv4->nfs4_cur_servinst->oldstate, oldstate, NULL);
      778 +                /* CSTYLED */
      779 +                rfs4_ss_oldstate(nsrv4->nfs4_cur_servinst->oldstate, state, oldstate);
 800  780          }
 801  781  
 802  782          kmem_free(state, MAXPATHLEN);
 803  783          kmem_free(oldstate, MAXPATHLEN);
 804  784  }
 805  785  
 806  786  
 807  787  /*
 808  788   * Check if we are still in grace and if the client can be
 809  789   * granted permission to perform reclaims.
 810  790   */
 811  791  void
 812      -rfs4_ss_chkclid(rfs4_client_t *cp)
      792 +rfs4_ss_chkclid(nfs4_srv_t *nsrv4, rfs4_client_t *cp)
 813  793  {
 814  794          rfs4_servinst_t *sip;
 815  795  
 816  796          /*
 817  797           * It should be sufficient to check the oldstate data for just
 818  798           * this client's instance. However, since our per-instance
 819  799           * client grouping is solely temporal, HA-NFSv4 RG failover
 820  800           * might result in clients of the same RG being partitioned into
 821  801           * separate instances.
 822  802           *
 823  803           * Until the client grouping is improved, we must check the
 824  804           * oldstate data for all instances with an active grace period.
 825  805           *
 826  806           * This also serves as the mechanism to remove stale oldstate data.
 827  807           * The first time we check an instance after its grace period has
 828  808           * expired, the oldstate data should be cleared.
 829  809           *
 830  810           * Start at the current instance, and walk the list backwards
 831  811           * to the first.
 832  812           */
 833      -        mutex_enter(&rfs4_servinst_lock);
 834      -        for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
      813 +        mutex_enter(&nsrv4->servinst_lock);
      814 +        for (sip = nsrv4->nfs4_cur_servinst; sip != NULL; sip = sip->prev) {
 835  815                  rfs4_ss_chkclid_sip(cp, sip);
 836  816  
 837  817                  /* if the above check found this client, we're done */
 838  818                  if (cp->rc_can_reclaim)
 839  819                          break;
 840  820          }
 841      -        mutex_exit(&rfs4_servinst_lock);
      821 +        mutex_exit(&nsrv4->servinst_lock);
 842  822  }
 843  823  
 844  824  static void
 845  825  rfs4_ss_chkclid_sip(rfs4_client_t *cp, rfs4_servinst_t *sip)
 846  826  {
 847  827          rfs4_oldstate_t *osp, *os_head;
 848  828  
 849  829          /* short circuit everything if this server instance has no oldstate */
 850  830          rw_enter(&sip->oldstate_lock, RW_READER);
 851  831          os_head = sip->oldstate;
↓ open down ↓ 29 lines elided ↑ open up ↑
 881  861  
 882  862          rw_exit(&sip->oldstate_lock);
 883  863  }
 884  864  
 885  865  /*
 886  866   * Place client information into stable storage: 1/3.
 887  867   * First, generate the leaf filename, from the client's IP address and
 888  868   * the server-generated short-hand clientid.
 889  869   */
 890  870  void
 891      -rfs4_ss_clid(rfs4_client_t *cp)
      871 +rfs4_ss_clid(nfs4_srv_t *nsrv4, rfs4_client_t *cp)
 892  872  {
 893  873          const char *kinet_ntop6(uchar_t *, char *, size_t);
 894  874          char leaf[MAXNAMELEN], buf[INET6_ADDRSTRLEN];
 895  875          struct sockaddr *ca;
 896  876          uchar_t *b;
 897  877  
 898  878          if (rfs4_ss_enabled == 0) {
 899  879                  return;
 900  880          }
 901  881  
↓ open down ↓ 11 lines elided ↑ open up ↑
 913  893          } else if (ca->sa_family == AF_INET6) {
 914  894                  struct sockaddr_in6 *sin6;
 915  895  
 916  896                  sin6 = (struct sockaddr_in6 *)ca;
 917  897                  (void) kinet_ntop6((uchar_t *)&sin6->sin6_addr,
 918  898                      buf, INET6_ADDRSTRLEN);
 919  899          }
 920  900  
 921  901          (void) snprintf(leaf, MAXNAMELEN, "%s-%llx", buf,
 922  902              (longlong_t)cp->rc_clientid);
 923      -        rfs4_ss_clid_write(cp, leaf);
      903 +        rfs4_ss_clid_write(nsrv4, cp, leaf);
 924  904  }
 925  905  
 926  906  /*
 927  907   * Place client information into stable storage: 2/3.
 928  908   * DSS: distributed stable storage: the file may need to be written to
 929  909   * multiple directories.
 930  910   */
 931  911  static void
 932      -rfs4_ss_clid_write(rfs4_client_t *cp, char *leaf)
      912 +rfs4_ss_clid_write(nfs4_srv_t *nsrv4, rfs4_client_t *cp, char *leaf)
 933  913  {
 934  914          rfs4_servinst_t *sip;
 935  915  
 936  916          /*
 937  917           * It should be sufficient to write the leaf file to (all) DSS paths
 938  918           * associated with just this client's instance. However, since our
 939  919           * per-instance client grouping is solely temporal, HA-NFSv4 RG
 940  920           * failover might result in us losing DSS data.
 941  921           *
 942  922           * Until the client grouping is improved, we must write the DSS data
 943  923           * to all instances' paths. Start at the current instance, and
 944  924           * walk the list backwards to the first.
 945  925           */
 946      -        mutex_enter(&rfs4_servinst_lock);
 947      -        for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
      926 +        mutex_enter(&nsrv4->servinst_lock);
      927 +        for (sip = nsrv4->nfs4_cur_servinst; sip != NULL; sip = sip->prev) {
 948  928                  int i, npaths = sip->dss_npaths;
 949  929  
 950  930                  /* write the leaf file to all DSS paths */
 951  931                  for (i = 0; i < npaths; i++) {
 952  932                          rfs4_dss_path_t *dss_path = sip->dss_paths[i];
 953  933  
 954  934                          /* HA-NFSv4 path might have been failed-away from us */
 955  935                          if (dss_path == NULL)
 956  936                                  continue;
 957  937  
 958  938                          rfs4_ss_clid_write_one(cp, dss_path->path, leaf);
 959  939                  }
 960  940          }
 961      -        mutex_exit(&rfs4_servinst_lock);
      941 +        mutex_exit(&nsrv4->servinst_lock);
 962  942  }
 963  943  
 964  944  /*
 965  945   * Place client information into stable storage: 3/3.
 966  946   * Write the stable storage data to the requested file.
 967  947   */
 968  948  static void
 969  949  rfs4_ss_clid_write_one(rfs4_client_t *cp, char *dss_path, char *leaf)
 970  950  {
 971  951          int ioflag;
↓ open down ↓ 172 lines elided ↑ open up ↑
1144 1124          }
1145 1125  }
1146 1126  
1147 1127  /*
1148 1128   * This is called from nfssys() in order to clear server state
1149 1129   * for the specified client IP Address.
1150 1130   */
1151 1131  void
1152 1132  rfs4_clear_client_state(struct nfs4clrst_args *clr)
1153 1133  {
1154      -        (void) rfs4_dbe_walk(rfs4_client_tab, rfs4_client_scrub, clr);
     1134 +        nfs4_srv_t *nsrv4;
     1135 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     1136 +        (void) rfs4_dbe_walk(nsrv4->rfs4_client_tab, rfs4_client_scrub, clr);
1155 1137  }
1156 1138  
1157 1139  /*
1158 1140   * Used to initialize the NFSv4 server's state or database.  All of
1159      - * the tables are created and timers are set. Only called when NFSv4
1160      - * service is provided.
     1141 + * the tables are created and timers are set.
1161 1142   */
1162 1143  void
1163      -rfs4_state_init()
     1144 +rfs4_state_g_init()
1164 1145  {
1165      -        int start_grace;
1166 1146          extern boolean_t rfs4_cpr_callb(void *, int);
1167      -        char *dss_path = NFS4_DSS_VAR_DIR;
1168      -        time_t start_time;
     1147 +        /*
     1148 +         * Add a CPR callback so that we can update client
     1149 +         * access times to extend the lease after a suspend
     1150 +         * and resume (using the same class as rpcmod/connmgr)
     1151 +         */
     1152 +        cpr_id = callb_add(rfs4_cpr_callb, 0, CB_CL_CPR_RPC, "rfs4");
1169 1153  
1170      -        mutex_enter(&rfs4_state_lock);
     1154 +        /*
     1155 +         * NFSv4 server state databases
     1156 +         *
     1157 +         * Initilized when the module is loaded and used by NFSv4 state tables.
     1158 +         * These kmem_cache free pools are used globally, the NFSv4 state
     1159 +         * tables which make use of these kmem_cache free pools are per zone.
     1160 +         *
     1161 +         * initialize the global kmem_cache free pools which will be used by
     1162 +         * the NFSv4 state tables.
     1163 +         */
     1164 +        /* CSTYLED */
     1165 +        rfs4_client_mem_cache = nfs4_init_mem_cache("Client_entry_cache", 2, sizeof (rfs4_client_t), 0);
     1166 +        /* CSTYLED */
     1167 +        rfs4_clntIP_mem_cache = nfs4_init_mem_cache("ClntIP_entry_cache", 1, sizeof (rfs4_clntip_t), 1);
     1168 +        /* CSTYLED */
     1169 +        rfs4_openown_mem_cache = nfs4_init_mem_cache("OpenOwner_entry_cache", 1, sizeof (rfs4_openowner_t), 2);
     1170 +        /* CSTYLED */
     1171 +        rfs4_openstID_mem_cache = nfs4_init_mem_cache("OpenStateID_entry_cache", 3, sizeof (rfs4_state_t), 3);
     1172 +        /* CSTYLED */
     1173 +        rfs4_lockstID_mem_cache = nfs4_init_mem_cache("LockStateID_entry_cache", 3, sizeof (rfs4_lo_state_t), 4);
     1174 +        /* CSTYLED */
     1175 +        rfs4_lockown_mem_cache = nfs4_init_mem_cache("Lockowner_entry_cache", 2, sizeof (rfs4_lockowner_t), 5);
     1176 +        /* CSTYLED */
     1177 +        rfs4_file_mem_cache = nfs4_init_mem_cache("File_entry_cache", 1, sizeof (rfs4_file_t), 6);
     1178 +        /* CSTYLED */
     1179 +        rfs4_delegstID_mem_cache = nfs4_init_mem_cache("DelegStateID_entry_cache", 2, sizeof (rfs4_deleg_state_t), 7);
1171 1180  
     1181 +        rfs4_client_clrst = rfs4_clear_client_state;
     1182 +}
     1183 +
     1184 +
     1185 +/*
     1186 + * Used at server shutdown to cleanup all of the NFSv4 server's structures
     1187 + * and other state.
     1188 + */
     1189 +void
     1190 +rfs4_state_g_fini()
     1191 +{
     1192 +        int i;
1172 1193          /*
1173      -         * If the server state database has already been initialized,
1174      -         * skip it
     1194 +         * Cleanup the CPR callback.
1175 1195           */
1176      -        if (rfs4_server_state != NULL) {
1177      -                mutex_exit(&rfs4_state_lock);
1178      -                return;
     1196 +        if (cpr_id)
     1197 +                (void) callb_delete(cpr_id);
     1198 +
     1199 +        rfs4_client_clrst = NULL;
     1200 +
     1201 +        /* free the NFSv4 state databases */
     1202 +        for (i = 0; i < RFS4_DB_MEM_CACHE_NUM; i++) {
     1203 +                kmem_cache_destroy(rfs4_db_mem_cache_table[i].r_db_mem_cache);
     1204 +                rfs4_db_mem_cache_table[i].r_db_mem_cache = NULL;
1179 1205          }
1180 1206  
1181      -        rw_init(&rfs4_findclient_lock, NULL, RW_DEFAULT, NULL);
     1207 +        rfs4_client_mem_cache = NULL;
     1208 +        rfs4_clntIP_mem_cache = NULL;
     1209 +        rfs4_openown_mem_cache = NULL;
     1210 +        rfs4_openstID_mem_cache = NULL;
     1211 +        rfs4_lockstID_mem_cache = NULL;
     1212 +        rfs4_lockown_mem_cache = NULL;
     1213 +        rfs4_file_mem_cache = NULL;
     1214 +        rfs4_delegstID_mem_cache = NULL;
1182 1215  
     1216 +        /* DSS: distributed stable storage */
     1217 +        nvlist_free(rfs4_dss_oldpaths);
     1218 +        nvlist_free(rfs4_dss_paths);
     1219 +        rfs4_dss_paths = rfs4_dss_oldpaths = NULL;
     1220 +}
     1221 +
     1222 +/*
     1223 + * Used to initialize the per zone NFSv4 server's state
     1224 + */
     1225 +void
     1226 +rfs4_state_zone_init(nfs4_srv_t *nsrv4)
     1227 +{
     1228 +        time_t start_time;
     1229 +        int start_grace;
     1230 +        char *dss_path = NFS4_DSS_VAR_DIR;
     1231 +
     1232 +        /* DSS: distributed stable storage: initialise served paths list */
     1233 +        nsrv4->dss_pathlist = NULL;
     1234 +
1183 1235          /*
1184 1236           * Set the boot time.  If the server
1185 1237           * has been restarted quickly and has had the opportunity to
1186 1238           * service clients, then the start_time needs to be bumped
1187 1239           * regardless.  A small window but it exists...
1188 1240           */
1189 1241          start_time = gethrestime_sec();
1190      -        if (rfs4_start_time < start_time)
1191      -                rfs4_start_time = start_time;
     1242 +        if (nsrv4->rfs4_start_time < start_time)
     1243 +                nsrv4->rfs4_start_time = start_time;
1192 1244          else
1193      -                rfs4_start_time++;
     1245 +                nsrv4->rfs4_start_time++;
1194 1246  
1195      -        /* DSS: distributed stable storage: initialise served paths list */
1196      -        rfs4_dss_pathlist = NULL;
1197      -
1198 1247          /*
1199 1248           * Create the first server instance, or a new one if the server has
1200 1249           * been restarted; see above comments on rfs4_start_time. Don't
1201 1250           * start its grace period; that will be done later, to maximise the
1202 1251           * clients' recovery window.
1203 1252           */
1204 1253          start_grace = 0;
1205      -        rfs4_servinst_create(start_grace, 1, &dss_path);
     1254 +        rfs4_servinst_create(nsrv4, start_grace, 1, &dss_path);
1206 1255  
1207 1256          /* reset the "first NFSv4 request" status */
1208      -        rfs4_seen_first_compound = 0;
     1257 +        nsrv4->seen_first_compound = 0;
1209 1258  
     1259 +        mutex_enter(&nsrv4->state_lock);
     1260 +
1210 1261          /*
1211      -         * Add a CPR callback so that we can update client
1212      -         * access times to extend the lease after a suspend
1213      -         * and resume (using the same class as rpcmod/connmgr)
     1262 +         * If the server state database has already been initialized,
     1263 +         * skip it
1214 1264           */
1215      -        cpr_id = callb_add(rfs4_cpr_callb, 0, CB_CL_CPR_RPC, "rfs4");
     1265 +        if (nsrv4->nfs4_server_state != NULL) {
     1266 +                mutex_exit(&nsrv4->state_lock);
     1267 +                return;
     1268 +        }
1216 1269  
     1270 +        rw_init(&nsrv4->rfs4_findclient_lock, NULL, RW_DEFAULT, NULL);
     1271 +
1217 1272          /* set the various cache timers for table creation */
1218      -        if (rfs4_client_cache_time == 0)
1219      -                rfs4_client_cache_time = CLIENT_CACHE_TIME;
1220      -        if (rfs4_openowner_cache_time == 0)
1221      -                rfs4_openowner_cache_time = OPENOWNER_CACHE_TIME;
1222      -        if (rfs4_state_cache_time == 0)
1223      -                rfs4_state_cache_time = STATE_CACHE_TIME;
1224      -        if (rfs4_lo_state_cache_time == 0)
1225      -                rfs4_lo_state_cache_time = LO_STATE_CACHE_TIME;
1226      -        if (rfs4_lockowner_cache_time == 0)
1227      -                rfs4_lockowner_cache_time = LOCKOWNER_CACHE_TIME;
1228      -        if (rfs4_file_cache_time == 0)
1229      -                rfs4_file_cache_time = FILE_CACHE_TIME;
1230      -        if (rfs4_deleg_state_cache_time == 0)
1231      -                rfs4_deleg_state_cache_time = DELEG_STATE_CACHE_TIME;
     1273 +        if (nsrv4->rfs4_client_cache_time == 0)
     1274 +                nsrv4->rfs4_client_cache_time = CLIENT_CACHE_TIME;
     1275 +        if (nsrv4->rfs4_openowner_cache_time == 0)
     1276 +                nsrv4->rfs4_openowner_cache_time = OPENOWNER_CACHE_TIME;
     1277 +        if (nsrv4->rfs4_state_cache_time == 0)
     1278 +                nsrv4->rfs4_state_cache_time = STATE_CACHE_TIME;
     1279 +        if (nsrv4->rfs4_lo_state_cache_time == 0)
     1280 +                nsrv4->rfs4_lo_state_cache_time = LO_STATE_CACHE_TIME;
     1281 +        if (nsrv4->rfs4_lockowner_cache_time == 0)
     1282 +                nsrv4->rfs4_lockowner_cache_time = LOCKOWNER_CACHE_TIME;
     1283 +        if (nsrv4->rfs4_file_cache_time == 0)
     1284 +                nsrv4->rfs4_file_cache_time = FILE_CACHE_TIME;
     1285 +        if (nsrv4->rfs4_deleg_state_cache_time == 0)
     1286 +                nsrv4->rfs4_deleg_state_cache_time = DELEG_STATE_CACHE_TIME;
1232 1287  
1233 1288          /* Create the overall database to hold all server state */
1234      -        rfs4_server_state = rfs4_database_create(rfs4_database_debug);
     1289 +        nsrv4->nfs4_server_state = rfs4_database_create(rfs4_database_debug);
1235 1290  
1236 1291          /* Now create the individual tables */
1237      -        rfs4_client_cache_time *= rfs4_lease_time;
1238      -        rfs4_client_tab = rfs4_table_create(rfs4_server_state,
     1292 +        nsrv4->rfs4_client_cache_time *= rfs4_lease_time;
     1293 +        nsrv4->rfs4_client_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1239 1294              "Client",
1240      -            rfs4_client_cache_time,
     1295 +            nsrv4->rfs4_client_cache_time,
1241 1296              2,
1242 1297              rfs4_client_create,
1243 1298              rfs4_client_destroy,
1244 1299              rfs4_client_expiry,
1245 1300              sizeof (rfs4_client_t),
1246 1301              TABSIZE,
1247 1302              MAXTABSZ/8, 100);
1248      -        rfs4_nfsclnt_idx = rfs4_index_create(rfs4_client_tab,
     1303 +        nsrv4->rfs4_nfsclnt_idx = rfs4_index_create(nsrv4->rfs4_client_tab,
1249 1304              "nfs_client_id4", nfsclnt_hash,
1250 1305              nfsclnt_compare, nfsclnt_mkkey,
1251 1306              TRUE);
1252      -        rfs4_clientid_idx = rfs4_index_create(rfs4_client_tab,
     1307 +        nsrv4->rfs4_clientid_idx = rfs4_index_create(nsrv4->rfs4_client_tab,
1253 1308              "client_id", clientid_hash,
1254 1309              clientid_compare, clientid_mkkey,
1255 1310              FALSE);
1256 1311  
1257      -        rfs4_clntip_cache_time = 86400 * 365;   /* about a year */
1258      -        rfs4_clntip_tab = rfs4_table_create(rfs4_server_state,
     1312 +        nsrv4->rfs4_clntip_cache_time = 86400 * 365;    /* about a year */
     1313 +        nsrv4->rfs4_clntip_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1259 1314              "ClntIP",
1260      -            rfs4_clntip_cache_time,
     1315 +            nsrv4->rfs4_clntip_cache_time,
1261 1316              1,
1262 1317              rfs4_clntip_create,
1263 1318              rfs4_clntip_destroy,
1264 1319              rfs4_clntip_expiry,
1265 1320              sizeof (rfs4_clntip_t),
1266 1321              TABSIZE,
1267 1322              MAXTABSZ, 100);
1268      -        rfs4_clntip_idx = rfs4_index_create(rfs4_clntip_tab,
     1323 +        nsrv4->rfs4_clntip_idx = rfs4_index_create(nsrv4->rfs4_clntip_tab,
1269 1324              "client_ip", clntip_hash,
1270 1325              clntip_compare, clntip_mkkey,
1271 1326              TRUE);
1272 1327  
1273      -        rfs4_openowner_cache_time *= rfs4_lease_time;
1274      -        rfs4_openowner_tab = rfs4_table_create(rfs4_server_state,
     1328 +        nsrv4->rfs4_openowner_cache_time *= rfs4_lease_time;
     1329 +        nsrv4->rfs4_openowner_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1275 1330              "OpenOwner",
1276      -            rfs4_openowner_cache_time,
     1331 +            nsrv4->rfs4_openowner_cache_time,
1277 1332              1,
1278 1333              rfs4_openowner_create,
1279 1334              rfs4_openowner_destroy,
1280 1335              rfs4_openowner_expiry,
1281 1336              sizeof (rfs4_openowner_t),
1282 1337              TABSIZE,
1283 1338              MAXTABSZ, 100);
1284      -        rfs4_openowner_idx = rfs4_index_create(rfs4_openowner_tab,
     1339 +        nsrv4->rfs4_openowner_idx = rfs4_index_create(nsrv4->rfs4_openowner_tab,
1285 1340              "open_owner4", openowner_hash,
1286 1341              openowner_compare,
1287 1342              openowner_mkkey, TRUE);
1288 1343  
1289      -        rfs4_state_cache_time *= rfs4_lease_time;
1290      -        rfs4_state_tab = rfs4_table_create(rfs4_server_state,
     1344 +        nsrv4->rfs4_state_cache_time *= rfs4_lease_time;
     1345 +        nsrv4->rfs4_state_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1291 1346              "OpenStateID",
1292      -            rfs4_state_cache_time,
     1347 +            nsrv4->rfs4_state_cache_time,
1293 1348              3,
1294 1349              rfs4_state_create,
1295 1350              rfs4_state_destroy,
1296 1351              rfs4_state_expiry,
1297 1352              sizeof (rfs4_state_t),
1298 1353              TABSIZE,
1299 1354              MAXTABSZ, 100);
1300 1355  
1301      -        rfs4_state_owner_file_idx = rfs4_index_create(rfs4_state_tab,
     1356 +        /* CSTYLED */
     1357 +        nsrv4->rfs4_state_owner_file_idx = rfs4_index_create(nsrv4->rfs4_state_tab,
1302 1358              "Openowner-File",
1303 1359              state_owner_file_hash,
1304 1360              state_owner_file_compare,
1305 1361              state_owner_file_mkkey, TRUE);
1306 1362  
1307      -        rfs4_state_idx = rfs4_index_create(rfs4_state_tab,
     1363 +        nsrv4->rfs4_state_idx = rfs4_index_create(nsrv4->rfs4_state_tab,
1308 1364              "State-id", state_hash,
1309 1365              state_compare, state_mkkey, FALSE);
1310 1366  
1311      -        rfs4_state_file_idx = rfs4_index_create(rfs4_state_tab,
     1367 +        nsrv4->rfs4_state_file_idx = rfs4_index_create(nsrv4->rfs4_state_tab,
1312 1368              "File", state_file_hash,
1313 1369              state_file_compare, state_file_mkkey,
1314 1370              FALSE);
1315 1371  
1316      -        rfs4_lo_state_cache_time *= rfs4_lease_time;
1317      -        rfs4_lo_state_tab = rfs4_table_create(rfs4_server_state,
     1372 +        nsrv4->rfs4_lo_state_cache_time *= rfs4_lease_time;
     1373 +        nsrv4->rfs4_lo_state_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1318 1374              "LockStateID",
1319      -            rfs4_lo_state_cache_time,
     1375 +            nsrv4->rfs4_lo_state_cache_time,
1320 1376              2,
1321 1377              rfs4_lo_state_create,
1322 1378              rfs4_lo_state_destroy,
1323 1379              rfs4_lo_state_expiry,
1324 1380              sizeof (rfs4_lo_state_t),
1325 1381              TABSIZE,
1326 1382              MAXTABSZ, 100);
1327 1383  
1328      -        rfs4_lo_state_owner_idx = rfs4_index_create(rfs4_lo_state_tab,
     1384 +        /* CSTYLED */
     1385 +        nsrv4->rfs4_lo_state_owner_idx = rfs4_index_create(nsrv4->rfs4_lo_state_tab,
1329 1386              "lockownerxstate",
1330 1387              lo_state_lo_hash,
1331 1388              lo_state_lo_compare,
1332 1389              lo_state_lo_mkkey, TRUE);
1333 1390  
1334      -        rfs4_lo_state_idx = rfs4_index_create(rfs4_lo_state_tab,
     1391 +        nsrv4->rfs4_lo_state_idx = rfs4_index_create(nsrv4->rfs4_lo_state_tab,
1335 1392              "State-id",
1336 1393              lo_state_hash, lo_state_compare,
1337 1394              lo_state_mkkey, FALSE);
1338 1395  
1339      -        rfs4_lockowner_cache_time *= rfs4_lease_time;
     1396 +        nsrv4->rfs4_lockowner_cache_time *= rfs4_lease_time;
1340 1397  
1341      -        rfs4_lockowner_tab = rfs4_table_create(rfs4_server_state,
     1398 +        nsrv4->rfs4_lockowner_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1342 1399              "Lockowner",
1343      -            rfs4_lockowner_cache_time,
     1400 +            nsrv4->rfs4_lockowner_cache_time,
1344 1401              2,
1345 1402              rfs4_lockowner_create,
1346 1403              rfs4_lockowner_destroy,
1347 1404              rfs4_lockowner_expiry,
1348 1405              sizeof (rfs4_lockowner_t),
1349 1406              TABSIZE,
1350 1407              MAXTABSZ, 100);
1351 1408  
1352      -        rfs4_lockowner_idx = rfs4_index_create(rfs4_lockowner_tab,
     1409 +        nsrv4->rfs4_lockowner_idx = rfs4_index_create(nsrv4->rfs4_lockowner_tab,
1353 1410              "lock_owner4", lockowner_hash,
1354 1411              lockowner_compare,
1355 1412              lockowner_mkkey, TRUE);
1356 1413  
1357      -        rfs4_lockowner_pid_idx = rfs4_index_create(rfs4_lockowner_tab,
     1414 +        /* CSTYLED */
     1415 +        nsrv4->rfs4_lockowner_pid_idx = rfs4_index_create(nsrv4->rfs4_lockowner_tab,
1358 1416              "pid", pid_hash,
1359 1417              pid_compare, pid_mkkey,
1360 1418              FALSE);
1361 1419  
1362      -        rfs4_file_cache_time *= rfs4_lease_time;
1363      -        rfs4_file_tab = rfs4_table_create(rfs4_server_state,
     1420 +        nsrv4->rfs4_file_cache_time *= rfs4_lease_time;
     1421 +        nsrv4->rfs4_file_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1364 1422              "File",
1365      -            rfs4_file_cache_time,
     1423 +            nsrv4->rfs4_file_cache_time,
1366 1424              1,
1367 1425              rfs4_file_create,
1368 1426              rfs4_file_destroy,
1369 1427              NULL,
1370 1428              sizeof (rfs4_file_t),
1371 1429              TABSIZE,
1372 1430              MAXTABSZ, -1);
1373 1431  
1374      -        rfs4_file_idx = rfs4_index_create(rfs4_file_tab,
     1432 +        nsrv4->rfs4_file_idx = rfs4_index_create(nsrv4->rfs4_file_tab,
1375 1433              "Filehandle", file_hash,
1376 1434              file_compare, file_mkkey, TRUE);
1377 1435  
1378      -        rfs4_deleg_state_cache_time *= rfs4_lease_time;
1379      -        rfs4_deleg_state_tab = rfs4_table_create(rfs4_server_state,
     1436 +        nsrv4->rfs4_deleg_state_cache_time *= rfs4_lease_time;
     1437 +        /* CSTYLED */
     1438 +        nsrv4->rfs4_deleg_state_tab = rfs4_table_create(nsrv4->nfs4_server_state,
1380 1439              "DelegStateID",
1381      -            rfs4_deleg_state_cache_time,
     1440 +            nsrv4->rfs4_deleg_state_cache_time,
1382 1441              2,
1383 1442              rfs4_deleg_state_create,
1384 1443              rfs4_deleg_state_destroy,
1385 1444              rfs4_deleg_state_expiry,
1386 1445              sizeof (rfs4_deleg_state_t),
1387 1446              TABSIZE,
1388 1447              MAXTABSZ, 100);
1389      -        rfs4_deleg_idx = rfs4_index_create(rfs4_deleg_state_tab,
     1448 +        nsrv4->rfs4_deleg_idx = rfs4_index_create(nsrv4->rfs4_deleg_state_tab,
1390 1449              "DelegByFileClient",
1391 1450              deleg_hash,
1392 1451              deleg_compare,
1393 1452              deleg_mkkey, TRUE);
1394 1453  
1395      -        rfs4_deleg_state_idx = rfs4_index_create(rfs4_deleg_state_tab,
     1454 +        /* CSTYLED */
     1455 +        nsrv4->rfs4_deleg_state_idx = rfs4_index_create(nsrv4->rfs4_deleg_state_tab,
1396 1456              "DelegState",
1397 1457              deleg_state_hash,
1398 1458              deleg_state_compare,
1399 1459              deleg_state_mkkey, FALSE);
1400 1460  
     1461 +        mutex_exit(&nsrv4->state_lock);
     1462 +
1401 1463          /*
1402 1464           * Init the stable storage.
1403 1465           */
1404      -        rfs4_ss_init();
1405      -
1406      -        rfs4_client_clrst = rfs4_clear_client_state;
1407      -
1408      -        mutex_exit(&rfs4_state_lock);
     1466 +        rfs4_ss_init(nsrv4);
1409 1467  }
1410 1468  
1411      -
1412 1469  /*
1413      - * Used at server shutdown to cleanup all of the NFSv4 server's structures
1414      - * and other state.
     1470 + * Used at server shutdown to cleanup all of NFSv4 server's zone structures
     1471 + * and state.
1415 1472   */
1416 1473  void
1417      -rfs4_state_fini()
     1474 +rfs4_state_zone_fini()
1418 1475  {
1419 1476          rfs4_database_t *dbp;
     1477 +        nfs4_srv_t *nsrv4;
     1478 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
1420 1479  
1421      -        mutex_enter(&rfs4_state_lock);
     1480 +        rfs4_set_deleg_policy(nsrv4, SRV_NEVER_DELEGATE);
1422 1481  
1423      -        if (rfs4_server_state == NULL) {
1424      -                mutex_exit(&rfs4_state_lock);
     1482 +        mutex_enter(&nsrv4->state_lock);
     1483 +
     1484 +        if (nsrv4->nfs4_server_state == NULL) {
     1485 +                mutex_exit(&nsrv4->state_lock);
1425 1486                  return;
1426 1487          }
1427 1488  
1428      -        rfs4_client_clrst = NULL;
     1489 +        /* destroy server instances and current instance ptr */
     1490 +        rfs4_servinst_destroy_all(nsrv4);
1429 1491  
1430      -        rfs4_set_deleg_policy(SRV_NEVER_DELEGATE);
1431      -        dbp = rfs4_server_state;
1432      -        rfs4_server_state = NULL;
     1492 +        /* reset the "first NFSv4 request" status */
     1493 +        nsrv4->seen_first_compound = 0;
1433 1494  
1434      -        /*
1435      -         * Cleanup the CPR callback.
1436      -         */
1437      -        if (cpr_id)
1438      -                (void) callb_delete(cpr_id);
     1495 +        dbp = nsrv4->nfs4_server_state;
     1496 +        nsrv4->nfs4_server_state = NULL;
1439 1497  
1440      -        rw_destroy(&rfs4_findclient_lock);
     1498 +        rw_destroy(&nsrv4->rfs4_findclient_lock);
1441 1499  
1442 1500          /* First stop all of the reaper threads in the database */
1443 1501          rfs4_database_shutdown(dbp);
1444      -        /* clean up any dangling stable storage structures */
1445      -        rfs4_ss_fini();
1446      -        /* Now actually destroy/release the database and its tables */
1447      -        rfs4_database_destroy(dbp);
     1502 +        /*
     1503 +         * XXX workaround
     1504 +         * Skip destrying the state database yet just in case there
     1505 +         * are unfinished operations depending on it.
     1506 +         */
     1507 +        /* Now destroy/release the database tables */
     1508 +        /* rfs4_database_destroy(dbp); */
1448 1509  
1449 1510          /* Reset the cache timers for next time */
1450      -        rfs4_client_cache_time = 0;
1451      -        rfs4_openowner_cache_time = 0;
1452      -        rfs4_state_cache_time = 0;
1453      -        rfs4_lo_state_cache_time = 0;
1454      -        rfs4_lockowner_cache_time = 0;
1455      -        rfs4_file_cache_time = 0;
1456      -        rfs4_deleg_state_cache_time = 0;
     1511 +        nsrv4->rfs4_client_cache_time = 0;
     1512 +        nsrv4->rfs4_openowner_cache_time = 0;
     1513 +        nsrv4->rfs4_state_cache_time = 0;
     1514 +        nsrv4->rfs4_lo_state_cache_time = 0;
     1515 +        nsrv4->rfs4_lockowner_cache_time = 0;
     1516 +        nsrv4->rfs4_file_cache_time = 0;
     1517 +        nsrv4->rfs4_deleg_state_cache_time = 0;
1457 1518  
1458      -        mutex_exit(&rfs4_state_lock);
     1519 +        mutex_exit(&nsrv4->state_lock);
1459 1520  
1460      -        /* destroy server instances and current instance ptr */
1461      -        rfs4_servinst_destroy_all();
1462      -
1463      -        /* reset the "first NFSv4 request" status */
1464      -        rfs4_seen_first_compound = 0;
1465      -
1466      -        /* DSS: distributed stable storage */
1467      -        nvlist_free(rfs4_dss_oldpaths);
1468      -        nvlist_free(rfs4_dss_paths);
1469      -        rfs4_dss_paths = rfs4_dss_oldpaths = NULL;
     1521 +        /* clean up any dangling stable storage structures */
     1522 +        rfs4_ss_fini(nsrv4);
1470 1523  }
1471 1524  
1472 1525  typedef union {
1473 1526          struct {
1474 1527                  uint32_t start_time;
1475 1528                  uint32_t c_id;
1476 1529          } impl_id;
1477 1530          clientid4 id4;
1478 1531  } cid;
1479 1532  
↓ open down ↓ 94 lines elided ↑ open up ↑
1574 1627                  cp->rc_ss_remove = 1;
1575 1628          return (cp_expired);
1576 1629  }
1577 1630  
1578 1631  /*
1579 1632   * Remove the leaf file from all distributed stable storage paths.
1580 1633   */
1581 1634  static void
1582 1635  rfs4_dss_remove_cpleaf(rfs4_client_t *cp)
1583 1636  {
     1637 +        nfs4_srv_t *nsrv4;
1584 1638          rfs4_servinst_t *sip;
1585 1639          char *leaf = cp->rc_ss_pn->leaf;
1586 1640  
1587 1641          /*
1588 1642           * since the state files are written to all DSS
1589 1643           * paths we must remove this leaf file instance
1590 1644           * from all server instances.
1591 1645           */
1592 1646  
1593      -        mutex_enter(&rfs4_servinst_lock);
1594      -        for (sip = rfs4_cur_servinst; sip != NULL; sip = sip->prev) {
     1647 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     1648 +        mutex_enter(&nsrv4->servinst_lock);
     1649 +        for (sip = nsrv4->nfs4_cur_servinst; sip != NULL; sip = sip->prev) {
1595 1650                  /* remove the leaf file associated with this server instance */
1596 1651                  rfs4_dss_remove_leaf(sip, NFS4_DSS_STATE_LEAF, leaf);
1597 1652          }
1598      -        mutex_exit(&rfs4_servinst_lock);
     1653 +        mutex_exit(&nsrv4->servinst_lock);
1599 1654  }
1600 1655  
1601 1656  static void
1602 1657  rfs4_dss_remove_leaf(rfs4_servinst_t *sip, char *dir_leaf, char *leaf)
1603 1658  {
1604 1659          int i, npaths = sip->dss_npaths;
1605 1660  
1606 1661          for (i = 0; i < npaths; i++) {
1607 1662                  rfs4_dss_path_t *dss_path = sip->dss_paths[i];
1608 1663                  char *path, *dir;
↓ open down ↓ 47 lines elided ↑ open up ↑
1656 1711  }
1657 1712  
1658 1713  static bool_t
1659 1714  rfs4_client_create(rfs4_entry_t u_entry, void *arg)
1660 1715  {
1661 1716          rfs4_client_t *cp = (rfs4_client_t *)u_entry;
1662 1717          nfs_client_id4 *client = (nfs_client_id4 *)arg;
1663 1718          struct sockaddr *ca;
1664 1719          cid *cidp;
1665 1720          scid_confirm_verf *scvp;
     1721 +        nfs4_srv_t *nsrv4;
1666 1722  
     1723 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     1724 +
1667 1725          /* Get a clientid to give to the client */
1668 1726          cidp = (cid *)&cp->rc_clientid;
1669      -        cidp->impl_id.start_time = rfs4_start_time;
     1727 +        cidp->impl_id.start_time = nsrv4->rfs4_start_time;
1670 1728          cidp->impl_id.c_id = (uint32_t)rfs4_dbe_getid(cp->rc_dbe);
1671 1729  
1672 1730          /* If we are booted as a cluster node, embed our nodeid */
1673 1731          if (cluster_bootflags & CLUSTER_BOOTED)
1674 1732                  embed_nodeid(cidp);
1675 1733  
1676 1734          /* Allocate and copy client's client id value */
1677 1735          cp->rc_nfs_client.id_val = kmem_alloc(client->id_len, KM_SLEEP);
1678 1736          cp->rc_nfs_client.id_len = client->id_len;
1679 1737          bcopy(client->id_val, cp->rc_nfs_client.id_val, client->id_len);
↓ open down ↓ 37 lines elided ↑ open up ↑
1717 1775          mutex_init(cp->rc_cbinfo.cb_lock, NULL, MUTEX_DEFAULT, NULL);
1718 1776          cv_init(cp->rc_cbinfo.cb_cv, NULL, CV_DEFAULT, NULL);
1719 1777          cv_init(cp->rc_cbinfo.cb_cv_nullcaller, NULL, CV_DEFAULT, NULL);
1720 1778  
1721 1779          /*
1722 1780           * Associate the client_t with the current server instance.
1723 1781           * The hold is solely to satisfy the calling requirement of
1724 1782           * rfs4_servinst_assign(). In this case it's not strictly necessary.
1725 1783           */
1726 1784          rfs4_dbe_hold(cp->rc_dbe);
1727      -        rfs4_servinst_assign(cp, rfs4_cur_servinst);
     1785 +        rfs4_servinst_assign(nsrv4, cp, nsrv4->nfs4_cur_servinst);
1728 1786          rfs4_dbe_rele(cp->rc_dbe);
1729 1787  
1730 1788          return (TRUE);
1731 1789  }
1732 1790  
1733 1791  /*
1734 1792   * Caller wants to generate/update the setclientid_confirm verifier
1735 1793   * associated with a client.  This is done during the SETCLIENTID
1736 1794   * processing.
1737 1795   */
↓ open down ↓ 10 lines elided ↑ open up ↑
1748 1806  void
1749 1807  rfs4_client_rele(rfs4_client_t *cp)
1750 1808  {
1751 1809          rfs4_dbe_rele(cp->rc_dbe);
1752 1810  }
1753 1811  
1754 1812  rfs4_client_t *
1755 1813  rfs4_findclient(nfs_client_id4 *client, bool_t *create, rfs4_client_t *oldcp)
1756 1814  {
1757 1815          rfs4_client_t *cp;
     1816 +        nfs4_srv_t *nsrv4;
     1817 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
1758 1818  
1759 1819  
1760 1820          if (oldcp) {
1761      -                rw_enter(&rfs4_findclient_lock, RW_WRITER);
     1821 +                rw_enter(&nsrv4->rfs4_findclient_lock, RW_WRITER);
1762 1822                  rfs4_dbe_hide(oldcp->rc_dbe);
1763 1823          } else {
1764      -                rw_enter(&rfs4_findclient_lock, RW_READER);
     1824 +                rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
1765 1825          }
1766 1826  
1767      -        cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_nfsclnt_idx, client,
     1827 +        cp = (rfs4_client_t *)rfs4_dbsearch(nsrv4->rfs4_nfsclnt_idx, client,
1768 1828              create, (void *)client, RFS4_DBS_VALID);
1769 1829  
1770 1830          if (oldcp)
1771 1831                  rfs4_dbe_unhide(oldcp->rc_dbe);
1772 1832  
1773      -        rw_exit(&rfs4_findclient_lock);
     1833 +        rw_exit(&nsrv4->rfs4_findclient_lock);
1774 1834  
1775 1835          return (cp);
1776 1836  }
1777 1837  
1778 1838  rfs4_client_t *
1779 1839  rfs4_findclient_by_id(clientid4 clientid, bool_t find_unconfirmed)
1780 1840  {
1781 1841          rfs4_client_t *cp;
1782 1842          bool_t create = FALSE;
1783 1843          cid *cidp = (cid *)&clientid;
     1844 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
1784 1845  
1785 1846          /* If we're a cluster and the nodeid isn't right, short-circuit */
1786 1847          if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp))
1787 1848                  return (NULL);
1788 1849  
1789      -        rw_enter(&rfs4_findclient_lock, RW_READER);
     1850 +        rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
1790 1851  
1791      -        cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx, &clientid,
     1852 +        cp = (rfs4_client_t *)rfs4_dbsearch(nsrv4->rfs4_clientid_idx, &clientid,
1792 1853              &create, NULL, RFS4_DBS_VALID);
1793 1854  
1794      -        rw_exit(&rfs4_findclient_lock);
     1855 +        rw_exit(&nsrv4->rfs4_findclient_lock);
1795 1856  
1796 1857          if (cp && cp->rc_need_confirm && find_unconfirmed == FALSE) {
1797 1858                  rfs4_client_rele(cp);
1798 1859                  return (NULL);
1799 1860          } else {
1800 1861                  return (cp);
1801 1862          }
1802 1863  }
1803 1864  
1804 1865  static uint32_t
↓ open down ↓ 87 lines elided ↑ open up ↑
1892 1953                  return (FALSE);
1893 1954          cp->ri_no_referrals = 1;
1894 1955  
1895 1956          return (TRUE);
1896 1957  }
1897 1958  
1898 1959  rfs4_clntip_t *
1899 1960  rfs4_find_clntip(struct sockaddr *addr, bool_t *create)
1900 1961  {
1901 1962          rfs4_clntip_t *cp;
     1963 +        nfs4_srv_t *nsrv4;
1902 1964  
1903      -        rw_enter(&rfs4_findclient_lock, RW_READER);
     1965 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
1904 1966  
1905      -        cp = (rfs4_clntip_t *)rfs4_dbsearch(rfs4_clntip_idx, addr,
     1967 +        rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
     1968 +
     1969 +        cp = (rfs4_clntip_t *)rfs4_dbsearch(nsrv4->rfs4_clntip_idx, addr,
1906 1970              create, addr, RFS4_DBS_VALID);
1907 1971  
1908      -        rw_exit(&rfs4_findclient_lock);
     1972 +        rw_exit(&nsrv4->rfs4_findclient_lock);
1909 1973  
1910 1974          return (cp);
1911 1975  }
1912 1976  
1913 1977  void
1914 1978  rfs4_invalidate_clntip(struct sockaddr *addr)
1915 1979  {
1916 1980          rfs4_clntip_t *cp;
1917 1981          bool_t create = FALSE;
     1982 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
1918 1983  
1919      -        rw_enter(&rfs4_findclient_lock, RW_READER);
     1984 +        rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
1920 1985  
1921      -        cp = (rfs4_clntip_t *)rfs4_dbsearch(rfs4_clntip_idx, addr,
     1986 +        cp = (rfs4_clntip_t *)rfs4_dbsearch(nsrv4->rfs4_clntip_idx, addr,
1922 1987              &create, NULL, RFS4_DBS_VALID);
1923 1988          if (cp == NULL) {
1924      -                rw_exit(&rfs4_findclient_lock);
     1989 +                rw_exit(&nsrv4->rfs4_findclient_lock);
1925 1990                  return;
1926 1991          }
1927 1992          rfs4_dbe_invalidate(cp->ri_dbe);
1928 1993          rfs4_dbe_rele(cp->ri_dbe);
1929 1994  
1930      -        rw_exit(&rfs4_findclient_lock);
     1995 +        rw_exit(&nsrv4->rfs4_findclient_lock);
1931 1996  }
1932 1997  
1933 1998  bool_t
1934 1999  rfs4_lease_expired(rfs4_client_t *cp)
1935 2000  {
1936 2001          bool_t rc;
1937 2002  
1938 2003          rfs4_dbe_lock(cp->rc_dbe);
1939 2004  
1940 2005          /*
↓ open down ↓ 127 lines elided ↑ open up ↑
2068 2133  
2069 2134  static bool_t
2070 2135  rfs4_openowner_create(rfs4_entry_t u_entry, void *arg)
2071 2136  {
2072 2137          rfs4_openowner_t *oo = (rfs4_openowner_t *)u_entry;
2073 2138          rfs4_openowner_t *argp = (rfs4_openowner_t *)arg;
2074 2139          open_owner4 *openowner = &argp->ro_owner;
2075 2140          seqid4 seqid = argp->ro_open_seqid;
2076 2141          rfs4_client_t *cp;
2077 2142          bool_t create = FALSE;
     2143 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2078 2144  
2079      -        rw_enter(&rfs4_findclient_lock, RW_READER);
     2145 +        rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
2080 2146  
2081      -        cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx,
     2147 +        cp = (rfs4_client_t *)rfs4_dbsearch(nsrv4->rfs4_clientid_idx,
2082 2148              &openowner->clientid,
2083 2149              &create, NULL, RFS4_DBS_VALID);
2084 2150  
2085      -        rw_exit(&rfs4_findclient_lock);
     2151 +        rw_exit(&nsrv4->rfs4_findclient_lock);
2086 2152  
2087 2153          if (cp == NULL)
2088 2154                  return (FALSE);
2089 2155  
2090 2156          oo->ro_reply_fh.nfs_fh4_len = 0;
2091 2157          oo->ro_reply_fh.nfs_fh4_val = NULL;
2092 2158  
2093 2159          oo->ro_owner.clientid = openowner->clientid;
2094 2160          oo->ro_owner.owner_val =
2095 2161              kmem_alloc(openowner->owner_len, KM_SLEEP);
↓ open down ↓ 21 lines elided ↑ open up ↑
2117 2183          rfs4_dbe_unlock(cp->rc_dbe);
2118 2184  
2119 2185          return (TRUE);
2120 2186  }
2121 2187  
2122 2188  rfs4_openowner_t *
2123 2189  rfs4_findopenowner(open_owner4 *openowner, bool_t *create, seqid4 seqid)
2124 2190  {
2125 2191          rfs4_openowner_t *oo;
2126 2192          rfs4_openowner_t arg;
     2193 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2127 2194  
2128 2195          arg.ro_owner = *openowner;
2129 2196          arg.ro_open_seqid = seqid;
2130      -        oo = (rfs4_openowner_t *)rfs4_dbsearch(rfs4_openowner_idx, openowner,
     2197 +        /* CSTYLED */
     2198 +        oo = (rfs4_openowner_t *)rfs4_dbsearch(nsrv4->rfs4_openowner_idx, openowner,
2131 2199              create, &arg, RFS4_DBS_VALID);
2132 2200  
2133 2201          return (oo);
2134 2202  }
2135 2203  
2136 2204  void
2137 2205  rfs4_update_open_sequence(rfs4_openowner_t *oo)
2138 2206  {
2139 2207  
2140 2208          rfs4_dbe_lock(oo->ro_dbe);
↓ open down ↓ 122 lines elided ↑ open up ↑
2263 2331          return (TRUE);
2264 2332  }
2265 2333  
2266 2334  static bool_t
2267 2335  rfs4_lockowner_create(rfs4_entry_t u_entry, void *arg)
2268 2336  {
2269 2337          rfs4_lockowner_t *lo = (rfs4_lockowner_t *)u_entry;
2270 2338          lock_owner4 *lockowner = (lock_owner4 *)arg;
2271 2339          rfs4_client_t *cp;
2272 2340          bool_t create = FALSE;
     2341 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2273 2342  
2274      -        rw_enter(&rfs4_findclient_lock, RW_READER);
     2343 +        rw_enter(&nsrv4->rfs4_findclient_lock, RW_READER);
2275 2344  
2276      -        cp = (rfs4_client_t *)rfs4_dbsearch(rfs4_clientid_idx,
     2345 +        cp = (rfs4_client_t *)rfs4_dbsearch(nsrv4->rfs4_clientid_idx,
2277 2346              &lockowner->clientid,
2278 2347              &create, NULL, RFS4_DBS_VALID);
2279 2348  
2280      -        rw_exit(&rfs4_findclient_lock);
     2349 +        rw_exit(&nsrv4->rfs4_findclient_lock);
2281 2350  
2282 2351          if (cp == NULL)
2283 2352                  return (FALSE);
2284 2353  
2285 2354          /* Reference client */
2286 2355          lo->rl_client = cp;
2287 2356          lo->rl_owner.clientid = lockowner->clientid;
2288 2357          lo->rl_owner.owner_val = kmem_alloc(lockowner->owner_len, KM_SLEEP);
2289 2358          bcopy(lockowner->owner_val, lo->rl_owner.owner_val,
2290 2359              lockowner->owner_len);
2291 2360          lo->rl_owner.owner_len = lockowner->owner_len;
2292 2361          lo->rl_pid = rfs4_dbe_getid(lo->rl_dbe);
2293 2362  
2294 2363          return (TRUE);
2295 2364  }
2296 2365  
2297 2366  rfs4_lockowner_t *
2298 2367  rfs4_findlockowner(lock_owner4 *lockowner, bool_t *create)
2299 2368  {
2300 2369          rfs4_lockowner_t *lo;
     2370 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2301 2371  
2302      -        lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_idx, lockowner,
     2372 +        /* CSTYLED */
     2373 +        lo = (rfs4_lockowner_t *)rfs4_dbsearch(nsrv4->rfs4_lockowner_idx, lockowner,
2303 2374              create, lockowner, RFS4_DBS_VALID);
2304 2375  
2305 2376          return (lo);
2306 2377  }
2307 2378  
2308 2379  rfs4_lockowner_t *
2309 2380  rfs4_findlockowner_by_pid(pid_t pid)
2310 2381  {
2311 2382          rfs4_lockowner_t *lo;
2312 2383          bool_t create = FALSE;
     2384 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2313 2385  
2314      -        lo = (rfs4_lockowner_t *)rfs4_dbsearch(rfs4_lockowner_pid_idx,
     2386 +        lo = (rfs4_lockowner_t *)rfs4_dbsearch(nsrv4->rfs4_lockowner_pid_idx,
2315 2387              (void *)(uintptr_t)pid, &create, NULL, RFS4_DBS_VALID);
2316 2388  
2317 2389          return (lo);
2318 2390  }
2319 2391  
2320 2392  
2321 2393  static uint32_t
2322 2394  file_hash(void *key)
2323 2395  {
2324 2396          return (ADDRHASH(key));
↓ open down ↓ 90 lines elided ↑ open up ↑
2415 2487          mutex_exit(&vp->v_vsd_lock);
2416 2488  
2417 2489          return (TRUE);
2418 2490  }
2419 2491  
2420 2492  rfs4_file_t *
2421 2493  rfs4_findfile(vnode_t *vp, nfs_fh4 *fh, bool_t *create)
2422 2494  {
2423 2495          rfs4_file_t *fp;
2424 2496          rfs4_fcreate_arg arg;
     2497 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2425 2498  
2426 2499          arg.vp = vp;
2427 2500          arg.fh = fh;
2428 2501  
2429 2502          if (*create == TRUE)
2430      -                fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create,
     2503 +                /* CSTYLED */
     2504 +                fp = (rfs4_file_t *)rfs4_dbsearch(nsrv4->rfs4_file_idx, vp, create,
2431 2505                      &arg, RFS4_DBS_VALID);
2432 2506          else {
2433 2507                  mutex_enter(&vp->v_vsd_lock);
2434 2508                  fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey);
2435 2509                  if (fp) {
2436 2510                          rfs4_dbe_lock(fp->rf_dbe);
2437 2511                          if (rfs4_dbe_is_invalid(fp->rf_dbe) ||
2438 2512                              (rfs4_dbe_refcnt(fp->rf_dbe) == 0)) {
2439 2513                                  rfs4_dbe_unlock(fp->rf_dbe);
2440 2514                                  fp = NULL;
↓ open down ↓ 14 lines elided ↑ open up ↑
2455 2529   * assumes that a file struct that has a NULL vnode pointer is marked
2456 2530   * at 'invalid' and will not be found in the db the second time
2457 2531   * around.
2458 2532   */
2459 2533  rfs4_file_t *
2460 2534  rfs4_findfile_withlock(vnode_t *vp, nfs_fh4 *fh, bool_t *create)
2461 2535  {
2462 2536          rfs4_file_t *fp;
2463 2537          rfs4_fcreate_arg arg;
2464 2538          bool_t screate = *create;
     2539 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2465 2540  
2466 2541          if (screate == FALSE) {
2467 2542                  mutex_enter(&vp->v_vsd_lock);
2468 2543                  fp = (rfs4_file_t *)vsd_get(vp, nfs4_srv_vkey);
2469 2544                  if (fp) {
2470 2545                          rfs4_dbe_lock(fp->rf_dbe);
2471 2546                          if (rfs4_dbe_is_invalid(fp->rf_dbe) ||
2472 2547                              (rfs4_dbe_refcnt(fp->rf_dbe) == 0)) {
2473 2548                                  rfs4_dbe_unlock(fp->rf_dbe);
2474 2549                                  mutex_exit(&vp->v_vsd_lock);
↓ open down ↓ 10 lines elided ↑ open up ↑
2485 2560                                  }
2486 2561                          }
2487 2562                  } else {
2488 2563                          mutex_exit(&vp->v_vsd_lock);
2489 2564                  }
2490 2565          } else {
2491 2566  retry:
2492 2567                  arg.vp = vp;
2493 2568                  arg.fh = fh;
2494 2569  
2495      -                fp = (rfs4_file_t *)rfs4_dbsearch(rfs4_file_idx, vp, create,
2496      -                    &arg, RFS4_DBS_VALID);
     2570 +                fp = (rfs4_file_t *)rfs4_dbsearch(nsrv4->rfs4_file_idx, vp,
     2571 +                    create, &arg, RFS4_DBS_VALID);
2497 2572                  if (fp != NULL) {
2498 2573                          rw_enter(&fp->rf_file_rwlock, RW_WRITER);
2499 2574                          if (fp->rf_vp == NULL) {
2500 2575                                  rw_exit(&fp->rf_file_rwlock);
2501 2576                                  rfs4_file_rele(fp);
2502 2577                                  *create = screate;
2503 2578                                  goto retry;
2504 2579                          }
2505 2580                  }
2506 2581          }
↓ open down ↓ 134 lines elided ↑ open up ↑
2641 2716          if (unlock_fp == TRUE)
2642 2717                  rw_exit(&lsp->rls_state->rs_finfo->rf_file_rwlock);
2643 2718          rfs4_dbe_rele(lsp->rls_dbe);
2644 2719  }
2645 2720  
2646 2721  static rfs4_lo_state_t *
2647 2722  rfs4_findlo_state(stateid_t *id, bool_t lock_fp)
2648 2723  {
2649 2724          rfs4_lo_state_t *lsp;
2650 2725          bool_t create = FALSE;
     2726 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2651 2727  
2652      -        lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_idx, id,
     2728 +        lsp = (rfs4_lo_state_t *)rfs4_dbsearch(nsrv4->rfs4_lo_state_idx, id,
2653 2729              &create, NULL, RFS4_DBS_VALID);
2654 2730          if (lock_fp == TRUE && lsp != NULL)
2655 2731                  rw_enter(&lsp->rls_state->rs_finfo->rf_file_rwlock, RW_READER);
2656 2732  
2657 2733          return (lsp);
2658 2734  }
2659 2735  
2660 2736  
2661 2737  static uint32_t
2662 2738  lo_state_lo_hash(void *key)
↓ open down ↓ 18 lines elided ↑ open up ↑
2681 2757  {
2682 2758          return (u_entry);
2683 2759  }
2684 2760  
2685 2761  rfs4_lo_state_t *
2686 2762  rfs4_findlo_state_by_owner(rfs4_lockowner_t *lo, rfs4_state_t *sp,
2687 2763      bool_t *create)
2688 2764  {
2689 2765          rfs4_lo_state_t *lsp;
2690 2766          rfs4_lo_state_t arg;
     2767 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2691 2768  
2692 2769          arg.rls_locker = lo;
2693 2770          arg.rls_state = sp;
2694 2771  
2695      -        lsp = (rfs4_lo_state_t *)rfs4_dbsearch(rfs4_lo_state_owner_idx, &arg,
2696      -            create, &arg, RFS4_DBS_VALID);
     2772 +        lsp = (rfs4_lo_state_t *)rfs4_dbsearch(nsrv4->rfs4_lo_state_owner_idx,
     2773 +            &arg, create, &arg, RFS4_DBS_VALID);
2697 2774  
2698 2775          return (lsp);
2699 2776  }
2700 2777  
2701 2778  static stateid_t
2702 2779  get_stateid(id_t eid)
2703 2780  {
2704 2781          stateid_t id;
     2782 +        nfs4_srv_t *nsrv4;
2705 2783  
2706      -        id.bits.boottime = rfs4_start_time;
     2784 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     2785 +
     2786 +        id.bits.boottime = nsrv4->rfs4_start_time;
2707 2787          id.bits.ident = eid;
2708 2788          id.bits.chgseq = 0;
2709 2789          id.bits.type = 0;
2710 2790          id.bits.pid = 0;
2711 2791  
2712 2792          /*
2713 2793           * If we are booted as a cluster node, embed our nodeid.
2714 2794           * We've already done sanity checks in rfs4_client_create() so no
2715 2795           * need to repeat them here.
2716 2796           */
↓ open down ↓ 235 lines elided ↑ open up ↑
2952 3032  
2953 3033          /* And now with the openowner */
2954 3034          rfs4_client_rele(dsp->rds_client);
2955 3035          dsp->rds_client = NULL;
2956 3036  }
2957 3037  
2958 3038  rfs4_deleg_state_t *
2959 3039  rfs4_finddeleg(rfs4_state_t *sp, bool_t *create)
2960 3040  {
2961 3041          rfs4_deleg_state_t ds, *dsp;
     3042 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2962 3043  
2963 3044          ds.rds_client = sp->rs_owner->ro_client;
2964 3045          ds.rds_finfo = sp->rs_finfo;
2965 3046  
2966      -        dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_idx, &ds,
     3047 +        dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(nsrv4->rfs4_deleg_idx, &ds,
2967 3048              create, &ds, RFS4_DBS_VALID);
2968 3049  
2969 3050          return (dsp);
2970 3051  }
2971 3052  
2972 3053  rfs4_deleg_state_t *
2973 3054  rfs4_finddelegstate(stateid_t *id)
2974 3055  {
2975 3056          rfs4_deleg_state_t *dsp;
2976 3057          bool_t create = FALSE;
     3058 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
2977 3059  
2978      -        dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(rfs4_deleg_state_idx, id,
2979      -            &create, NULL, RFS4_DBS_VALID);
     3060 +        dsp = (rfs4_deleg_state_t *)rfs4_dbsearch(nsrv4->rfs4_deleg_state_idx,
     3061 +            id, &create, NULL, RFS4_DBS_VALID);
2980 3062  
2981 3063          return (dsp);
2982 3064  }
2983 3065  
2984 3066  void
2985 3067  rfs4_deleg_state_rele(rfs4_deleg_state_t *dsp)
2986 3068  {
2987 3069          rfs4_dbe_rele(dsp->rds_dbe);
2988 3070  }
2989 3071  
↓ open down ↓ 94 lines elided ↑ open up ↑
3084 3166  static void *
3085 3167  state_file_mkkey(rfs4_entry_t u_entry)
3086 3168  {
3087 3169          rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3088 3170  
3089 3171          return (sp->rs_finfo);
3090 3172  }
3091 3173  
3092 3174  rfs4_state_t *
3093 3175  rfs4_findstate_by_owner_file(rfs4_openowner_t *oo, rfs4_file_t *fp,
3094      -        bool_t *create)
     3176 +    bool_t *create)
3095 3177  {
3096 3178          rfs4_state_t *sp;
3097 3179          rfs4_state_t key;
     3180 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
3098 3181  
3099 3182          key.rs_owner = oo;
3100 3183          key.rs_finfo = fp;
3101 3184  
3102      -        sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_owner_file_idx, &key,
3103      -            create, &key, RFS4_DBS_VALID);
     3185 +        sp = (rfs4_state_t *)rfs4_dbsearch(nsrv4->rfs4_state_owner_file_idx,
     3186 +            &key, create, &key, RFS4_DBS_VALID);
3104 3187  
3105 3188          return (sp);
3106 3189  }
3107 3190  
3108 3191  /* This returns ANY state struct that refers to this file */
3109 3192  static rfs4_state_t *
3110 3193  rfs4_findstate_by_file(rfs4_file_t *fp)
3111 3194  {
3112 3195          bool_t create = FALSE;
     3196 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
3113 3197  
3114      -        return ((rfs4_state_t *)rfs4_dbsearch(rfs4_state_file_idx, fp,
     3198 +        return ((rfs4_state_t *)rfs4_dbsearch(nsrv4->rfs4_state_file_idx, fp,
3115 3199              &create, fp, RFS4_DBS_VALID));
3116 3200  }
3117 3201  
3118 3202  static bool_t
3119 3203  rfs4_state_expiry(rfs4_entry_t u_entry)
3120 3204  {
3121 3205          rfs4_state_t *sp = (rfs4_state_t *)u_entry;
3122 3206  
3123 3207          if (rfs4_dbe_is_invalid(sp->rs_dbe))
3124 3208                  return (TRUE);
↓ open down ↓ 30 lines elided ↑ open up ↑
3155 3239          rfs4_dbe_unlock(oo->ro_dbe);
3156 3240  
3157 3241          return (TRUE);
3158 3242  }
3159 3243  
3160 3244  static rfs4_state_t *
3161 3245  rfs4_findstate(stateid_t *id, rfs4_dbsearch_type_t find_invalid, bool_t lock_fp)
3162 3246  {
3163 3247          rfs4_state_t *sp;
3164 3248          bool_t create = FALSE;
     3249 +        nfs4_srv_t *nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
3165 3250  
3166      -        sp = (rfs4_state_t *)rfs4_dbsearch(rfs4_state_idx, id,
     3251 +        sp = (rfs4_state_t *)rfs4_dbsearch(nsrv4->rfs4_state_idx, id,
3167 3252              &create, NULL, find_invalid);
3168 3253          if (lock_fp == TRUE && sp != NULL)
3169 3254                  rw_enter(&sp->rs_finfo->rf_file_rwlock, RW_READER);
3170 3255  
3171 3256          return (sp);
3172 3257  }
3173 3258  
3174 3259  void
3175 3260  rfs4_state_close(rfs4_state_t *sp, bool_t lock_held, bool_t close_of_client,
3176 3261      cred_t *cr)
↓ open down ↓ 47 lines elided ↑ open up ↑
3224 3309          rfs4_client_state_remove(cp);
3225 3310  
3226 3311          /* Release the client */
3227 3312          rfs4_client_rele(cp);
3228 3313  }
3229 3314  
3230 3315  nfsstat4
3231 3316  rfs4_check_clientid(clientid4 *cp, int setclid_confirm)
3232 3317  {
3233 3318          cid *cidp = (cid *) cp;
     3319 +        nfs4_srv_t *nsrv4;
3234 3320  
     3321 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     3322 +
3235 3323          /*
3236 3324           * If we are booted as a cluster node, check the embedded nodeid.
3237 3325           * If it indicates that this clientid was generated on another node,
3238 3326           * inform the client accordingly.
3239 3327           */
3240 3328          if (cluster_bootflags & CLUSTER_BOOTED && foreign_clientid(cidp))
3241 3329                  return (NFS4ERR_STALE_CLIENTID);
3242 3330  
3243 3331          /*
3244 3332           * If the server start time matches the time provided
3245 3333           * by the client (via the clientid) and this is NOT a
3246 3334           * setclientid_confirm then return EXPIRED.
3247 3335           */
3248      -        if (!setclid_confirm && cidp->impl_id.start_time == rfs4_start_time)
     3336 +        if (!setclid_confirm &&
     3337 +            cidp->impl_id.start_time == nsrv4->rfs4_start_time)
3249 3338                  return (NFS4ERR_EXPIRED);
3250 3339  
3251 3340          return (NFS4ERR_STALE_CLIENTID);
3252 3341  }
3253 3342  
3254 3343  /*
3255 3344   * This is used when a stateid has not been found amongst the
3256 3345   * current server's state.  Check the stateid to see if it
3257 3346   * was from this server instantiation or not.
3258 3347   */
3259 3348  static nfsstat4
3260 3349  what_stateid_error(stateid_t *id, stateid_type_t type)
3261 3350  {
     3351 +        nfs4_srv_t *nsrv4;
     3352 +
     3353 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     3354 +
3262 3355          /* If we are booted as a cluster node, was stateid locally generated? */
3263 3356          if ((cluster_bootflags & CLUSTER_BOOTED) && foreign_stateid(id))
3264 3357                  return (NFS4ERR_STALE_STATEID);
3265 3358  
3266 3359          /* If types don't match then no use checking further */
3267 3360          if (type != id->bits.type)
3268 3361                  return (NFS4ERR_BAD_STATEID);
3269 3362  
3270 3363          /* From a different server instantiation, return STALE */
3271      -        if (id->bits.boottime != rfs4_start_time)
     3364 +        if (id->bits.boottime != nsrv4->rfs4_start_time)
3272 3365                  return (NFS4ERR_STALE_STATEID);
3273 3366  
3274 3367          /*
3275 3368           * From this server but the state is most likely beyond lease
3276 3369           * timeout: return NFS4ERR_EXPIRED.  However, there is the
3277 3370           * case of a delegation stateid.  For delegations, there is a
3278 3371           * case where the state can be removed without the client's
3279 3372           * knowledge/consent: revocation.  In the case of delegation
3280 3373           * revocation, the delegation state will be removed and will
3281 3374           * not be found.  If the client does something like a
3282 3375           * DELEGRETURN or even a READ/WRITE with a delegatoin stateid
3283 3376           * that has been revoked, the server should return BAD_STATEID
3284 3377           * instead of the more common EXPIRED error.
3285 3378           */
3286      -        if (id->bits.boottime == rfs4_start_time) {
     3379 +        if (id->bits.boottime == nsrv4->rfs4_start_time) {
3287 3380                  if (type == DELEGID)
3288 3381                          return (NFS4ERR_BAD_STATEID);
3289 3382                  else
3290 3383                          return (NFS4ERR_EXPIRED);
3291 3384          }
3292 3385  
3293 3386          return (NFS4ERR_BAD_STATEID);
3294 3387  }
3295 3388  
3296 3389  /*
↓ open down ↓ 481 lines elided ↑ open up ↑
3778 3871   */
3779 3872  void
3780 3873  rfs4_close_all_state(rfs4_file_t *fp)
3781 3874  {
3782 3875          rfs4_state_t *sp;
3783 3876  
3784 3877          rfs4_dbe_lock(fp->rf_dbe);
3785 3878  
3786 3879  #ifdef DEBUG
3787 3880          /* only applies when server is handing out delegations */
3788      -        if (rfs4_deleg_policy != SRV_NEVER_DELEGATE)
     3881 +        if (nfs4_get_deleg_policy() != SRV_NEVER_DELEGATE)
3789 3882                  ASSERT(fp->rf_dinfo.rd_hold_grant > 0);
3790 3883  #endif
3791 3884  
3792 3885          /* No delegations for this file */
3793 3886          ASSERT(list_is_empty(&fp->rf_delegstatelist));
3794 3887  
3795 3888          /* Make sure that it can not be found */
3796 3889          rfs4_dbe_invalidate(fp->rf_dbe);
3797 3890  
3798 3891          if (fp->rf_vp == NULL) {
↓ open down ↓ 189 lines elided ↑ open up ↑
3988 4081  
3989 4082  /*
3990 4083   * Given a directory that is being unexported, cleanup/release all
3991 4084   * state in the server that refers to objects residing underneath this
3992 4085   * particular export.  The ordering of the release is important.
3993 4086   * Lock_owner, then state and then file.
3994 4087   */
3995 4088  void
3996 4089  rfs4_clean_state_exi(struct exportinfo *exi)
3997 4090  {
3998      -        mutex_enter(&rfs4_state_lock);
     4091 +        nfs4_srv_t *nsrv4;
3999 4092  
4000      -        if (rfs4_server_state == NULL) {
4001      -                mutex_exit(&rfs4_state_lock);
     4093 +        nsrv4 = zone_getspecific(rfs4_zone_key, curzone);
     4094 +        mutex_enter(&nsrv4->state_lock);
     4095 +
     4096 +        if (nsrv4->nfs4_server_state == NULL) {
     4097 +                mutex_exit(&nsrv4->state_lock);
4002 4098                  return;
4003 4099          }
4004 4100  
4005      -        rfs4_dbe_walk(rfs4_lo_state_tab, rfs4_lo_state_walk_callout, exi);
4006      -        rfs4_dbe_walk(rfs4_state_tab, rfs4_state_walk_callout, exi);
4007      -        rfs4_dbe_walk(rfs4_deleg_state_tab, rfs4_deleg_state_walk_callout, exi);
4008      -        rfs4_dbe_walk(rfs4_file_tab, rfs4_file_walk_callout, exi);
     4101 +        /* CSTYLED */
     4102 +        rfs4_dbe_walk(nsrv4->rfs4_lo_state_tab, rfs4_lo_state_walk_callout, exi);
     4103 +        rfs4_dbe_walk(nsrv4->rfs4_state_tab, rfs4_state_walk_callout, exi);
     4104 +        /* CSTYLED */
     4105 +        rfs4_dbe_walk(nsrv4->rfs4_deleg_state_tab, rfs4_deleg_state_walk_callout, exi);
     4106 +        rfs4_dbe_walk(nsrv4->rfs4_file_tab, rfs4_file_walk_callout, exi);
4009 4107  
4010      -        mutex_exit(&rfs4_state_lock);
     4108 +        mutex_exit(&nsrv4->state_lock);
4011 4109  }
    
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX