Print this page 
Fix NFS design problems re. multiple zone keys
Make NFS server zone-specific data all have the same lifetime
Fix rfs4_clean_state_exi
Fix exi_cache_reclaim
Fix mistakes in zone keys work
More fixes re. exi_zoneid and exi_tree
(danmcd -> Keep some ASSERT()s around for readability.)
Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/fs/nfs/nfs_server.c
          +++ new/usr/src/uts/common/fs/nfs/nfs_server.c
↓ open down ↓ 107 lines elided ↑ open up ↑ 
 108  108   */
 109  109  
 110  110  static struct modlmisc modlmisc = {
 111  111          &mod_miscops, "NFS server module"
 112  112  };
 113  113  
 114  114  static struct modlinkage modlinkage = {
 115  115          MODREV_1, (void *)&modlmisc, NULL
 116  116  };
 117  117  
 118      -zone_key_t nfssrv_zone_key;
      118 +zone_key_t      nfssrv_zone_key;
      119 +list_t          nfssrv_globals_list;
      120 +krwlock_t       nfssrv_globals_rwl;
      121 +
 119  122  kmem_cache_t *nfs_xuio_cache;
 120  123  int nfs_loaned_buffers = 0;
 121  124  
 122  125  int
 123  126  _init(void)
 124  127  {
 125  128          int status;
 126  129  
 127  130          nfs_srvinit();
 128  131
↓ open down ↓ 65 lines elided ↑ open up ↑ 
 194  197  static void     acl_dispatch(struct svc_req *, SVCXPRT *);
 195  198  static void     common_dispatch(struct svc_req *, SVCXPRT *,
 196  199                  rpcvers_t, rpcvers_t, char *,
 197  200                  struct rpc_disptable *);
 198  201  static  int     checkauth(struct exportinfo *, struct svc_req *, cred_t *, int,
 199  202                  bool_t, bool_t *);
 200  203  static char     *client_name(struct svc_req *req);
 201  204  static char     *client_addr(struct svc_req *req, char *buf);
 202  205  extern  int     sec_svc_getcred(struct svc_req *, cred_t *cr, char **, int *);
 203  206  extern  bool_t  sec_svc_inrootlist(int, caddr_t, int, caddr_t *);
 204      -static void     *nfs_srv_zone_init(zoneid_t);
 205      -static void     nfs_srv_zone_fini(zoneid_t, void *);
      207 +static void     *nfs_server_zone_init(zoneid_t);
      208 +static void     nfs_server_zone_fini(zoneid_t, void *);
      209 +static void     nfs_server_zone_shutdown(zoneid_t, void *);
 206  210  
 207  211  #define NFSLOG_COPY_NETBUF(exi, xprt, nb)       {               \
 208  212          (nb)->maxlen = (xprt)->xp_rtaddr.maxlen;                \
 209  213          (nb)->len = (xprt)->xp_rtaddr.len;                      \
 210  214          (nb)->buf = kmem_alloc((nb)->len, KM_SLEEP);            \
 211  215          bcopy((xprt)->xp_rtaddr.buf, (nb)->buf, (nb)->len);     \
 212  216          }
 213  217  
 214  218  /*
 215  219   * Public Filehandle common nfs routines
↓ open down ↓ 40 lines elided ↑ open up ↑ 
 256  260  
 257  261  /*
 258  262   * DSS: distributed stable storage
 259  263   * lists of all DSS paths: current, and before last warmstart
 260  264   */
 261  265  nvlist_t *rfs4_dss_paths, *rfs4_dss_oldpaths;
 262  266  
 263  267  int rfs4_dispatch(struct rpcdisp *, struct svc_req *, SVCXPRT *, char *);
 264  268  bool_t rfs4_minorvers_mismatch(struct svc_req *, SVCXPRT *, void *);
 265  269  
      270 +nfs_globals_t *
      271 +nfs_srv_getzg(void)
      272 +{
      273 +        nfs_globals_t *ng;
      274 +
      275 +        ng = zone_getspecific(nfssrv_zone_key, curzone);
      276 +        return (ng);
      277 +}
      278 +
 266  279  /*
 267  280   * Will be called at the point the server pool is being unregistered
 268  281   * from the pool list. From that point onwards, the pool is waiting
 269  282   * to be drained and as such the server state is stale and pertains
 270  283   * to the old instantiation of the NFS server pool.
 271  284   */
 272  285  void
 273  286  nfs_srv_offline(void)
 274  287  {
 275  288          nfs_globals_t *ng;
 276  289  
 277      -        ng = zone_getspecific(nfssrv_zone_key, curzone);
      290 +        ng = nfs_srv_getzg();
 278  291  
 279  292          mutex_enter(&ng->nfs_server_upordown_lock);
 280  293          if (ng->nfs_server_upordown == NFS_SERVER_RUNNING) {
 281  294                  ng->nfs_server_upordown = NFS_SERVER_OFFLINE;
 282  295          }
 283  296          mutex_exit(&ng->nfs_server_upordown_lock);
 284  297  }
 285  298  
 286  299  /*
 287  300   * Will be called at the point the server pool is being destroyed so
↓ open down ↓ 16 lines elided ↑ open up ↑ 
 304  317  void
 305  318  nfs_srv_quiesce_all(void)
 306  319  {
 307  320          int quiesce = 1;
 308  321          nfs_srv_shutdown_all(quiesce);
 309  322  }
 310  323  
 311  324  static void
 312  325  nfs_srv_shutdown_all(int quiesce)
 313  326  {
 314      -        nfs_globals_t *ng = zone_getspecific(nfssrv_zone_key, curzone);
      327 +        nfs_globals_t *ng = nfs_srv_getzg();
 315  328  
 316  329          mutex_enter(&ng->nfs_server_upordown_lock);
 317  330          if (quiesce) {
 318  331                  if (ng->nfs_server_upordown == NFS_SERVER_RUNNING ||
 319  332                      ng->nfs_server_upordown == NFS_SERVER_OFFLINE) {
 320  333                          ng->nfs_server_upordown = NFS_SERVER_QUIESCED;
 321  334                          cv_signal(&ng->nfs_server_upordown_cv);
 322  335  
 323  336                          /* reset DSS state */
 324  337                          rfs4_dss_numnewpaths = 0;
↓ open down ↓ 94 lines elided ↑ open up ↑ 
 419  432          char buf[KNC_STRSIZE];
 420  433          size_t len;
 421  434          STRUCT_HANDLE(nfs_svc_args, uap);
 422  435          struct netbuf addrmask;
 423  436          SVC_CALLOUT_TABLE *sctp = NULL;
 424  437  
 425  438  #ifdef lint
 426  439          model = model;          /* STRUCT macros don't always refer to it */
 427  440  #endif
 428  441  
 429      -        ng = zone_getspecific(nfssrv_zone_key, curzone);
      442 +        ng = nfs_srv_getzg();
 430  443          STRUCT_SET_HANDLE(uap, model, arg);
 431  444  
 432  445          /* Check privileges in nfssys() */
 433  446  
 434  447          if ((fp = getf(STRUCT_FGET(uap, fd))) == NULL)
 435  448                  return (EBADF);
 436  449  
      450 +        /* Setup global file handle in nfs_export */
      451 +        if ((error = nfs_export_get_rootfh(ng)) != 0)
      452 +                return (error);
      453 +
 437  454          /*
 438  455           * Set read buffer size to rsize
 439  456           * and add room for RPC headers.
 440  457           */
 441  458          readsize = nfs3tsize() + (RPC_MAXDATASIZE - NFS_MAXDATA);
 442  459          if (readsize < RPC_MAXDATASIZE)
 443  460                  readsize = RPC_MAXDATASIZE;
 444  461  
 445  462          error = copyinstr((const char *)STRUCT_FGETP(uap, netid), buf,
 446  463              KNC_STRSIZE, &len);
↓ open down ↓ 98 lines elided ↑ open up ↑ 
 545  562          int svc_state = 0;
 546  563  
 547  564          /* Double check the vers min/max ranges */
 548  565          if ((rsa->nfs_versmin > rsa->nfs_versmax) ||
 549  566              (rsa->nfs_versmin < NFS_VERSMIN) ||
 550  567              (rsa->nfs_versmax > NFS_VERSMAX)) {
 551  568                  rsa->nfs_versmin = NFS_VERSMIN_DEFAULT;
 552  569                  rsa->nfs_versmax = NFS_VERSMAX_DEFAULT;
 553  570          }
 554  571  
 555      -        ng = zone_getspecific(nfssrv_zone_key, curzone);
      572 +        ng = nfs_srv_getzg();
 556  573          ng->nfs_versmin = rsa->nfs_versmin;
 557  574          ng->nfs_versmax = rsa->nfs_versmax;
 558  575  
 559  576          /* Set the versions in the callout table */
 560  577          __nfs_sc_rdma[0].sc_versmin = rsa->nfs_versmin;
 561  578          __nfs_sc_rdma[0].sc_versmax = rsa->nfs_versmax;
 562  579          /* For the NFS_ACL program, check the max version */
 563  580          __nfs_sc_rdma[1].sc_versmin = rsa->nfs_versmin;
 564  581          if (rsa->nfs_versmax > NFS_ACL_VERSMAX)
 565  582                  __nfs_sc_rdma[1].sc_versmax = NFS_ACL_VERSMAX;
↓ open down ↓ 1974 lines elided ↑ open up ↑ 
2540 2557  /*
2541 2558   * NFS Server initialization routine.  This routine should only be called
2542 2559   * once.  It performs the following tasks:
2543 2560   *      - Call sub-initialization routines (localize access to variables)
2544 2561   *      - Initialize all locks
2545 2562   *      - initialize the version 3 write verifier
2546 2563   */
2547 2564  void
2548 2565  nfs_srvinit(void)
2549 2566  {
2550      -        /* NFS server zone-specific global variables */
2551      -        zone_key_create(&nfssrv_zone_key, nfs_srv_zone_init,
2552      -            NULL, nfs_srv_zone_fini);
2553 2567  
     2568 +        /* Truly global stuff in this module (not per zone) */
     2569 +        rw_init(&nfssrv_globals_rwl, NULL, RW_DEFAULT, NULL);
     2570 +        list_create(&nfssrv_globals_list, sizeof (nfs_globals_t),
     2571 +            offsetof (nfs_globals_t, nfs_g_link));
     2572 +
     2573 +        /* The order here is important */
2554 2574          nfs_exportinit();
2555 2575          rfs_srvrinit();
2556 2576          rfs3_srvrinit();
2557 2577          rfs4_srvrinit();
2558 2578          nfsauth_init();
     2579 +
     2580 +        /*
     2581 +         * NFS server zone-specific global variables
     2582 +         * Note the zone_init is called for the GZ here.
     2583 +         */
     2584 +        zone_key_create(&nfssrv_zone_key, nfs_server_zone_init,
     2585 +            nfs_server_zone_shutdown, nfs_server_zone_fini);
2559 2586  }
2560 2587  
2561 2588  /*
2562 2589   * NFS Server finalization routine. This routine is called to cleanup the
2563 2590   * initialization work previously performed if the NFS server module could
2564 2591   * not be loaded correctly.
2565 2592   */
2566 2593  void
2567 2594  nfs_srvfini(void)
2568 2595  {
     2596 +
     2597 +        /*
     2598 +         * NFS server zone-specific global variables
     2599 +         * Note the zone_fini is called for the GZ here.
     2600 +         */
     2601 +        (void) zone_key_delete(nfssrv_zone_key);
     2602 +
     2603 +        /* The order here is important (reverse of init) */
2569 2604          nfsauth_fini();
2570 2605          rfs4_srvrfini();
2571 2606          rfs3_srvrfini();
2572 2607          rfs_srvrfini();
2573 2608          nfs_exportfini();
2574 2609  
2575      -        (void) zone_key_delete(nfssrv_zone_key);
     2610 +        /* Truly global stuff in this module (not per zone) */
     2611 +        list_destroy(&nfssrv_globals_list);
     2612 +        rw_destroy(&nfssrv_globals_rwl);
2576 2613  }
2577 2614  
2578      -/* ARGSUSED */
     2615 +/*
     2616 + * Zone init, shutdown, fini functions for the NFS server
     2617 + *
     2618 + * This design is careful to create the entire hierarhcy of
     2619 + * NFS server "globals" (including those created by various
     2620 + * per-module *_zone_init functions, etc.) so that all these
     2621 + * objects have exactly the same lifetime.
     2622 + *
     2623 + * These objects are also kept on a list for two reasons:
     2624 + * 1: It makes finding these in  mdb _much_ easier.
     2625 + * 2: It allows operating across all zone globals for
     2626 + *    functions like nfs_auth.c:exi_cache_reclaim
     2627 + */
2579 2628  static void *
2580      -nfs_srv_zone_init(zoneid_t zoneid)
     2629 +nfs_server_zone_init(zoneid_t zoneid)
2581 2630  {
2582 2631          nfs_globals_t *ng;
2583 2632  
2584 2633          ng = kmem_zalloc(sizeof (*ng), KM_SLEEP);
2585 2634  
2586 2635          ng->nfs_versmin = NFS_VERSMIN_DEFAULT;
2587 2636          ng->nfs_versmax = NFS_VERSMAX_DEFAULT;
2588 2637  
2589 2638          /* Init the stuff to control start/stop */
2590 2639          ng->nfs_server_upordown = NFS_SERVER_STOPPED;
2591 2640          mutex_init(&ng->nfs_server_upordown_lock, NULL, MUTEX_DEFAULT, NULL);
2592 2641          cv_init(&ng->nfs_server_upordown_cv, NULL, CV_DEFAULT, NULL);
2593 2642          mutex_init(&ng->rdma_wait_mutex, NULL, MUTEX_DEFAULT, NULL);
2594 2643          cv_init(&ng->rdma_wait_cv, NULL, CV_DEFAULT, NULL);
2595 2644  
     2645 +        ng->nfs_zoneid = zoneid;
     2646 +
     2647 +        /*
     2648 +         * Order here is important.
     2649 +         * export init must precede srv init calls.
     2650 +         */
     2651 +        nfs_export_zone_init(ng);
     2652 +        rfs_srv_zone_init(ng);
     2653 +        rfs3_srv_zone_init(ng);
     2654 +        rfs4_srv_zone_init(ng);
     2655 +        nfsauth_zone_init(ng);
     2656 +
     2657 +        rw_enter(&nfssrv_globals_rwl, RW_WRITER);
     2658 +        list_insert_tail(&nfssrv_globals_list, ng);
     2659 +        rw_exit(&nfssrv_globals_rwl);
     2660 +
2596 2661          return (ng);
2597 2662  }
2598 2663  
2599 2664  /* ARGSUSED */
2600 2665  static void
2601      -nfs_srv_zone_fini(zoneid_t zoneid, void *data)
     2666 +nfs_server_zone_shutdown(zoneid_t zoneid, void *data)
2602 2667  {
2603 2668          nfs_globals_t *ng;
2604 2669  
2605 2670          ng = (nfs_globals_t *)data;
     2671 +
     2672 +        /*
     2673 +         * Order is like _fini, but only
     2674 +         * some modules need this hook.
     2675 +         */
     2676 +        nfsauth_zone_shutdown(ng);
     2677 +        nfs_export_zone_shutdown(ng);
     2678 +}
     2679 +
     2680 +/* ARGSUSED */
     2681 +static void
     2682 +nfs_server_zone_fini(zoneid_t zoneid, void *data)
     2683 +{
     2684 +        nfs_globals_t *ng;
     2685 +
     2686 +        ng = (nfs_globals_t *)data;
     2687 +
     2688 +        rw_enter(&nfssrv_globals_rwl, RW_WRITER);
     2689 +        list_remove(&nfssrv_globals_list, ng);
     2690 +        rw_exit(&nfssrv_globals_rwl);
     2691 +
     2692 +        /*
     2693 +         * Order here is important.
     2694 +         * reverse order from init
     2695 +         */
     2696 +        nfsauth_zone_fini(ng);
     2697 +        rfs4_srv_zone_fini(ng);
     2698 +        rfs3_srv_zone_fini(ng);
     2699 +        rfs_srv_zone_fini(ng);
     2700 +        nfs_export_zone_fini(ng);
     2701 +
2606 2702          mutex_destroy(&ng->nfs_server_upordown_lock);
2607 2703          cv_destroy(&ng->nfs_server_upordown_cv);
2608 2704          mutex_destroy(&ng->rdma_wait_mutex);
2609 2705          cv_destroy(&ng->rdma_wait_cv);
2610 2706  
2611 2707          kmem_free(ng, sizeof (*ng));
2612 2708  }
2613 2709  
2614 2710  /*
2615 2711   * Set up an iovec array of up to cnt pointers.
↓ open down ↓ 690 lines elided ↑ open up ↑ 
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX