Print this page
NEX-15279 support NFS server in zone
NEX-15520 online NFS shares cause zoneadm halt to hang in nfs_export_zone_fini
Portions contributed by: Dan Kruchinin dan.kruchinin@nexenta.com
Portions contributed by: Stepan Zastupov stepan.zastupov@gmail.com
Reviewed by: Joyce McIntosh <joyce.mcintosh@nexenta.com>
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
NEX-9275 Got "bad mutex" panic when run IO to nfs share from clients
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
NEX-5667 nfssrv_stats_flags does not work for aggregated kstats
NEX-4472 nfsauth_retrieve() flood caused by NFS clients with personal identity problems
Reviewed by: Yuri Pankov <yuri.pankov@nexenta.com>
NEX-2345 nfsauth_cache_get() could spend a lot of time walking exi_cache
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
NEX-2972 bad free at checkauth+0x1a2()
Reviewed by: Jan Kryl <jan.kryl@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
NEX-2949 Panic due to bad mutex, from auth_cache being previously freed
NEX-1974 Support for more than 16 groups with AUTH_SYS
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
Reviewed by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com>
NEX-1128 NFS server: Generic uid and gid remapping for AUTH_SYS
Reviewed by: Jan Kryl <jan.kryl@nexenta.com>
OS-152 NFS extremely slow in nfsauth
Reviewed by: Jan Kryl <jan.kryl@nexenta.com>
Reviewed by: Ilya Usvyatsky <ilya.usvyatsky@nexenta.com>
Reviewed by: Josef Sipek <josef.sipek@nexenta.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/nfs/nfs_auth.c
+++ new/usr/src/uts/common/fs/nfs/nfs_auth.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
|
↓ open down ↓ |
12 lines elided |
↑ open up ↑ |
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 - * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
24 23 * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
24 + */
25 +
26 +/*
27 + * Copyright 2018 Nexenta Systems, Inc.
25 28 * Copyright (c) 2015 by Delphix. All rights reserved.
26 29 */
27 30
28 31 #include <sys/param.h>
29 32 #include <sys/errno.h>
30 33 #include <sys/vfs.h>
31 34 #include <sys/vnode.h>
32 35 #include <sys/cred.h>
33 36 #include <sys/cmn_err.h>
34 37 #include <sys/systm.h>
35 38 #include <sys/kmem.h>
36 39 #include <sys/pathname.h>
37 40 #include <sys/utsname.h>
38 41 #include <sys/debug.h>
39 42 #include <sys/door.h>
40 43 #include <sys/sdt.h>
41 44 #include <sys/thread.h>
42 45 #include <sys/avl.h>
43 46
44 47 #include <rpc/types.h>
45 48 #include <rpc/auth.h>
|
↓ open down ↓ |
11 lines elided |
↑ open up ↑ |
46 49 #include <rpc/clnt.h>
47 50
48 51 #include <nfs/nfs.h>
49 52 #include <nfs/export.h>
50 53 #include <nfs/nfs_clnt.h>
51 54 #include <nfs/auth.h>
52 55
53 56 static struct kmem_cache *exi_cache_handle;
54 57 static void exi_cache_reclaim(void *);
55 58 static void exi_cache_trim(struct exportinfo *exi);
59 +static void *nfsauth_zone_init(zoneid_t);
60 +static void nfsauth_zone_shutdown(zoneid_t zoneid, void *data);
61 +static void nfsauth_zone_fini(zoneid_t, void *);
56 62
57 63 extern pri_t minclsyspri;
58 64
65 +/* NFS auth cache statistics */
59 66 volatile uint_t nfsauth_cache_hit;
60 67 volatile uint_t nfsauth_cache_miss;
61 68 volatile uint_t nfsauth_cache_refresh;
62 69 volatile uint_t nfsauth_cache_reclaim;
63 70 volatile uint_t exi_cache_auth_reclaim_failed;
64 71 volatile uint_t exi_cache_clnt_reclaim_failed;
65 72
66 73 /*
67 74 * The lifetime of an auth cache entry:
68 75 * ------------------------------------
69 76 *
70 77 * An auth cache entry is created with both the auth_time
71 78 * and auth_freshness times set to the current time.
72 79 *
73 80 * Upon every client access which results in a hit, the
74 81 * auth_time will be updated.
75 82 *
76 83 * If a client access determines that the auth_freshness
77 84 * indicates that the entry is STALE, then it will be
78 85 * refreshed. Note that this will explicitly reset
79 86 * auth_time.
80 87 *
81 88 * When the REFRESH successfully occurs, then the
82 89 * auth_freshness is updated.
83 90 *
84 91 * There are two ways for an entry to leave the cache:
85 92 *
86 93 * 1) Purged by an action on the export (remove or changed)
87 94 * 2) Memory backpressure from the kernel (check against NFSAUTH_CACHE_TRIM)
88 95 *
89 96 * For 2) we check the timeout value against auth_time.
90 97 */
91 98
92 99 /*
93 100 * Number of seconds until we mark for refresh an auth cache entry.
94 101 */
95 102 #define NFSAUTH_CACHE_REFRESH 600
96 103
97 104 /*
98 105 * Number of idle seconds until we yield to backpressure
99 106 * to trim a cache entry.
100 107 */
101 108 #define NFSAUTH_CACHE_TRIM 3600
102 109
103 110 /*
104 111 * While we could encapuslate the exi_list inside the
105 112 * exi structure, we can't do that for the auth_list.
106 113 * So, to keep things looking clean, we keep them both
107 114 * in these external lists.
108 115 */
109 116 typedef struct refreshq_exi_node {
110 117 struct exportinfo *ren_exi;
111 118 list_t ren_authlist;
|
↓ open down ↓ |
43 lines elided |
↑ open up ↑ |
112 119 list_node_t ren_node;
113 120 } refreshq_exi_node_t;
114 121
115 122 typedef struct refreshq_auth_node {
116 123 struct auth_cache *ran_auth;
117 124 char *ran_netid;
118 125 list_node_t ran_node;
119 126 } refreshq_auth_node_t;
120 127
121 128 /*
122 - * Used to manipulate things on the refreshq_queue.
123 - * Note that the refresh thread will effectively
124 - * pop a node off of the queue, at which point it
129 + * Used to manipulate things on the refreshq_queue. Note that the refresh
130 + * thread will effectively pop a node off of the queue, at which point it
125 131 * will no longer need to hold the mutex.
126 132 */
127 133 static kmutex_t refreshq_lock;
128 134 static list_t refreshq_queue;
129 135 static kcondvar_t refreshq_cv;
130 136
131 137 /*
132 - * If there is ever a problem with loading the
133 - * module, then nfsauth_fini() needs to be called
134 - * to remove state. In that event, since the
135 - * refreshq thread has been started, they need to
136 - * work together to get rid of state.
138 + * If there is ever a problem with loading the module, then nfsauth_fini()
139 + * needs to be called to remove state. In that event, since the refreshq
140 + * thread has been started, they need to work together to get rid of state.
137 141 */
138 142 typedef enum nfsauth_refreshq_thread_state {
139 143 REFRESHQ_THREAD_RUNNING,
140 144 REFRESHQ_THREAD_FINI_REQ,
141 - REFRESHQ_THREAD_HALTED
145 + REFRESHQ_THREAD_HALTED,
146 + REFRESHQ_THREAD_NEED_CREATE
142 147 } nfsauth_refreshq_thread_state_t;
143 148
144 -nfsauth_refreshq_thread_state_t
145 -refreshq_thread_state = REFRESHQ_THREAD_HALTED;
149 +typedef struct nfsauth_globals {
150 + kmutex_t mountd_lock;
151 + door_handle_t mountd_dh;
146 152
153 + /*
154 + * Used to manipulate things on the refreshq_queue. Note that the
155 + * refresh thread will effectively pop a node off of the queue,
156 + * at which point it will no longer need to hold the mutex.
157 + */
158 + kmutex_t refreshq_lock;
159 + list_t refreshq_queue;
160 + kcondvar_t refreshq_cv;
161 +
162 + /*
163 + * A list_t would be overkill. These are auth_cache entries which are
164 + * no longer linked to an exi. It should be the case that all of their
165 + * states are NFS_AUTH_INVALID, i.e., the only way to be put on this
166 + * list is iff their state indicated that they had been placed on the
167 + * refreshq_queue.
168 + *
169 + * Note that while there is no link from the exi or back to the exi,
170 + * the exi can not go away until these entries are harvested.
171 + */
172 + struct auth_cache *refreshq_dead_entries;
173 + nfsauth_refreshq_thread_state_t refreshq_thread_state;
174 +
175 +} nfsauth_globals_t;
176 +
147 177 static void nfsauth_free_node(struct auth_cache *);
148 -static void nfsauth_refresh_thread(void);
178 +static void nfsauth_refresh_thread(nfsauth_globals_t *);
149 179
150 180 static int nfsauth_cache_compar(const void *, const void *);
151 181
152 -/*
153 - * mountd is a server-side only daemon. This will need to be
154 - * revisited if the NFS server is ever made zones-aware.
155 - */
156 -kmutex_t mountd_lock;
157 -door_handle_t mountd_dh;
182 +static zone_key_t nfsauth_zone_key;
158 183
159 184 void
160 185 mountd_args(uint_t did)
161 186 {
162 - mutex_enter(&mountd_lock);
163 - if (mountd_dh != NULL)
164 - door_ki_rele(mountd_dh);
165 - mountd_dh = door_ki_lookup(did);
166 - mutex_exit(&mountd_lock);
187 + nfsauth_globals_t *nag;
188 +
189 + nag = zone_getspecific(nfsauth_zone_key, curzone);
190 + mutex_enter(&nag->mountd_lock);
191 + if (nag->mountd_dh != NULL)
192 + door_ki_rele(nag->mountd_dh);
193 + nag->mountd_dh = door_ki_lookup(did);
194 + mutex_exit(&nag->mountd_lock);
167 195 }
168 196
169 197 void
170 198 nfsauth_init(void)
171 199 {
172 - /*
173 - * mountd can be restarted by smf(5). We need to make sure
174 - * the updated door handle will safely make it to mountd_dh
175 - */
176 - mutex_init(&mountd_lock, NULL, MUTEX_DEFAULT, NULL);
200 + zone_key_create(&nfsauth_zone_key, nfsauth_zone_init,
201 + nfsauth_zone_shutdown, nfsauth_zone_fini);
177 202
178 - mutex_init(&refreshq_lock, NULL, MUTEX_DEFAULT, NULL);
179 - list_create(&refreshq_queue, sizeof (refreshq_exi_node_t),
180 - offsetof(refreshq_exi_node_t, ren_node));
181 -
182 - cv_init(&refreshq_cv, NULL, CV_DEFAULT, NULL);
183 -
184 - /*
185 - * Allocate nfsauth cache handle
186 - */
187 203 exi_cache_handle = kmem_cache_create("exi_cache_handle",
188 204 sizeof (struct auth_cache), 0, NULL, NULL,
189 205 exi_cache_reclaim, NULL, NULL, 0);
190 -
191 - refreshq_thread_state = REFRESHQ_THREAD_RUNNING;
192 - (void) zthread_create(NULL, 0, nfsauth_refresh_thread,
193 - NULL, 0, minclsyspri);
194 206 }
195 207
196 -/*
197 - * Finalization routine for nfsauth. It is important to call this routine
198 - * before destroying the exported_lock.
199 - */
200 208 void
201 209 nfsauth_fini(void)
202 210 {
203 - refreshq_exi_node_t *ren;
211 + kmem_cache_destroy(exi_cache_handle);
212 +}
204 213
214 +/*ARGSUSED*/
215 +static void *
216 +nfsauth_zone_init(zoneid_t zoneid)
217 +{
218 + nfsauth_globals_t *nag;
219 +
220 + nag = kmem_zalloc(sizeof (*nag), KM_SLEEP);
221 +
205 222 /*
206 - * Prevent the nfsauth_refresh_thread from getting new
207 - * work.
223 + * mountd can be restarted by smf(5). We need to make sure
224 + * the updated door handle will safely make it to mountd_dh.
208 225 */
209 - mutex_enter(&refreshq_lock);
210 - if (refreshq_thread_state != REFRESHQ_THREAD_HALTED) {
211 - refreshq_thread_state = REFRESHQ_THREAD_FINI_REQ;
212 - cv_broadcast(&refreshq_cv);
226 + mutex_init(&nag->mountd_lock, NULL, MUTEX_DEFAULT, NULL);
227 + mutex_init(&nag->refreshq_lock, NULL, MUTEX_DEFAULT, NULL);
228 + list_create(&nag->refreshq_queue, sizeof (refreshq_exi_node_t),
229 + offsetof(refreshq_exi_node_t, ren_node));
230 + cv_init(&nag->refreshq_cv, NULL, CV_DEFAULT, NULL);
231 + nag->refreshq_thread_state = REFRESHQ_THREAD_NEED_CREATE;
213 232
214 - /*
215 - * Also, wait for nfsauth_refresh_thread() to exit.
216 - */
217 - while (refreshq_thread_state != REFRESHQ_THREAD_HALTED) {
218 - cv_wait(&refreshq_cv, &refreshq_lock);
219 - }
233 + return (nag);
234 +}
235 +
236 +/*ARGSUSED*/
237 +static void
238 +nfsauth_zone_shutdown(zoneid_t zoneid, void *data)
239 +{
240 + refreshq_exi_node_t *ren;
241 + nfsauth_globals_t *nag = data;
242 +
243 + /* Prevent the nfsauth_refresh_thread from getting new work */
244 + mutex_enter(&nag->refreshq_lock);
245 + if (nag->refreshq_thread_state == REFRESHQ_THREAD_RUNNING) {
246 + nag->refreshq_thread_state = REFRESHQ_THREAD_FINI_REQ;
247 + cv_broadcast(&nag->refreshq_cv);
248 +
249 + /* Wait for nfsauth_refresh_thread() to exit */
250 + while (nag->refreshq_thread_state != REFRESHQ_THREAD_HALTED)
251 + cv_wait(&nag->refreshq_cv, &nag->refreshq_lock);
220 252 }
221 - mutex_exit(&refreshq_lock);
253 + mutex_exit(&nag->refreshq_lock);
222 254
223 255 /*
224 256 * Walk the exi_list and in turn, walk the auth_lists and free all
225 257 * lists. In addition, free INVALID auth_cache entries.
226 258 */
227 - while ((ren = list_remove_head(&refreshq_queue))) {
259 + while ((ren = list_remove_head(&nag->refreshq_queue))) {
228 260 refreshq_auth_node_t *ran;
229 261
230 262 while ((ran = list_remove_head(&ren->ren_authlist)) != NULL) {
231 263 struct auth_cache *p = ran->ran_auth;
232 264 if (p->auth_state == NFS_AUTH_INVALID)
233 265 nfsauth_free_node(p);
234 266 strfree(ran->ran_netid);
235 - kmem_free(ran, sizeof (refreshq_auth_node_t));
267 + kmem_free(ran, sizeof (*ran));
236 268 }
237 269
238 270 list_destroy(&ren->ren_authlist);
239 - exi_rele(ren->ren_exi);
240 - kmem_free(ren, sizeof (refreshq_exi_node_t));
271 + exi_rele(&ren->ren_exi);
272 + kmem_free(ren, sizeof (*ren));
241 273 }
242 - list_destroy(&refreshq_queue);
274 +}
243 275
244 - cv_destroy(&refreshq_cv);
245 - mutex_destroy(&refreshq_lock);
276 +/*ARGSUSED*/
277 +static void
278 +nfsauth_zone_fini(zoneid_t zoneid, void *data)
279 +{
280 + nfsauth_globals_t *nag = data;
246 281
247 - mutex_destroy(&mountd_lock);
248 -
249 - /*
250 - * Deallocate nfsauth cache handle
251 - */
252 - kmem_cache_destroy(exi_cache_handle);
282 + list_destroy(&nag->refreshq_queue);
283 + cv_destroy(&nag->refreshq_cv);
284 + mutex_destroy(&nag->refreshq_lock);
285 + mutex_destroy(&nag->mountd_lock);
286 + kmem_free(nag, sizeof (*nag));
253 287 }
254 288
255 289 /*
256 290 * Convert the address in a netbuf to
257 291 * a hash index for the auth_cache table.
258 292 */
259 293 static int
260 294 hash(struct netbuf *a)
261 295 {
262 296 int i, h = 0;
263 297
264 298 for (i = 0; i < a->len; i++)
265 299 h ^= a->buf[i];
266 300
267 301 return (h & (AUTH_TABLESIZE - 1));
268 302 }
269 303
270 304 /*
271 305 * Mask out the components of an
272 306 * address that do not identify
273 307 * a host. For socket addresses the
274 308 * masking gets rid of the port number.
275 309 */
276 310 static void
277 311 addrmask(struct netbuf *addr, struct netbuf *mask)
278 312 {
279 313 int i;
280 314
281 315 for (i = 0; i < addr->len; i++)
282 316 addr->buf[i] &= mask->buf[i];
283 317 }
284 318
285 319 /*
286 320 * nfsauth4_access is used for NFS V4 auth checking. Besides doing
287 321 * the common nfsauth_access(), it will check if the client can
288 322 * have a limited access to this vnode even if the security flavor
289 323 * used does not meet the policy.
290 324 */
291 325 int
292 326 nfsauth4_access(struct exportinfo *exi, vnode_t *vp, struct svc_req *req,
293 327 cred_t *cr, uid_t *uid, gid_t *gid, uint_t *ngids, gid_t **gids)
294 328 {
295 329 int access;
296 330
297 331 access = nfsauth_access(exi, req, cr, uid, gid, ngids, gids);
298 332
299 333 /*
300 334 * There are cases that the server needs to allow the client
301 335 * to have a limited view.
302 336 *
303 337 * e.g.
304 338 * /export is shared as "sec=sys,rw=dfs-test-4,sec=krb5,rw"
305 339 * /export/home is shared as "sec=sys,rw"
306 340 *
307 341 * When the client mounts /export with sec=sys, the client
308 342 * would get a limited view with RO access on /export to see
309 343 * "home" only because the client is allowed to access
310 344 * /export/home with auth_sys.
311 345 */
312 346 if (access & NFSAUTH_DENIED || access & NFSAUTH_WRONGSEC) {
313 347 /*
314 348 * Allow ro permission with LIMITED view if there is a
315 349 * sub-dir exported under vp.
316 350 */
317 351 if (has_visible(exi, vp))
318 352 return (NFSAUTH_LIMITED);
319 353 }
320 354
321 355 return (access);
322 356 }
323 357
324 358 static void
325 359 sys_log(const char *msg)
326 360 {
327 361 static time_t tstamp = 0;
328 362 time_t now;
329 363
330 364 /*
331 365 * msg is shown (at most) once per minute
332 366 */
333 367 now = gethrestime_sec();
|
↓ open down ↓ |
71 lines elided |
↑ open up ↑ |
334 368 if ((tstamp + 60) < now) {
335 369 tstamp = now;
336 370 cmn_err(CE_WARN, msg);
337 371 }
338 372 }
339 373
340 374 /*
341 375 * Callup to the mountd to get access information in the kernel.
342 376 */
343 377 static bool_t
344 -nfsauth_retrieve(struct exportinfo *exi, char *req_netid, int flavor,
345 - struct netbuf *addr, int *access, cred_t *clnt_cred, uid_t *srv_uid,
346 - gid_t *srv_gid, uint_t *srv_gids_cnt, gid_t **srv_gids)
378 +nfsauth_retrieve(nfsauth_globals_t *nag, struct exportinfo *exi,
379 + char *req_netid, int flavor, struct netbuf *addr, int *access,
380 + cred_t *clnt_cred, uid_t *srv_uid, gid_t *srv_gid, uint_t *srv_gids_cnt,
381 + gid_t **srv_gids)
347 382 {
348 383 varg_t varg = {0};
349 384 nfsauth_res_t res = {0};
350 385 XDR xdrs;
351 386 size_t absz;
352 387 caddr_t abuf;
353 388 int last = 0;
354 389 door_arg_t da;
355 390 door_info_t di;
356 391 door_handle_t dh;
357 392 uint_t ntries = 0;
358 393
359 394 /*
360 395 * No entry in the cache for this client/flavor
361 396 * so we need to call the nfsauth service in the
362 397 * mount daemon.
363 398 */
364 399
365 400 varg.vers = V_PROTO;
366 401 varg.arg_u.arg.cmd = NFSAUTH_ACCESS;
367 402 varg.arg_u.arg.areq.req_client.n_len = addr->len;
368 403 varg.arg_u.arg.areq.req_client.n_bytes = addr->buf;
369 404 varg.arg_u.arg.areq.req_netid = req_netid;
370 405 varg.arg_u.arg.areq.req_path = exi->exi_export.ex_path;
371 406 varg.arg_u.arg.areq.req_flavor = flavor;
372 407 varg.arg_u.arg.areq.req_clnt_uid = crgetuid(clnt_cred);
373 408 varg.arg_u.arg.areq.req_clnt_gid = crgetgid(clnt_cred);
374 409 varg.arg_u.arg.areq.req_clnt_gids.len = crgetngroups(clnt_cred);
375 410 varg.arg_u.arg.areq.req_clnt_gids.val = (gid_t *)crgetgroups(clnt_cred);
376 411
377 412 DTRACE_PROBE1(nfsserv__func__nfsauth__varg, varg_t *, &varg);
378 413
379 414 /*
380 415 * Setup the XDR stream for encoding the arguments. Notice that
381 416 * in addition to the args having variable fields (req_netid and
382 417 * req_path), the argument data structure is itself versioned,
383 418 * so we need to make sure we can size the arguments buffer
384 419 * appropriately to encode all the args. If we can't get sizing
385 420 * info _or_ properly encode the arguments, there's really no
386 421 * point in continuting, so we fail the request.
387 422 */
388 423 if ((absz = xdr_sizeof(xdr_varg, &varg)) == 0) {
389 424 *access = NFSAUTH_DENIED;
390 425 return (FALSE);
391 426 }
392 427
393 428 abuf = (caddr_t)kmem_alloc(absz, KM_SLEEP);
394 429 xdrmem_create(&xdrs, abuf, absz, XDR_ENCODE);
395 430 if (!xdr_varg(&xdrs, &varg)) {
396 431 XDR_DESTROY(&xdrs);
397 432 goto fail;
398 433 }
399 434 XDR_DESTROY(&xdrs);
400 435
401 436 /*
402 437 * Prepare the door arguments
403 438 *
404 439 * We don't know the size of the message the daemon
405 440 * will pass back to us. By setting rbuf to NULL,
406 441 * we force the door code to allocate a buf of the
407 442 * appropriate size. We must set rsize > 0, however,
408 443 * else the door code acts as if no response was
|
↓ open down ↓ |
52 lines elided |
↑ open up ↑ |
409 444 * expected and doesn't pass the data to us.
410 445 */
411 446 da.data_ptr = (char *)abuf;
412 447 da.data_size = absz;
413 448 da.desc_ptr = NULL;
414 449 da.desc_num = 0;
415 450 da.rbuf = NULL;
416 451 da.rsize = 1;
417 452
418 453 retry:
419 - mutex_enter(&mountd_lock);
420 - dh = mountd_dh;
454 + mutex_enter(&nag->mountd_lock);
455 + dh = nag->mountd_dh;
421 456 if (dh != NULL)
422 457 door_ki_hold(dh);
423 - mutex_exit(&mountd_lock);
458 + mutex_exit(&nag->mountd_lock);
424 459
425 460 if (dh == NULL) {
426 461 /*
427 462 * The rendezvous point has not been established yet!
428 463 * This could mean that either mountd(1m) has not yet
429 464 * been started or that _this_ routine nuked the door
430 465 * handle after receiving an EINTR for a REVOKED door.
431 466 *
432 467 * Returning NFSAUTH_DROP will cause the NFS client
433 468 * to retransmit the request, so let's try to be more
434 469 * rescillient and attempt for ntries before we bail.
435 470 */
436 471 if (++ntries % NFSAUTH_DR_TRYCNT) {
437 472 delay(hz);
438 473 goto retry;
439 474 }
440 475
441 476 kmem_free(abuf, absz);
442 477
443 478 sys_log("nfsauth: mountd has not established door");
444 479 *access = NFSAUTH_DROP;
445 480 return (FALSE);
446 481 }
447 482
448 483 ntries = 0;
449 484
450 485 /*
451 486 * Now that we've got what we need, place the call.
452 487 */
453 488 switch (door_ki_upcall_limited(dh, &da, NULL, SIZE_MAX, 0)) {
454 489 case 0: /* Success */
455 490 door_ki_rele(dh);
456 491
457 492 if (da.data_ptr == NULL && da.data_size == 0) {
458 493 /*
459 494 * The door_return that contained the data
460 495 * failed! We're here because of the 2nd
461 496 * door_return (w/o data) such that we can
462 497 * get control of the thread (and exit
463 498 * gracefully).
464 499 */
465 500 DTRACE_PROBE1(nfsserv__func__nfsauth__door__nil,
466 501 door_arg_t *, &da);
467 502 goto fail;
468 503 }
469 504
470 505 break;
471 506
472 507 case EAGAIN:
473 508 /*
474 509 * Server out of resources; back off for a bit
475 510 */
476 511 door_ki_rele(dh);
477 512 delay(hz);
478 513 goto retry;
479 514 /* NOTREACHED */
480 515
481 516 case EINTR:
482 517 if (!door_ki_info(dh, &di)) {
|
↓ open down ↓ |
49 lines elided |
↑ open up ↑ |
483 518 door_ki_rele(dh);
484 519
485 520 if (di.di_attributes & DOOR_REVOKED) {
486 521 /*
487 522 * The server barfed and revoked
488 523 * the (existing) door on us; we
489 524 * want to wait to give smf(5) a
490 525 * chance to restart mountd(1m)
491 526 * and establish a new door handle.
492 527 */
493 - mutex_enter(&mountd_lock);
494 - if (dh == mountd_dh) {
495 - door_ki_rele(mountd_dh);
496 - mountd_dh = NULL;
528 + mutex_enter(&nag->mountd_lock);
529 + if (dh == nag->mountd_dh) {
530 + door_ki_rele(nag->mountd_dh);
531 + nag->mountd_dh = NULL;
497 532 }
498 - mutex_exit(&mountd_lock);
533 + mutex_exit(&nag->mountd_lock);
499 534 delay(hz);
500 535 goto retry;
501 536 }
502 537 /*
503 538 * If the door was _not_ revoked on us,
504 539 * then more than likely we took an INTR,
505 540 * so we need to fail the operation.
506 541 */
507 542 goto fail;
508 543 }
509 544 /*
510 545 * The only failure that can occur from getting
511 546 * the door info is EINVAL, so we let the code
512 547 * below handle it.
513 548 */
514 549 /* FALLTHROUGH */
515 550
516 551 case EBADF:
517 552 case EINVAL:
518 553 default:
519 554 /*
520 555 * If we have a stale door handle, give smf a last
521 556 * chance to start it by sleeping for a little bit.
522 557 * If we're still hosed, we'll fail the call.
523 558 *
524 559 * Since we're going to reacquire the door handle
525 560 * upon the retry, we opt to sleep for a bit and
526 561 * _not_ to clear mountd_dh. If mountd restarted
527 562 * and was able to set mountd_dh, we should see
528 563 * the new instance; if not, we won't get caught
529 564 * up in the retry/DELAY loop.
530 565 */
531 566 door_ki_rele(dh);
532 567 if (!last) {
533 568 delay(hz);
534 569 last++;
535 570 goto retry;
536 571 }
537 572 sys_log("nfsauth: stale mountd door handle");
538 573 goto fail;
539 574 }
540 575
541 576 ASSERT(da.rbuf != NULL);
542 577
543 578 /*
544 579 * No door errors encountered; setup the XDR stream for decoding
545 580 * the results. If we fail to decode the results, we've got no
546 581 * other recourse than to fail the request.
547 582 */
548 583 xdrmem_create(&xdrs, da.rbuf, da.rsize, XDR_DECODE);
549 584 if (!xdr_nfsauth_res(&xdrs, &res)) {
550 585 xdr_free(xdr_nfsauth_res, (char *)&res);
551 586 XDR_DESTROY(&xdrs);
552 587 kmem_free(da.rbuf, da.rsize);
553 588 goto fail;
554 589 }
555 590 XDR_DESTROY(&xdrs);
556 591 kmem_free(da.rbuf, da.rsize);
557 592
558 593 DTRACE_PROBE1(nfsserv__func__nfsauth__results, nfsauth_res_t *, &res);
559 594 switch (res.stat) {
560 595 case NFSAUTH_DR_OKAY:
561 596 *access = res.ares.auth_perm;
562 597 *srv_uid = res.ares.auth_srv_uid;
563 598 *srv_gid = res.ares.auth_srv_gid;
564 599 *srv_gids_cnt = res.ares.auth_srv_gids.len;
565 600 *srv_gids = kmem_alloc(*srv_gids_cnt * sizeof (gid_t),
566 601 KM_SLEEP);
567 602 bcopy(res.ares.auth_srv_gids.val, *srv_gids,
568 603 *srv_gids_cnt * sizeof (gid_t));
569 604 break;
570 605
571 606 case NFSAUTH_DR_EFAIL:
572 607 case NFSAUTH_DR_DECERR:
573 608 case NFSAUTH_DR_BADCMD:
574 609 default:
575 610 xdr_free(xdr_nfsauth_res, (char *)&res);
576 611 fail:
577 612 *access = NFSAUTH_DENIED;
578 613 kmem_free(abuf, absz);
579 614 return (FALSE);
|
↓ open down ↓ |
71 lines elided |
↑ open up ↑ |
580 615 /* NOTREACHED */
581 616 }
582 617
583 618 xdr_free(xdr_nfsauth_res, (char *)&res);
584 619 kmem_free(abuf, absz);
585 620
586 621 return (TRUE);
587 622 }
588 623
589 624 static void
590 -nfsauth_refresh_thread(void)
625 +nfsauth_refresh_thread(nfsauth_globals_t *nag)
591 626 {
592 627 refreshq_exi_node_t *ren;
593 628 refreshq_auth_node_t *ran;
594 629
595 630 struct exportinfo *exi;
596 631
597 632 int access;
598 633 bool_t retrieval;
599 634
600 635 callb_cpr_t cprinfo;
601 636
602 - CALLB_CPR_INIT(&cprinfo, &refreshq_lock, callb_generic_cpr,
637 + CALLB_CPR_INIT(&cprinfo, &nag->refreshq_lock, callb_generic_cpr,
603 638 "nfsauth_refresh");
604 639
605 640 for (;;) {
606 - mutex_enter(&refreshq_lock);
607 - if (refreshq_thread_state != REFRESHQ_THREAD_RUNNING) {
641 + mutex_enter(&nag->refreshq_lock);
642 + if (nag->refreshq_thread_state != REFRESHQ_THREAD_RUNNING) {
608 643 /* Keep the hold on the lock! */
609 644 break;
610 645 }
611 646
612 - ren = list_remove_head(&refreshq_queue);
647 + ren = list_remove_head(&nag->refreshq_queue);
613 648 if (ren == NULL) {
614 649 CALLB_CPR_SAFE_BEGIN(&cprinfo);
615 - cv_wait(&refreshq_cv, &refreshq_lock);
616 - CALLB_CPR_SAFE_END(&cprinfo, &refreshq_lock);
617 - mutex_exit(&refreshq_lock);
650 + cv_wait(&nag->refreshq_cv, &nag->refreshq_lock);
651 + CALLB_CPR_SAFE_END(&cprinfo, &nag->refreshq_lock);
652 + mutex_exit(&nag->refreshq_lock);
618 653 continue;
619 654 }
620 - mutex_exit(&refreshq_lock);
655 + mutex_exit(&nag->refreshq_lock);
621 656
622 657 exi = ren->ren_exi;
623 658 ASSERT(exi != NULL);
624 659
625 660 /*
626 661 * Since the ren was removed from the refreshq_queue above,
627 662 * this is the only thread aware about the ren existence, so we
628 663 * have the exclusive ownership of it and we do not need to
629 664 * protect it by any lock.
630 665 */
631 666 while ((ran = list_remove_head(&ren->ren_authlist))) {
632 667 uid_t uid;
633 668 gid_t gid;
634 669 uint_t ngids;
635 670 gid_t *gids;
636 671 struct auth_cache *p = ran->ran_auth;
637 672 char *netid = ran->ran_netid;
638 673
639 674 ASSERT(p != NULL);
640 675 ASSERT(netid != NULL);
641 676
642 677 kmem_free(ran, sizeof (refreshq_auth_node_t));
643 678
644 679 mutex_enter(&p->auth_lock);
645 680
646 681 /*
647 682 * Once the entry goes INVALID, it can not change
648 683 * state.
649 684 *
650 685 * No need to refresh entries also in a case we are
651 686 * just shutting down.
652 687 *
653 688 * In general, there is no need to hold the
654 689 * refreshq_lock to test the refreshq_thread_state. We
655 690 * do hold it at other places because there is some
656 691 * related thread synchronization (or some other tasks)
|
↓ open down ↓ |
26 lines elided |
↑ open up ↑ |
657 692 * close to the refreshq_thread_state check.
658 693 *
659 694 * The check for the refreshq_thread_state value here
660 695 * is purely advisory to allow the faster
661 696 * nfsauth_refresh_thread() shutdown. In a case we
662 697 * will miss such advisory, nothing catastrophic
663 698 * happens: we will just spin longer here before the
664 699 * shutdown.
665 700 */
666 701 if (p->auth_state == NFS_AUTH_INVALID ||
667 - refreshq_thread_state != REFRESHQ_THREAD_RUNNING) {
702 + nag->refreshq_thread_state !=
703 + REFRESHQ_THREAD_RUNNING) {
668 704 mutex_exit(&p->auth_lock);
669 705
670 706 if (p->auth_state == NFS_AUTH_INVALID)
671 707 nfsauth_free_node(p);
672 708
673 709 strfree(netid);
674 710
675 711 continue;
676 712 }
677 713
678 714 /*
679 715 * Make sure the state is valid. Note that once we
680 716 * change the state to NFS_AUTH_REFRESHING, no other
681 717 * thread will be able to work on this entry.
682 718 */
683 719 ASSERT(p->auth_state == NFS_AUTH_STALE);
684 720
685 721 p->auth_state = NFS_AUTH_REFRESHING;
686 722 mutex_exit(&p->auth_lock);
687 723
688 724 DTRACE_PROBE2(nfsauth__debug__cache__refresh,
689 725 struct exportinfo *, exi,
690 726 struct auth_cache *, p);
691 727
|
↓ open down ↓ |
14 lines elided |
↑ open up ↑ |
692 728 /*
693 729 * The first caching of the access rights
694 730 * is done with the netid pulled out of the
695 731 * request from the client. All subsequent
696 732 * users of the cache may or may not have
697 733 * the same netid. It doesn't matter. So
698 734 * when we refresh, we simply use the netid
699 735 * of the request which triggered the
700 736 * refresh attempt.
701 737 */
702 - retrieval = nfsauth_retrieve(exi, netid,
738 + retrieval = nfsauth_retrieve(nag, exi, netid,
703 739 p->auth_flavor, &p->auth_clnt->authc_addr, &access,
704 740 p->auth_clnt_cred, &uid, &gid, &ngids, &gids);
705 741
706 742 /*
707 743 * This can only be set in one other place
708 744 * and the state has to be NFS_AUTH_FRESH.
709 745 */
710 746 strfree(netid);
711 747
712 748 mutex_enter(&p->auth_lock);
713 749 if (p->auth_state == NFS_AUTH_INVALID) {
714 750 mutex_exit(&p->auth_lock);
715 751 nfsauth_free_node(p);
716 752 if (retrieval == TRUE)
717 753 kmem_free(gids, ngids * sizeof (gid_t));
718 754 } else {
719 755 /*
720 756 * If we got an error, do not reset the
721 757 * time. This will cause the next access
722 758 * check for the client to reschedule this
723 759 * node.
724 760 */
725 761 if (retrieval == TRUE) {
726 762 p->auth_access = access;
727 763
728 764 p->auth_srv_uid = uid;
729 765 p->auth_srv_gid = gid;
730 766 kmem_free(p->auth_srv_gids,
731 767 p->auth_srv_ngids * sizeof (gid_t));
732 768 p->auth_srv_ngids = ngids;
733 769 p->auth_srv_gids = gids;
734 770
|
↓ open down ↓ |
22 lines elided |
↑ open up ↑ |
735 771 p->auth_freshness = gethrestime_sec();
736 772 }
737 773 p->auth_state = NFS_AUTH_FRESH;
738 774
739 775 cv_broadcast(&p->auth_cv);
740 776 mutex_exit(&p->auth_lock);
741 777 }
742 778 }
743 779
744 780 list_destroy(&ren->ren_authlist);
745 - exi_rele(ren->ren_exi);
781 + exi_rele(&ren->ren_exi);
746 782 kmem_free(ren, sizeof (refreshq_exi_node_t));
747 783 }
748 784
749 - refreshq_thread_state = REFRESHQ_THREAD_HALTED;
750 - cv_broadcast(&refreshq_cv);
785 + nag->refreshq_thread_state = REFRESHQ_THREAD_HALTED;
786 + cv_broadcast(&nag->refreshq_cv);
751 787 CALLB_CPR_EXIT(&cprinfo);
788 + DTRACE_PROBE(nfsauth__nfsauth__refresh__thread__exit);
752 789 zthread_exit();
753 790 }
754 791
755 792 int
756 793 nfsauth_cache_clnt_compar(const void *v1, const void *v2)
757 794 {
758 795 int c;
759 796
760 797 const struct auth_cache_clnt *a1 = (const struct auth_cache_clnt *)v1;
761 798 const struct auth_cache_clnt *a2 = (const struct auth_cache_clnt *)v2;
762 799
763 800 if (a1->authc_addr.len < a2->authc_addr.len)
764 801 return (-1);
765 802 if (a1->authc_addr.len > a2->authc_addr.len)
766 803 return (1);
767 804
768 805 c = memcmp(a1->authc_addr.buf, a2->authc_addr.buf, a1->authc_addr.len);
769 806 if (c < 0)
770 807 return (-1);
771 808 if (c > 0)
772 809 return (1);
773 810
774 811 return (0);
775 812 }
776 813
777 814 static int
778 815 nfsauth_cache_compar(const void *v1, const void *v2)
779 816 {
780 817 int c;
781 818
782 819 const struct auth_cache *a1 = (const struct auth_cache *)v1;
783 820 const struct auth_cache *a2 = (const struct auth_cache *)v2;
784 821
785 822 if (a1->auth_flavor < a2->auth_flavor)
786 823 return (-1);
787 824 if (a1->auth_flavor > a2->auth_flavor)
788 825 return (1);
789 826
790 827 if (crgetuid(a1->auth_clnt_cred) < crgetuid(a2->auth_clnt_cred))
791 828 return (-1);
792 829 if (crgetuid(a1->auth_clnt_cred) > crgetuid(a2->auth_clnt_cred))
793 830 return (1);
794 831
795 832 if (crgetgid(a1->auth_clnt_cred) < crgetgid(a2->auth_clnt_cred))
796 833 return (-1);
797 834 if (crgetgid(a1->auth_clnt_cred) > crgetgid(a2->auth_clnt_cred))
798 835 return (1);
799 836
800 837 if (crgetngroups(a1->auth_clnt_cred) < crgetngroups(a2->auth_clnt_cred))
801 838 return (-1);
802 839 if (crgetngroups(a1->auth_clnt_cred) > crgetngroups(a2->auth_clnt_cred))
803 840 return (1);
804 841
805 842 c = memcmp(crgetgroups(a1->auth_clnt_cred),
806 843 crgetgroups(a2->auth_clnt_cred), crgetngroups(a1->auth_clnt_cred));
807 844 if (c < 0)
808 845 return (-1);
809 846 if (c > 0)
810 847 return (1);
811 848
812 849 return (0);
|
↓ open down ↓ |
51 lines elided |
↑ open up ↑ |
813 850 }
814 851
815 852 /*
816 853 * Get the access information from the cache or callup to the mountd
817 854 * to get and cache the access information in the kernel.
818 855 */
819 856 static int
820 857 nfsauth_cache_get(struct exportinfo *exi, struct svc_req *req, int flavor,
821 858 cred_t *cr, uid_t *uid, gid_t *gid, uint_t *ngids, gid_t **gids)
822 859 {
860 + nfsauth_globals_t *nag;
823 861 struct netbuf *taddrmask;
824 862 struct netbuf addr; /* temporary copy of client's address */
825 863 const struct netbuf *claddr;
826 864 avl_tree_t *tree;
827 865 struct auth_cache ac; /* used as a template for avl_find() */
828 866 struct auth_cache_clnt *c;
829 867 struct auth_cache_clnt acc; /* used as a template for avl_find() */
830 868 struct auth_cache *p = NULL;
831 869 int access;
832 870
833 871 uid_t tmpuid;
834 872 gid_t tmpgid;
835 873 uint_t tmpngids;
836 874 gid_t *tmpgids;
837 875
838 876 avl_index_t where; /* used for avl_find()/avl_insert() */
839 877
840 878 ASSERT(cr != NULL);
841 879
880 + nag = zone_getspecific(nfsauth_zone_key, curzone);
881 +
842 882 /*
843 883 * Now check whether this client already
844 884 * has an entry for this flavor in the cache
845 885 * for this export.
846 886 * Get the caller's address, mask off the
847 887 * parts of the address that do not identify
848 888 * the host (port number, etc), and then hash
849 889 * it to find the chain of cache entries.
850 890 */
851 891
852 892 claddr = svc_getrpccaller(req->rq_xprt);
853 893 addr = *claddr;
854 894 addr.buf = kmem_alloc(addr.maxlen, KM_SLEEP);
855 895 bcopy(claddr->buf, addr.buf, claddr->len);
856 896
857 897 SVC_GETADDRMASK(req->rq_xprt, SVC_TATTR_ADDRMASK, (void **)&taddrmask);
858 898 ASSERT(taddrmask != NULL);
859 899 addrmask(&addr, taddrmask);
860 900
861 901 ac.auth_flavor = flavor;
862 902 ac.auth_clnt_cred = crdup(cr);
863 903
864 904 acc.authc_addr = addr;
865 905
866 906 tree = exi->exi_cache[hash(&addr)];
867 907
868 908 rw_enter(&exi->exi_cache_lock, RW_READER);
869 909 c = (struct auth_cache_clnt *)avl_find(tree, &acc, NULL);
870 910
871 911 if (c == NULL) {
872 912 struct auth_cache_clnt *nc;
873 913
874 914 rw_exit(&exi->exi_cache_lock);
875 915
876 916 nc = kmem_alloc(sizeof (*nc), KM_NOSLEEP | KM_NORMALPRI);
877 917 if (nc == NULL)
878 918 goto retrieve;
879 919
880 920 /*
881 921 * Initialize the new auth_cache_clnt
882 922 */
883 923 nc->authc_addr = addr;
884 924 nc->authc_addr.buf = kmem_alloc(addr.maxlen,
885 925 KM_NOSLEEP | KM_NORMALPRI);
886 926 if (addr.maxlen != 0 && nc->authc_addr.buf == NULL) {
887 927 kmem_free(nc, sizeof (*nc));
888 928 goto retrieve;
889 929 }
890 930 bcopy(addr.buf, nc->authc_addr.buf, addr.len);
891 931 rw_init(&nc->authc_lock, NULL, RW_DEFAULT, NULL);
892 932 avl_create(&nc->authc_tree, nfsauth_cache_compar,
893 933 sizeof (struct auth_cache),
894 934 offsetof(struct auth_cache, auth_link));
895 935
896 936 rw_enter(&exi->exi_cache_lock, RW_WRITER);
897 937 c = (struct auth_cache_clnt *)avl_find(tree, &acc, &where);
898 938 if (c == NULL) {
899 939 avl_insert(tree, nc, where);
900 940 rw_downgrade(&exi->exi_cache_lock);
901 941 c = nc;
902 942 } else {
903 943 rw_downgrade(&exi->exi_cache_lock);
904 944
905 945 avl_destroy(&nc->authc_tree);
906 946 rw_destroy(&nc->authc_lock);
907 947 kmem_free(nc->authc_addr.buf, nc->authc_addr.maxlen);
908 948 kmem_free(nc, sizeof (*nc));
909 949 }
910 950 }
911 951
912 952 ASSERT(c != NULL);
913 953
914 954 rw_enter(&c->authc_lock, RW_READER);
915 955 p = (struct auth_cache *)avl_find(&c->authc_tree, &ac, NULL);
916 956
917 957 if (p == NULL) {
918 958 struct auth_cache *np;
919 959
920 960 rw_exit(&c->authc_lock);
921 961
922 962 np = kmem_cache_alloc(exi_cache_handle,
923 963 KM_NOSLEEP | KM_NORMALPRI);
924 964 if (np == NULL) {
925 965 rw_exit(&exi->exi_cache_lock);
926 966 goto retrieve;
927 967 }
928 968
929 969 /*
930 970 * Initialize the new auth_cache
931 971 */
932 972 np->auth_clnt = c;
933 973 np->auth_flavor = flavor;
934 974 np->auth_clnt_cred = ac.auth_clnt_cred;
935 975 np->auth_srv_ngids = 0;
936 976 np->auth_srv_gids = NULL;
937 977 np->auth_time = np->auth_freshness = gethrestime_sec();
938 978 np->auth_state = NFS_AUTH_NEW;
939 979 mutex_init(&np->auth_lock, NULL, MUTEX_DEFAULT, NULL);
940 980 cv_init(&np->auth_cv, NULL, CV_DEFAULT, NULL);
941 981
942 982 rw_enter(&c->authc_lock, RW_WRITER);
943 983 rw_exit(&exi->exi_cache_lock);
944 984
945 985 p = (struct auth_cache *)avl_find(&c->authc_tree, &ac, &where);
946 986 if (p == NULL) {
947 987 avl_insert(&c->authc_tree, np, where);
948 988 rw_downgrade(&c->authc_lock);
949 989 p = np;
950 990 } else {
951 991 rw_downgrade(&c->authc_lock);
952 992
953 993 cv_destroy(&np->auth_cv);
954 994 mutex_destroy(&np->auth_lock);
955 995 crfree(ac.auth_clnt_cred);
956 996 kmem_cache_free(exi_cache_handle, np);
957 997 }
958 998 } else {
959 999 rw_exit(&exi->exi_cache_lock);
960 1000 crfree(ac.auth_clnt_cred);
961 1001 }
962 1002
963 1003 mutex_enter(&p->auth_lock);
964 1004 rw_exit(&c->authc_lock);
965 1005
966 1006 /*
967 1007 * If the entry is in the WAITING state then some other thread is just
968 1008 * retrieving the required info. The entry was either NEW, or the list
969 1009 * of client's supplemental groups is going to be changed (either by
970 1010 * this thread, or by some other thread). We need to wait until the
971 1011 * nfsauth_retrieve() is done.
972 1012 */
973 1013 while (p->auth_state == NFS_AUTH_WAITING)
974 1014 cv_wait(&p->auth_cv, &p->auth_lock);
975 1015
976 1016 /*
977 1017 * Here the entry cannot be in WAITING or INVALID state.
978 1018 */
979 1019 ASSERT(p->auth_state != NFS_AUTH_WAITING);
980 1020 ASSERT(p->auth_state != NFS_AUTH_INVALID);
981 1021
982 1022 /*
983 1023 * If the cache entry is not valid yet, we need to retrieve the
984 1024 * info ourselves.
985 1025 */
986 1026 if (p->auth_state == NFS_AUTH_NEW) {
987 1027 bool_t res;
988 1028 /*
989 1029 * NFS_AUTH_NEW is the default output auth_state value in a
990 1030 * case we failed somewhere below.
|
↓ open down ↓ |
139 lines elided |
↑ open up ↑ |
991 1031 */
992 1032 auth_state_t state = NFS_AUTH_NEW;
993 1033
994 1034 p->auth_state = NFS_AUTH_WAITING;
995 1035 mutex_exit(&p->auth_lock);
996 1036 kmem_free(addr.buf, addr.maxlen);
997 1037 addr = p->auth_clnt->authc_addr;
998 1038
999 1039 atomic_inc_uint(&nfsauth_cache_miss);
1000 1040
1001 - res = nfsauth_retrieve(exi, svc_getnetid(req->rq_xprt), flavor,
1002 - &addr, &access, cr, &tmpuid, &tmpgid, &tmpngids, &tmpgids);
1041 + res = nfsauth_retrieve(nag, exi, svc_getnetid(req->rq_xprt),
1042 + flavor, &addr, &access, cr, &tmpuid, &tmpgid, &tmpngids,
1043 + &tmpgids);
1003 1044
1004 1045 p->auth_access = access;
1005 1046 p->auth_time = p->auth_freshness = gethrestime_sec();
1006 1047
1007 1048 if (res == TRUE) {
1008 1049 if (uid != NULL)
1009 1050 *uid = tmpuid;
1010 1051 if (gid != NULL)
1011 1052 *gid = tmpgid;
1012 1053 if (ngids != NULL && gids != NULL) {
1013 1054 *ngids = tmpngids;
1014 1055 *gids = tmpgids;
1015 1056
1016 1057 /*
1017 1058 * We need a copy of gids for the
1018 1059 * auth_cache entry
1019 1060 */
1020 1061 tmpgids = kmem_alloc(tmpngids * sizeof (gid_t),
1021 1062 KM_NOSLEEP | KM_NORMALPRI);
1022 1063 if (tmpgids != NULL)
1023 1064 bcopy(*gids, tmpgids,
1024 1065 tmpngids * sizeof (gid_t));
1025 1066 }
1026 1067
1027 1068 if (tmpgids != NULL || tmpngids == 0) {
1028 1069 p->auth_srv_uid = tmpuid;
1029 1070 p->auth_srv_gid = tmpgid;
1030 1071 p->auth_srv_ngids = tmpngids;
1031 1072 p->auth_srv_gids = tmpgids;
1032 1073
1033 1074 state = NFS_AUTH_FRESH;
1034 1075 }
1035 1076 }
1036 1077
1037 1078 /*
1038 1079 * Set the auth_state and notify waiters.
1039 1080 */
1040 1081 mutex_enter(&p->auth_lock);
1041 1082 p->auth_state = state;
1042 1083 cv_broadcast(&p->auth_cv);
1043 1084 mutex_exit(&p->auth_lock);
1044 1085 } else {
1045 1086 uint_t nach;
1046 1087 time_t refresh;
1047 1088
1048 1089 refresh = gethrestime_sec() - p->auth_freshness;
1049 1090
1050 1091 p->auth_time = gethrestime_sec();
1051 1092
1052 1093 if (uid != NULL)
1053 1094 *uid = p->auth_srv_uid;
1054 1095 if (gid != NULL)
1055 1096 *gid = p->auth_srv_gid;
1056 1097 if (ngids != NULL && gids != NULL) {
1057 1098 *ngids = p->auth_srv_ngids;
1058 1099 *gids = kmem_alloc(*ngids * sizeof (gid_t), KM_SLEEP);
1059 1100 bcopy(p->auth_srv_gids, *gids, *ngids * sizeof (gid_t));
1060 1101 }
1061 1102
1062 1103 access = p->auth_access;
1063 1104
1064 1105 if ((refresh > NFSAUTH_CACHE_REFRESH) &&
1065 1106 p->auth_state == NFS_AUTH_FRESH) {
1066 1107 refreshq_auth_node_t *ran;
1067 1108 uint_t nacr;
1068 1109
1069 1110 p->auth_state = NFS_AUTH_STALE;
1070 1111 mutex_exit(&p->auth_lock);
1071 1112
1072 1113 nacr = atomic_inc_uint_nv(&nfsauth_cache_refresh);
|
↓ open down ↓ |
60 lines elided |
↑ open up ↑ |
1073 1114 DTRACE_PROBE3(nfsauth__debug__cache__stale,
1074 1115 struct exportinfo *, exi,
1075 1116 struct auth_cache *, p,
1076 1117 uint_t, nacr);
1077 1118
1078 1119 ran = kmem_alloc(sizeof (refreshq_auth_node_t),
1079 1120 KM_SLEEP);
1080 1121 ran->ran_auth = p;
1081 1122 ran->ran_netid = strdup(svc_getnetid(req->rq_xprt));
1082 1123
1083 - mutex_enter(&refreshq_lock);
1124 + mutex_enter(&nag->refreshq_lock);
1125 +
1126 + if (nag->refreshq_thread_state ==
1127 + REFRESHQ_THREAD_NEED_CREATE) {
1128 + /* Launch nfsauth refresh thread */
1129 + nag->refreshq_thread_state =
1130 + REFRESHQ_THREAD_RUNNING;
1131 + (void) zthread_create(NULL, 0,
1132 + nfsauth_refresh_thread, nag, 0,
1133 + minclsyspri);
1134 + }
1135 +
1084 1136 /*
1085 - * We should not add a work queue
1086 - * item if the thread is not
1087 - * accepting them.
1137 + * We should not add a work queue item if the thread
1138 + * is not accepting them.
1088 1139 */
1089 - if (refreshq_thread_state == REFRESHQ_THREAD_RUNNING) {
1140 + if (nag->refreshq_thread_state ==
1141 + REFRESHQ_THREAD_RUNNING) {
1090 1142 refreshq_exi_node_t *ren;
1091 1143
1092 1144 /*
1093 1145 * Is there an existing exi_list?
1094 1146 */
1095 - for (ren = list_head(&refreshq_queue);
1147 + for (ren = list_head(&nag->refreshq_queue);
1096 1148 ren != NULL;
1097 - ren = list_next(&refreshq_queue, ren)) {
1149 + ren = list_next(&nag->refreshq_queue,
1150 + ren)) {
1098 1151 if (ren->ren_exi == exi) {
1099 1152 list_insert_tail(
1100 1153 &ren->ren_authlist, ran);
1101 1154 break;
1102 1155 }
1103 1156 }
1104 1157
1105 1158 if (ren == NULL) {
1106 1159 ren = kmem_alloc(
1107 1160 sizeof (refreshq_exi_node_t),
1108 1161 KM_SLEEP);
1109 1162
|
↓ open down ↓ |
2 lines elided |
↑ open up ↑ |
1110 1163 exi_hold(exi);
1111 1164 ren->ren_exi = exi;
1112 1165
1113 1166 list_create(&ren->ren_authlist,
1114 1167 sizeof (refreshq_auth_node_t),
1115 1168 offsetof(refreshq_auth_node_t,
1116 1169 ran_node));
1117 1170
1118 1171 list_insert_tail(&ren->ren_authlist,
1119 1172 ran);
1120 - list_insert_tail(&refreshq_queue, ren);
1173 + list_insert_tail(&nag->refreshq_queue,
1174 + ren);
1121 1175 }
1122 1176
1123 - cv_broadcast(&refreshq_cv);
1177 + cv_broadcast(&nag->refreshq_cv);
1124 1178 } else {
1125 1179 strfree(ran->ran_netid);
1126 1180 kmem_free(ran, sizeof (refreshq_auth_node_t));
1127 1181 }
1128 1182
1129 - mutex_exit(&refreshq_lock);
1183 + mutex_exit(&nag->refreshq_lock);
1130 1184 } else {
1131 1185 mutex_exit(&p->auth_lock);
1132 1186 }
1133 1187
1134 1188 nach = atomic_inc_uint_nv(&nfsauth_cache_hit);
1135 1189 DTRACE_PROBE2(nfsauth__debug__cache__hit,
1136 1190 uint_t, nach,
1137 1191 time_t, refresh);
1138 1192
1139 1193 kmem_free(addr.buf, addr.maxlen);
1140 1194 }
1141 1195
1142 1196 return (access);
1143 1197
1144 1198 retrieve:
|
↓ open down ↓ |
5 lines elided |
↑ open up ↑ |
1145 1199 crfree(ac.auth_clnt_cred);
1146 1200
1147 1201 /*
1148 1202 * Retrieve the required data without caching.
1149 1203 */
1150 1204
1151 1205 ASSERT(p == NULL);
1152 1206
1153 1207 atomic_inc_uint(&nfsauth_cache_miss);
1154 1208
1155 - if (nfsauth_retrieve(exi, svc_getnetid(req->rq_xprt), flavor, &addr,
1156 - &access, cr, &tmpuid, &tmpgid, &tmpngids, &tmpgids)) {
1209 + if (nfsauth_retrieve(nag, exi, svc_getnetid(req->rq_xprt), flavor,
1210 + &addr, &access, cr, &tmpuid, &tmpgid, &tmpngids, &tmpgids)) {
1157 1211 if (uid != NULL)
1158 1212 *uid = tmpuid;
1159 1213 if (gid != NULL)
1160 1214 *gid = tmpgid;
1161 1215 if (ngids != NULL && gids != NULL) {
1162 1216 *ngids = tmpngids;
1163 1217 *gids = tmpgids;
1164 1218 } else {
1165 1219 kmem_free(tmpgids, tmpngids * sizeof (gid_t));
1166 1220 }
1167 1221 }
1168 1222
1169 1223 kmem_free(addr.buf, addr.maxlen);
1170 1224
1171 1225 return (access);
1172 1226 }
1173 1227
1174 1228 /*
1175 1229 * Check if the requesting client has access to the filesystem with
1176 1230 * a given nfs flavor number which is an explicitly shared flavor.
1177 1231 */
1178 1232 int
1179 1233 nfsauth4_secinfo_access(struct exportinfo *exi, struct svc_req *req,
1180 1234 int flavor, int perm, cred_t *cr)
1181 1235 {
1182 1236 int access;
1183 1237
1184 1238 if (! (perm & M_4SEC_EXPORTED)) {
1185 1239 return (NFSAUTH_DENIED);
1186 1240 }
1187 1241
1188 1242 /*
1189 1243 * Optimize if there are no lists
1190 1244 */
1191 1245 if ((perm & (M_ROOT | M_NONE | M_MAP)) == 0) {
1192 1246 perm &= ~M_4SEC_EXPORTED;
1193 1247 if (perm == M_RO)
1194 1248 return (NFSAUTH_RO);
1195 1249 if (perm == M_RW)
1196 1250 return (NFSAUTH_RW);
1197 1251 }
1198 1252
1199 1253 access = nfsauth_cache_get(exi, req, flavor, cr, NULL, NULL, NULL,
1200 1254 NULL);
1201 1255
1202 1256 return (access);
1203 1257 }
1204 1258
1205 1259 int
1206 1260 nfsauth_access(struct exportinfo *exi, struct svc_req *req, cred_t *cr,
1207 1261 uid_t *uid, gid_t *gid, uint_t *ngids, gid_t **gids)
1208 1262 {
1209 1263 int access, mapaccess;
1210 1264 struct secinfo *sp;
1211 1265 int i, flavor, perm;
1212 1266 int authnone_entry = -1;
1213 1267
1214 1268 /*
1215 1269 * By default root is mapped to anonymous user.
1216 1270 * This might get overriden later in nfsauth_cache_get().
1217 1271 */
1218 1272 if (crgetuid(cr) == 0) {
1219 1273 if (uid != NULL)
1220 1274 *uid = exi->exi_export.ex_anon;
1221 1275 if (gid != NULL)
1222 1276 *gid = exi->exi_export.ex_anon;
1223 1277 } else {
1224 1278 if (uid != NULL)
1225 1279 *uid = crgetuid(cr);
1226 1280 if (gid != NULL)
1227 1281 *gid = crgetgid(cr);
1228 1282 }
1229 1283
1230 1284 if (ngids != NULL)
1231 1285 *ngids = 0;
1232 1286 if (gids != NULL)
1233 1287 *gids = NULL;
1234 1288
1235 1289 /*
1236 1290 * Get the nfs flavor number from xprt.
1237 1291 */
1238 1292 flavor = (int)(uintptr_t)req->rq_xprt->xp_cookie;
1239 1293
1240 1294 /*
1241 1295 * First check the access restrictions on the filesystem. If
1242 1296 * there are no lists associated with this flavor then there's no
1243 1297 * need to make an expensive call to the nfsauth service or to
1244 1298 * cache anything.
1245 1299 */
1246 1300
1247 1301 sp = exi->exi_export.ex_secinfo;
1248 1302 for (i = 0; i < exi->exi_export.ex_seccnt; i++) {
1249 1303 if (flavor != sp[i].s_secinfo.sc_nfsnum) {
1250 1304 if (sp[i].s_secinfo.sc_nfsnum == AUTH_NONE)
1251 1305 authnone_entry = i;
1252 1306 continue;
1253 1307 }
1254 1308 break;
1255 1309 }
1256 1310
1257 1311 mapaccess = 0;
1258 1312
1259 1313 if (i >= exi->exi_export.ex_seccnt) {
1260 1314 /*
1261 1315 * Flavor not found, but use AUTH_NONE if it exists
1262 1316 */
1263 1317 if (authnone_entry == -1)
1264 1318 return (NFSAUTH_DENIED);
1265 1319 flavor = AUTH_NONE;
1266 1320 mapaccess = NFSAUTH_MAPNONE;
1267 1321 i = authnone_entry;
1268 1322 }
1269 1323
1270 1324 /*
1271 1325 * If the flavor is in the ex_secinfo list, but not an explicitly
1272 1326 * shared flavor by the user, it is a result of the nfsv4 server
1273 1327 * namespace setup. We will grant an RO permission similar for
1274 1328 * a pseudo node except that this node is a shared one.
1275 1329 *
1276 1330 * e.g. flavor in (flavor) indicates that it is not explictly
1277 1331 * shared by the user:
1278 1332 *
1279 1333 * / (sys, krb5)
1280 1334 * |
1281 1335 * export #share -o sec=sys (krb5)
1282 1336 * |
1283 1337 * secure #share -o sec=krb5
1284 1338 *
1285 1339 * In this case, when a krb5 request coming in to access
1286 1340 * /export, RO permission is granted.
1287 1341 */
1288 1342 if (!(sp[i].s_flags & M_4SEC_EXPORTED))
1289 1343 return (mapaccess | NFSAUTH_RO);
1290 1344
1291 1345 /*
1292 1346 * Optimize if there are no lists.
1293 1347 * We cannot optimize for AUTH_SYS with NGRPS (16) supplemental groups.
1294 1348 */
1295 1349 perm = sp[i].s_flags;
1296 1350 if ((perm & (M_ROOT | M_NONE | M_MAP)) == 0 && (ngroups_max <= NGRPS ||
1297 1351 flavor != AUTH_SYS || crgetngroups(cr) < NGRPS)) {
1298 1352 perm &= ~M_4SEC_EXPORTED;
1299 1353 if (perm == M_RO)
1300 1354 return (mapaccess | NFSAUTH_RO);
1301 1355 if (perm == M_RW)
1302 1356 return (mapaccess | NFSAUTH_RW);
1303 1357 }
1304 1358
1305 1359 access = nfsauth_cache_get(exi, req, flavor, cr, uid, gid, ngids, gids);
1306 1360
1307 1361 /*
1308 1362 * For both NFSAUTH_DENIED and NFSAUTH_WRONGSEC we do not care about
1309 1363 * the supplemental groups.
1310 1364 */
1311 1365 if (access & NFSAUTH_DENIED || access & NFSAUTH_WRONGSEC) {
1312 1366 if (ngids != NULL && gids != NULL) {
1313 1367 kmem_free(*gids, *ngids * sizeof (gid_t));
1314 1368 *ngids = 0;
1315 1369 *gids = NULL;
1316 1370 }
1317 1371 }
1318 1372
1319 1373 /*
1320 1374 * Client's security flavor doesn't match with "ro" or
1321 1375 * "rw" list. Try again using AUTH_NONE if present.
1322 1376 */
1323 1377 if ((access & NFSAUTH_WRONGSEC) && (flavor != AUTH_NONE)) {
1324 1378 /*
1325 1379 * Have we already encountered AUTH_NONE ?
1326 1380 */
1327 1381 if (authnone_entry != -1) {
1328 1382 mapaccess = NFSAUTH_MAPNONE;
1329 1383 access = nfsauth_cache_get(exi, req, AUTH_NONE, cr,
1330 1384 NULL, NULL, NULL, NULL);
1331 1385 } else {
1332 1386 /*
1333 1387 * Check for AUTH_NONE presence.
1334 1388 */
1335 1389 for (; i < exi->exi_export.ex_seccnt; i++) {
1336 1390 if (sp[i].s_secinfo.sc_nfsnum == AUTH_NONE) {
1337 1391 mapaccess = NFSAUTH_MAPNONE;
1338 1392 access = nfsauth_cache_get(exi, req,
1339 1393 AUTH_NONE, cr, NULL, NULL, NULL,
1340 1394 NULL);
1341 1395 break;
1342 1396 }
1343 1397 }
1344 1398 }
1345 1399 }
1346 1400
1347 1401 if (access & NFSAUTH_DENIED)
1348 1402 access = NFSAUTH_DENIED;
1349 1403
1350 1404 return (access | mapaccess);
1351 1405 }
1352 1406
1353 1407 static void
1354 1408 nfsauth_free_clnt_node(struct auth_cache_clnt *p)
1355 1409 {
1356 1410 void *cookie = NULL;
1357 1411 struct auth_cache *node;
1358 1412
1359 1413 while ((node = avl_destroy_nodes(&p->authc_tree, &cookie)) != NULL)
1360 1414 nfsauth_free_node(node);
1361 1415 avl_destroy(&p->authc_tree);
1362 1416
1363 1417 kmem_free(p->authc_addr.buf, p->authc_addr.maxlen);
1364 1418 rw_destroy(&p->authc_lock);
1365 1419
1366 1420 kmem_free(p, sizeof (*p));
1367 1421 }
1368 1422
1369 1423 static void
1370 1424 nfsauth_free_node(struct auth_cache *p)
1371 1425 {
1372 1426 crfree(p->auth_clnt_cred);
1373 1427 kmem_free(p->auth_srv_gids, p->auth_srv_ngids * sizeof (gid_t));
1374 1428 mutex_destroy(&p->auth_lock);
1375 1429 cv_destroy(&p->auth_cv);
1376 1430 kmem_cache_free(exi_cache_handle, p);
1377 1431 }
1378 1432
1379 1433 /*
1380 1434 * Free the nfsauth cache for a given export
1381 1435 */
1382 1436 void
1383 1437 nfsauth_cache_free(struct exportinfo *exi)
1384 1438 {
1385 1439 int i;
1386 1440
1387 1441 /*
1388 1442 * The only way we got here was with an exi_rele, which means that no
1389 1443 * auth cache entry is being refreshed.
1390 1444 */
1391 1445
1392 1446 for (i = 0; i < AUTH_TABLESIZE; i++) {
1393 1447 avl_tree_t *tree = exi->exi_cache[i];
1394 1448 void *cookie = NULL;
1395 1449 struct auth_cache_clnt *node;
1396 1450
1397 1451 while ((node = avl_destroy_nodes(tree, &cookie)) != NULL)
1398 1452 nfsauth_free_clnt_node(node);
1399 1453 }
1400 1454 }
1401 1455
1402 1456 /*
1403 1457 * Called by the kernel memory allocator when
|
↓ open down ↓ |
237 lines elided |
↑ open up ↑ |
1404 1458 * memory is low. Free unused cache entries.
1405 1459 * If that's not enough, the VM system will
1406 1460 * call again for some more.
1407 1461 */
1408 1462 /*ARGSUSED*/
1409 1463 void
1410 1464 exi_cache_reclaim(void *cdrarg)
1411 1465 {
1412 1466 int i;
1413 1467 struct exportinfo *exi;
1468 + nfs_export_t *ne = nfs_get_export();
1414 1469
1415 - rw_enter(&exported_lock, RW_READER);
1470 + rw_enter(&ne->exported_lock, RW_READER);
1416 1471
1417 1472 for (i = 0; i < EXPTABLESIZE; i++) {
1418 - for (exi = exptable[i]; exi; exi = exi->fid_hash.next) {
1473 + for (exi = ne->exptable[i]; exi; exi = exi->fid_hash.next)
1419 1474 exi_cache_trim(exi);
1420 - }
1421 1475 }
1422 1476
1423 - rw_exit(&exported_lock);
1477 + rw_exit(&ne->exported_lock);
1424 1478
1425 1479 atomic_inc_uint(&nfsauth_cache_reclaim);
1426 1480 }
1427 1481
1428 1482 void
1429 1483 exi_cache_trim(struct exportinfo *exi)
1430 1484 {
1431 1485 struct auth_cache_clnt *c;
1432 1486 struct auth_cache_clnt *nextc;
1433 1487 struct auth_cache *p;
1434 1488 struct auth_cache *next;
1435 1489 int i;
1436 1490 time_t stale_time;
1437 1491 avl_tree_t *tree;
1438 1492
1439 1493 for (i = 0; i < AUTH_TABLESIZE; i++) {
1440 1494 tree = exi->exi_cache[i];
1441 1495 stale_time = gethrestime_sec() - NFSAUTH_CACHE_TRIM;
1442 1496 rw_enter(&exi->exi_cache_lock, RW_READER);
1443 1497
1444 1498 /*
1445 1499 * Free entries that have not been
1446 1500 * used for NFSAUTH_CACHE_TRIM seconds.
1447 1501 */
1448 1502 for (c = avl_first(tree); c != NULL; c = AVL_NEXT(tree, c)) {
1449 1503 /*
1450 1504 * We are being called by the kmem subsystem to reclaim
1451 1505 * memory so don't block if we can't get the lock.
1452 1506 */
1453 1507 if (rw_tryenter(&c->authc_lock, RW_WRITER) == 0) {
1454 1508 exi_cache_auth_reclaim_failed++;
1455 1509 rw_exit(&exi->exi_cache_lock);
1456 1510 return;
1457 1511 }
1458 1512
1459 1513 for (p = avl_first(&c->authc_tree); p != NULL;
1460 1514 p = next) {
1461 1515 next = AVL_NEXT(&c->authc_tree, p);
1462 1516
1463 1517 ASSERT(p->auth_state != NFS_AUTH_INVALID);
1464 1518
1465 1519 mutex_enter(&p->auth_lock);
1466 1520
1467 1521 /*
1468 1522 * We won't trim recently used and/or WAITING
1469 1523 * entries.
1470 1524 */
1471 1525 if (p->auth_time > stale_time ||
1472 1526 p->auth_state == NFS_AUTH_WAITING) {
1473 1527 mutex_exit(&p->auth_lock);
1474 1528 continue;
1475 1529 }
1476 1530
1477 1531 DTRACE_PROBE1(nfsauth__debug__trim__state,
1478 1532 auth_state_t, p->auth_state);
1479 1533
1480 1534 /*
1481 1535 * STALE and REFRESHING entries needs to be
1482 1536 * marked INVALID only because they are
1483 1537 * referenced by some other structures or
1484 1538 * threads. They will be freed later.
1485 1539 */
1486 1540 if (p->auth_state == NFS_AUTH_STALE ||
1487 1541 p->auth_state == NFS_AUTH_REFRESHING) {
1488 1542 p->auth_state = NFS_AUTH_INVALID;
1489 1543 mutex_exit(&p->auth_lock);
1490 1544
1491 1545 avl_remove(&c->authc_tree, p);
1492 1546 } else {
1493 1547 mutex_exit(&p->auth_lock);
1494 1548
1495 1549 avl_remove(&c->authc_tree, p);
1496 1550 nfsauth_free_node(p);
1497 1551 }
1498 1552 }
1499 1553 rw_exit(&c->authc_lock);
1500 1554 }
1501 1555
1502 1556 if (rw_tryupgrade(&exi->exi_cache_lock) == 0) {
1503 1557 rw_exit(&exi->exi_cache_lock);
1504 1558 exi_cache_clnt_reclaim_failed++;
1505 1559 continue;
1506 1560 }
1507 1561
1508 1562 for (c = avl_first(tree); c != NULL; c = nextc) {
1509 1563 nextc = AVL_NEXT(tree, c);
1510 1564
1511 1565 if (avl_is_empty(&c->authc_tree) == B_FALSE)
1512 1566 continue;
1513 1567
1514 1568 avl_remove(tree, c);
1515 1569
1516 1570 nfsauth_free_clnt_node(c);
1517 1571 }
1518 1572
1519 1573 rw_exit(&exi->exi_cache_lock);
1520 1574 }
1521 1575 }
|
↓ open down ↓ |
88 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX