Print this page
5056 ZFS deadlock on db_mtx and dn_holds
Reviewed by: Will Andrews <willa@spectralogic.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/fs/zfs/dsl_prop.c
+++ new/usr/src/uts/common/fs/zfs/dsl_prop.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21 /*
22 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 23 * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
24 24 * Copyright (c) 2013 Martin Matuska. All rights reserved.
25 25 */
26 26
27 27 #include <sys/zfs_context.h>
28 28 #include <sys/dmu.h>
29 29 #include <sys/dmu_objset.h>
30 30 #include <sys/dmu_tx.h>
31 31 #include <sys/dsl_dataset.h>
32 32 #include <sys/dsl_dir.h>
33 33 #include <sys/dsl_prop.h>
34 34 #include <sys/dsl_synctask.h>
35 35 #include <sys/spa.h>
36 36 #include <sys/zap.h>
37 37 #include <sys/fs/zfs.h>
38 38
39 39 #include "zfs_prop.h"
40 40
41 41 #define ZPROP_INHERIT_SUFFIX "$inherit"
42 42 #define ZPROP_RECVD_SUFFIX "$recvd"
43 43
44 44 static int
45 45 dodefault(const char *propname, int intsz, int numints, void *buf)
46 46 {
47 47 zfs_prop_t prop;
48 48
49 49 /*
50 50 * The setonce properties are read-only, BUT they still
51 51 * have a default value that can be used as the initial
52 52 * value.
53 53 */
54 54 if ((prop = zfs_name_to_prop(propname)) == ZPROP_INVAL ||
55 55 (zfs_prop_readonly(prop) && !zfs_prop_setonce(prop)))
56 56 return (SET_ERROR(ENOENT));
57 57
58 58 if (zfs_prop_get_type(prop) == PROP_TYPE_STRING) {
59 59 if (intsz != 1)
60 60 return (SET_ERROR(EOVERFLOW));
61 61 (void) strncpy(buf, zfs_prop_default_string(prop),
62 62 numints);
63 63 } else {
64 64 if (intsz != 8 || numints < 1)
65 65 return (SET_ERROR(EOVERFLOW));
66 66
67 67 *(uint64_t *)buf = zfs_prop_default_numeric(prop);
68 68 }
69 69
70 70 return (0);
71 71 }
72 72
73 73 int
74 74 dsl_prop_get_dd(dsl_dir_t *dd, const char *propname,
75 75 int intsz, int numints, void *buf, char *setpoint, boolean_t snapshot)
76 76 {
77 77 int err = ENOENT;
78 78 dsl_dir_t *target = dd;
79 79 objset_t *mos = dd->dd_pool->dp_meta_objset;
80 80 zfs_prop_t prop;
81 81 boolean_t inheritable;
82 82 boolean_t inheriting = B_FALSE;
83 83 char *inheritstr;
84 84 char *recvdstr;
85 85
86 86 ASSERT(dsl_pool_config_held(dd->dd_pool));
87 87
88 88 if (setpoint)
89 89 setpoint[0] = '\0';
90 90
91 91 prop = zfs_name_to_prop(propname);
92 92 inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
93 93 inheritstr = kmem_asprintf("%s%s", propname, ZPROP_INHERIT_SUFFIX);
94 94 recvdstr = kmem_asprintf("%s%s", propname, ZPROP_RECVD_SUFFIX);
95 95
96 96 /*
97 97 * Note: dd may become NULL, therefore we shouldn't dereference it
98 98 * after this loop.
99 99 */
100 100 for (; dd != NULL; dd = dd->dd_parent) {
101 101 if (dd != target || snapshot) {
102 102 if (!inheritable)
103 103 break;
104 104 inheriting = B_TRUE;
105 105 }
106 106
107 107 /* Check for a local value. */
108 108 err = zap_lookup(mos, dsl_dir_phys(dd)->dd_props_zapobj,
109 109 propname, intsz, numints, buf);
110 110 if (err != ENOENT) {
111 111 if (setpoint != NULL && err == 0)
112 112 dsl_dir_name(dd, setpoint);
113 113 break;
114 114 }
115 115
116 116 /*
117 117 * Skip the check for a received value if there is an explicit
118 118 * inheritance entry.
119 119 */
120 120 err = zap_contains(mos, dsl_dir_phys(dd)->dd_props_zapobj,
121 121 inheritstr);
122 122 if (err != 0 && err != ENOENT)
123 123 break;
124 124
125 125 if (err == ENOENT) {
126 126 /* Check for a received value. */
127 127 err = zap_lookup(mos, dsl_dir_phys(dd)->dd_props_zapobj,
128 128 recvdstr, intsz, numints, buf);
129 129 if (err != ENOENT) {
130 130 if (setpoint != NULL && err == 0) {
131 131 if (inheriting) {
132 132 dsl_dir_name(dd, setpoint);
133 133 } else {
134 134 (void) strcpy(setpoint,
135 135 ZPROP_SOURCE_VAL_RECVD);
136 136 }
137 137 }
138 138 break;
139 139 }
140 140 }
141 141
142 142 /*
143 143 * If we found an explicit inheritance entry, err is zero even
144 144 * though we haven't yet found the value, so reinitializing err
145 145 * at the end of the loop (instead of at the beginning) ensures
146 146 * that err has a valid post-loop value.
147 147 */
148 148 err = SET_ERROR(ENOENT);
149 149 }
150 150
151 151 if (err == ENOENT)
152 152 err = dodefault(propname, intsz, numints, buf);
153 153
154 154 strfree(inheritstr);
155 155 strfree(recvdstr);
|
↓ open down ↓ |
155 lines elided |
↑ open up ↑ |
156 156
157 157 return (err);
158 158 }
159 159
160 160 int
161 161 dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
162 162 int intsz, int numints, void *buf, char *setpoint)
163 163 {
164 164 zfs_prop_t prop = zfs_name_to_prop(propname);
165 165 boolean_t inheritable;
166 - boolean_t snapshot;
167 166 uint64_t zapobj;
168 167
169 168 ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
170 169 inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
171 - snapshot = dsl_dataset_is_snapshot(ds);
172 170 zapobj = dsl_dataset_phys(ds)->ds_props_obj;
173 171
174 172 if (zapobj != 0) {
175 173 objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
176 174 int err;
177 175
178 - ASSERT(snapshot);
176 + ASSERT(ds->ds_is_snapshot);
179 177
180 178 /* Check for a local value. */
181 179 err = zap_lookup(mos, zapobj, propname, intsz, numints, buf);
182 180 if (err != ENOENT) {
183 181 if (setpoint != NULL && err == 0)
184 182 dsl_dataset_name(ds, setpoint);
185 183 return (err);
186 184 }
187 185
188 186 /*
189 187 * Skip the check for a received value if there is an explicit
190 188 * inheritance entry.
191 189 */
192 190 if (inheritable) {
193 191 char *inheritstr = kmem_asprintf("%s%s", propname,
194 192 ZPROP_INHERIT_SUFFIX);
195 193 err = zap_contains(mos, zapobj, inheritstr);
196 194 strfree(inheritstr);
197 195 if (err != 0 && err != ENOENT)
198 196 return (err);
199 197 }
200 198
201 199 if (err == ENOENT) {
202 200 /* Check for a received value. */
203 201 char *recvdstr = kmem_asprintf("%s%s", propname,
204 202 ZPROP_RECVD_SUFFIX);
205 203 err = zap_lookup(mos, zapobj, recvdstr,
206 204 intsz, numints, buf);
207 205 strfree(recvdstr);
|
↓ open down ↓ |
19 lines elided |
↑ open up ↑ |
208 206 if (err != ENOENT) {
209 207 if (setpoint != NULL && err == 0)
210 208 (void) strcpy(setpoint,
211 209 ZPROP_SOURCE_VAL_RECVD);
212 210 return (err);
213 211 }
214 212 }
215 213 }
216 214
217 215 return (dsl_prop_get_dd(ds->ds_dir, propname,
218 - intsz, numints, buf, setpoint, snapshot));
216 + intsz, numints, buf, setpoint, ds->ds_is_snapshot));
219 217 }
220 218
221 219 /*
222 220 * Register interest in the named property. We'll call the callback
223 221 * once to notify it of the current property value, and again each time
224 222 * the property changes, until this callback is unregistered.
225 223 *
226 224 * Return 0 on success, errno if the prop is not an integer value.
227 225 */
228 226 int
229 227 dsl_prop_register(dsl_dataset_t *ds, const char *propname,
230 228 dsl_prop_changed_cb_t *callback, void *cbarg)
231 229 {
232 230 dsl_dir_t *dd = ds->ds_dir;
233 231 dsl_pool_t *dp = dd->dd_pool;
234 232 uint64_t value;
235 233 dsl_prop_cb_record_t *cbr;
236 234 int err;
237 235
238 236 ASSERT(dsl_pool_config_held(dp));
239 237
240 238 err = dsl_prop_get_int_ds(ds, propname, &value);
241 239 if (err != 0)
242 240 return (err);
243 241
244 242 cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_SLEEP);
245 243 cbr->cbr_ds = ds;
246 244 cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_SLEEP);
247 245 (void) strcpy((char *)cbr->cbr_propname, propname);
248 246 cbr->cbr_func = callback;
249 247 cbr->cbr_arg = cbarg;
250 248 mutex_enter(&dd->dd_lock);
251 249 list_insert_head(&dd->dd_prop_cbs, cbr);
252 250 mutex_exit(&dd->dd_lock);
253 251
254 252 cbr->cbr_func(cbr->cbr_arg, value);
255 253 return (0);
256 254 }
257 255
258 256 int
259 257 dsl_prop_get(const char *dsname, const char *propname,
260 258 int intsz, int numints, void *buf, char *setpoint)
261 259 {
262 260 objset_t *os;
263 261 int error;
264 262
265 263 error = dmu_objset_hold(dsname, FTAG, &os);
266 264 if (error != 0)
267 265 return (error);
268 266
269 267 error = dsl_prop_get_ds(dmu_objset_ds(os), propname,
270 268 intsz, numints, buf, setpoint);
271 269
272 270 dmu_objset_rele(os, FTAG);
273 271 return (error);
274 272 }
275 273
276 274 /*
277 275 * Get the current property value. It may have changed by the time this
278 276 * function returns, so it is NOT safe to follow up with
279 277 * dsl_prop_register() and assume that the value has not changed in
280 278 * between.
281 279 *
282 280 * Return 0 on success, ENOENT if ddname is invalid.
283 281 */
284 282 int
285 283 dsl_prop_get_integer(const char *ddname, const char *propname,
286 284 uint64_t *valuep, char *setpoint)
287 285 {
288 286 return (dsl_prop_get(ddname, propname, 8, 1, valuep, setpoint));
289 287 }
290 288
291 289 int
292 290 dsl_prop_get_int_ds(dsl_dataset_t *ds, const char *propname,
293 291 uint64_t *valuep)
294 292 {
295 293 return (dsl_prop_get_ds(ds, propname, 8, 1, valuep, NULL));
296 294 }
297 295
298 296 /*
299 297 * Predict the effective value of the given special property if it were set with
300 298 * the given value and source. This is not a general purpose function. It exists
301 299 * only to handle the special requirements of the quota and reservation
302 300 * properties. The fact that these properties are non-inheritable greatly
303 301 * simplifies the prediction logic.
304 302 *
305 303 * Returns 0 on success, a positive error code on failure, or -1 if called with
306 304 * a property not handled by this function.
307 305 */
308 306 int
309 307 dsl_prop_predict(dsl_dir_t *dd, const char *propname,
310 308 zprop_source_t source, uint64_t value, uint64_t *newvalp)
311 309 {
312 310 zfs_prop_t prop = zfs_name_to_prop(propname);
313 311 objset_t *mos;
314 312 uint64_t zapobj;
315 313 uint64_t version;
316 314 char *recvdstr;
317 315 int err = 0;
318 316
319 317 switch (prop) {
320 318 case ZFS_PROP_QUOTA:
321 319 case ZFS_PROP_RESERVATION:
322 320 case ZFS_PROP_REFQUOTA:
323 321 case ZFS_PROP_REFRESERVATION:
324 322 break;
325 323 default:
326 324 return (-1);
327 325 }
328 326
329 327 mos = dd->dd_pool->dp_meta_objset;
330 328 zapobj = dsl_dir_phys(dd)->dd_props_zapobj;
331 329 recvdstr = kmem_asprintf("%s%s", propname, ZPROP_RECVD_SUFFIX);
332 330
333 331 version = spa_version(dd->dd_pool->dp_spa);
334 332 if (version < SPA_VERSION_RECVD_PROPS) {
335 333 if (source & ZPROP_SRC_NONE)
336 334 source = ZPROP_SRC_NONE;
337 335 else if (source & ZPROP_SRC_RECEIVED)
338 336 source = ZPROP_SRC_LOCAL;
339 337 }
340 338
341 339 switch (source) {
342 340 case ZPROP_SRC_NONE:
343 341 /* Revert to the received value, if any. */
344 342 err = zap_lookup(mos, zapobj, recvdstr, 8, 1, newvalp);
345 343 if (err == ENOENT)
346 344 *newvalp = 0;
347 345 break;
348 346 case ZPROP_SRC_LOCAL:
349 347 *newvalp = value;
350 348 break;
351 349 case ZPROP_SRC_RECEIVED:
352 350 /*
353 351 * If there's no local setting, then the new received value will
354 352 * be the effective value.
355 353 */
356 354 err = zap_lookup(mos, zapobj, propname, 8, 1, newvalp);
357 355 if (err == ENOENT)
358 356 *newvalp = value;
359 357 break;
360 358 case (ZPROP_SRC_NONE | ZPROP_SRC_RECEIVED):
361 359 /*
362 360 * We're clearing the received value, so the local setting (if
363 361 * it exists) remains the effective value.
364 362 */
365 363 err = zap_lookup(mos, zapobj, propname, 8, 1, newvalp);
366 364 if (err == ENOENT)
367 365 *newvalp = 0;
368 366 break;
369 367 default:
370 368 panic("unexpected property source: %d", source);
371 369 }
372 370
373 371 strfree(recvdstr);
374 372
375 373 if (err == ENOENT)
376 374 return (0);
377 375
378 376 return (err);
379 377 }
380 378
381 379 /*
382 380 * Unregister this callback. Return 0 on success, ENOENT if ddname is
383 381 * invalid, or ENOMSG if no matching callback registered.
384 382 */
385 383 int
386 384 dsl_prop_unregister(dsl_dataset_t *ds, const char *propname,
387 385 dsl_prop_changed_cb_t *callback, void *cbarg)
388 386 {
389 387 dsl_dir_t *dd = ds->ds_dir;
390 388 dsl_prop_cb_record_t *cbr;
391 389
392 390 mutex_enter(&dd->dd_lock);
393 391 for (cbr = list_head(&dd->dd_prop_cbs);
394 392 cbr; cbr = list_next(&dd->dd_prop_cbs, cbr)) {
395 393 if (cbr->cbr_ds == ds &&
396 394 cbr->cbr_func == callback &&
397 395 cbr->cbr_arg == cbarg &&
398 396 strcmp(cbr->cbr_propname, propname) == 0)
399 397 break;
400 398 }
401 399
402 400 if (cbr == NULL) {
403 401 mutex_exit(&dd->dd_lock);
404 402 return (SET_ERROR(ENOMSG));
405 403 }
406 404
407 405 list_remove(&dd->dd_prop_cbs, cbr);
408 406 mutex_exit(&dd->dd_lock);
409 407 kmem_free((void*)cbr->cbr_propname, strlen(cbr->cbr_propname)+1);
410 408 kmem_free(cbr, sizeof (dsl_prop_cb_record_t));
411 409
412 410 return (0);
413 411 }
414 412
415 413 boolean_t
416 414 dsl_prop_hascb(dsl_dataset_t *ds)
417 415 {
418 416 dsl_dir_t *dd = ds->ds_dir;
419 417 boolean_t rv = B_FALSE;
420 418 dsl_prop_cb_record_t *cbr;
421 419
422 420 mutex_enter(&dd->dd_lock);
423 421 for (cbr = list_head(&dd->dd_prop_cbs); cbr;
424 422 cbr = list_next(&dd->dd_prop_cbs, cbr)) {
425 423 if (cbr->cbr_ds == ds) {
426 424 rv = B_TRUE;
427 425 break;
428 426 }
429 427 }
430 428 mutex_exit(&dd->dd_lock);
431 429 return (rv);
432 430 }
433 431
434 432 /* ARGSUSED */
435 433 static int
436 434 dsl_prop_notify_all_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
437 435 {
438 436 dsl_dir_t *dd = ds->ds_dir;
439 437 dsl_prop_cb_record_t *cbr;
440 438
441 439 mutex_enter(&dd->dd_lock);
442 440 for (cbr = list_head(&dd->dd_prop_cbs); cbr;
443 441 cbr = list_next(&dd->dd_prop_cbs, cbr)) {
444 442 uint64_t value;
445 443
446 444 if (dsl_prop_get_ds(cbr->cbr_ds, cbr->cbr_propname,
447 445 sizeof (value), 1, &value, NULL) == 0)
448 446 cbr->cbr_func(cbr->cbr_arg, value);
449 447 }
450 448 mutex_exit(&dd->dd_lock);
451 449
452 450 return (0);
453 451 }
454 452
455 453 /*
456 454 * Update all property values for ddobj & its descendants. This is used
457 455 * when renaming the dir.
458 456 */
459 457 void
460 458 dsl_prop_notify_all(dsl_dir_t *dd)
461 459 {
462 460 dsl_pool_t *dp = dd->dd_pool;
463 461 ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
464 462 (void) dmu_objset_find_dp(dp, dd->dd_object, dsl_prop_notify_all_cb,
465 463 NULL, DS_FIND_CHILDREN);
466 464 }
467 465
468 466 static void
469 467 dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
470 468 const char *propname, uint64_t value, int first)
471 469 {
472 470 dsl_dir_t *dd;
473 471 dsl_prop_cb_record_t *cbr;
474 472 objset_t *mos = dp->dp_meta_objset;
475 473 zap_cursor_t zc;
476 474 zap_attribute_t *za;
477 475 int err;
478 476
479 477 ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
480 478 err = dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd);
481 479 if (err)
482 480 return;
483 481
484 482 if (!first) {
485 483 /*
486 484 * If the prop is set here, then this change is not
487 485 * being inherited here or below; stop the recursion.
488 486 */
489 487 err = zap_contains(mos, dsl_dir_phys(dd)->dd_props_zapobj,
490 488 propname);
491 489 if (err == 0) {
492 490 dsl_dir_rele(dd, FTAG);
493 491 return;
494 492 }
495 493 ASSERT3U(err, ==, ENOENT);
496 494 }
497 495
498 496 mutex_enter(&dd->dd_lock);
499 497 for (cbr = list_head(&dd->dd_prop_cbs); cbr;
500 498 cbr = list_next(&dd->dd_prop_cbs, cbr)) {
501 499 uint64_t propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
502 500
503 501 if (strcmp(cbr->cbr_propname, propname) != 0)
504 502 continue;
505 503
506 504 /*
507 505 * If the property is set on this ds, then it is not
508 506 * inherited here; don't call the callback.
509 507 */
510 508 if (propobj && 0 == zap_contains(mos, propobj, propname))
511 509 continue;
512 510
513 511 cbr->cbr_func(cbr->cbr_arg, value);
514 512 }
515 513 mutex_exit(&dd->dd_lock);
516 514
517 515 za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
518 516 for (zap_cursor_init(&zc, mos,
519 517 dsl_dir_phys(dd)->dd_child_dir_zapobj);
520 518 zap_cursor_retrieve(&zc, za) == 0;
521 519 zap_cursor_advance(&zc)) {
522 520 dsl_prop_changed_notify(dp, za->za_first_integer,
523 521 propname, value, FALSE);
524 522 }
525 523 kmem_free(za, sizeof (zap_attribute_t));
526 524 zap_cursor_fini(&zc);
527 525 dsl_dir_rele(dd, FTAG);
528 526 }
529 527
530 528 void
531 529 dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
532 530 zprop_source_t source, int intsz, int numints, const void *value,
533 531 dmu_tx_t *tx)
534 532 {
535 533 objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
536 534 uint64_t zapobj, intval, dummy;
537 535 int isint;
|
↓ open down ↓ |
309 lines elided |
↑ open up ↑ |
538 536 char valbuf[32];
539 537 const char *valstr = NULL;
540 538 char *inheritstr;
541 539 char *recvdstr;
542 540 char *tbuf = NULL;
543 541 int err;
544 542 uint64_t version = spa_version(ds->ds_dir->dd_pool->dp_spa);
545 543
546 544 isint = (dodefault(propname, 8, 1, &intval) == 0);
547 545
548 - if (dsl_dataset_is_snapshot(ds)) {
546 + if (ds->ds_is_snapshot) {
549 547 ASSERT(version >= SPA_VERSION_SNAP_PROPS);
550 548 if (dsl_dataset_phys(ds)->ds_props_obj == 0) {
551 549 dmu_buf_will_dirty(ds->ds_dbuf, tx);
552 550 dsl_dataset_phys(ds)->ds_props_obj =
553 551 zap_create(mos,
554 552 DMU_OT_DSL_PROPS, DMU_OT_NONE, 0, tx);
555 553 }
556 554 zapobj = dsl_dataset_phys(ds)->ds_props_obj;
557 555 } else {
558 556 zapobj = dsl_dir_phys(ds->ds_dir)->dd_props_zapobj;
559 557 }
560 558
561 559 if (version < SPA_VERSION_RECVD_PROPS) {
562 560 if (source & ZPROP_SRC_NONE)
563 561 source = ZPROP_SRC_NONE;
564 562 else if (source & ZPROP_SRC_RECEIVED)
565 563 source = ZPROP_SRC_LOCAL;
566 564 }
567 565
568 566 inheritstr = kmem_asprintf("%s%s", propname, ZPROP_INHERIT_SUFFIX);
569 567 recvdstr = kmem_asprintf("%s%s", propname, ZPROP_RECVD_SUFFIX);
570 568
571 569 switch (source) {
572 570 case ZPROP_SRC_NONE:
573 571 /*
574 572 * revert to received value, if any (inherit -S)
575 573 * - remove propname
576 574 * - remove propname$inherit
577 575 */
578 576 err = zap_remove(mos, zapobj, propname, tx);
579 577 ASSERT(err == 0 || err == ENOENT);
580 578 err = zap_remove(mos, zapobj, inheritstr, tx);
581 579 ASSERT(err == 0 || err == ENOENT);
582 580 break;
583 581 case ZPROP_SRC_LOCAL:
584 582 /*
585 583 * remove propname$inherit
586 584 * set propname -> value
587 585 */
588 586 err = zap_remove(mos, zapobj, inheritstr, tx);
589 587 ASSERT(err == 0 || err == ENOENT);
590 588 VERIFY0(zap_update(mos, zapobj, propname,
591 589 intsz, numints, value, tx));
592 590 break;
593 591 case ZPROP_SRC_INHERITED:
594 592 /*
595 593 * explicitly inherit
596 594 * - remove propname
597 595 * - set propname$inherit
598 596 */
599 597 err = zap_remove(mos, zapobj, propname, tx);
600 598 ASSERT(err == 0 || err == ENOENT);
601 599 if (version >= SPA_VERSION_RECVD_PROPS &&
602 600 dsl_prop_get_int_ds(ds, ZPROP_HAS_RECVD, &dummy) == 0) {
603 601 dummy = 0;
604 602 VERIFY0(zap_update(mos, zapobj, inheritstr,
605 603 8, 1, &dummy, tx));
606 604 }
607 605 break;
608 606 case ZPROP_SRC_RECEIVED:
609 607 /*
610 608 * set propname$recvd -> value
611 609 */
612 610 err = zap_update(mos, zapobj, recvdstr,
613 611 intsz, numints, value, tx);
614 612 ASSERT(err == 0);
615 613 break;
616 614 case (ZPROP_SRC_NONE | ZPROP_SRC_LOCAL | ZPROP_SRC_RECEIVED):
617 615 /*
618 616 * clear local and received settings
619 617 * - remove propname
620 618 * - remove propname$inherit
621 619 * - remove propname$recvd
622 620 */
623 621 err = zap_remove(mos, zapobj, propname, tx);
624 622 ASSERT(err == 0 || err == ENOENT);
625 623 err = zap_remove(mos, zapobj, inheritstr, tx);
626 624 ASSERT(err == 0 || err == ENOENT);
627 625 /* FALLTHRU */
628 626 case (ZPROP_SRC_NONE | ZPROP_SRC_RECEIVED):
629 627 /*
630 628 * remove propname$recvd
631 629 */
632 630 err = zap_remove(mos, zapobj, recvdstr, tx);
633 631 ASSERT(err == 0 || err == ENOENT);
634 632 break;
|
↓ open down ↓ |
76 lines elided |
↑ open up ↑ |
635 633 default:
636 634 cmn_err(CE_PANIC, "unexpected property source: %d", source);
637 635 }
638 636
639 637 strfree(inheritstr);
640 638 strfree(recvdstr);
641 639
642 640 if (isint) {
643 641 VERIFY0(dsl_prop_get_int_ds(ds, propname, &intval));
644 642
645 - if (dsl_dataset_is_snapshot(ds)) {
643 + if (ds->ds_is_snapshot) {
646 644 dsl_prop_cb_record_t *cbr;
647 645 /*
648 646 * It's a snapshot; nothing can inherit this
649 647 * property, so just look for callbacks on this
650 648 * ds here.
651 649 */
652 650 mutex_enter(&ds->ds_dir->dd_lock);
653 651 for (cbr = list_head(&ds->ds_dir->dd_prop_cbs); cbr;
654 652 cbr = list_next(&ds->ds_dir->dd_prop_cbs, cbr)) {
655 653 if (cbr->cbr_ds == ds &&
656 654 strcmp(cbr->cbr_propname, propname) == 0)
657 655 cbr->cbr_func(cbr->cbr_arg, intval);
658 656 }
659 657 mutex_exit(&ds->ds_dir->dd_lock);
660 658 } else {
661 659 dsl_prop_changed_notify(ds->ds_dir->dd_pool,
662 660 ds->ds_dir->dd_object, propname, intval, TRUE);
663 661 }
664 662
665 663 (void) snprintf(valbuf, sizeof (valbuf),
666 664 "%lld", (longlong_t)intval);
667 665 valstr = valbuf;
668 666 } else {
669 667 if (source == ZPROP_SRC_LOCAL) {
670 668 valstr = value;
671 669 } else {
672 670 tbuf = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP);
673 671 if (dsl_prop_get_ds(ds, propname, 1,
674 672 ZAP_MAXVALUELEN, tbuf, NULL) == 0)
675 673 valstr = tbuf;
676 674 }
677 675 }
678 676
679 677 spa_history_log_internal_ds(ds, (source == ZPROP_SRC_NONE ||
680 678 source == ZPROP_SRC_INHERITED) ? "inherit" : "set", tx,
681 679 "%s=%s", propname, (valstr == NULL ? "" : valstr));
682 680
683 681 if (tbuf != NULL)
684 682 kmem_free(tbuf, ZAP_MAXVALUELEN);
685 683 }
686 684
687 685 int
688 686 dsl_prop_set_int(const char *dsname, const char *propname,
689 687 zprop_source_t source, uint64_t value)
690 688 {
691 689 nvlist_t *nvl = fnvlist_alloc();
692 690 int error;
693 691
694 692 fnvlist_add_uint64(nvl, propname, value);
695 693 error = dsl_props_set(dsname, source, nvl);
696 694 fnvlist_free(nvl);
697 695 return (error);
698 696 }
699 697
700 698 int
701 699 dsl_prop_set_string(const char *dsname, const char *propname,
702 700 zprop_source_t source, const char *value)
703 701 {
704 702 nvlist_t *nvl = fnvlist_alloc();
705 703 int error;
706 704
707 705 fnvlist_add_string(nvl, propname, value);
708 706 error = dsl_props_set(dsname, source, nvl);
709 707 fnvlist_free(nvl);
710 708 return (error);
711 709 }
712 710
713 711 int
714 712 dsl_prop_inherit(const char *dsname, const char *propname,
715 713 zprop_source_t source)
716 714 {
717 715 nvlist_t *nvl = fnvlist_alloc();
718 716 int error;
719 717
720 718 fnvlist_add_boolean(nvl, propname);
721 719 error = dsl_props_set(dsname, source, nvl);
722 720 fnvlist_free(nvl);
723 721 return (error);
724 722 }
725 723
726 724 typedef struct dsl_props_set_arg {
727 725 const char *dpsa_dsname;
728 726 zprop_source_t dpsa_source;
729 727 nvlist_t *dpsa_props;
730 728 } dsl_props_set_arg_t;
731 729
732 730 static int
733 731 dsl_props_set_check(void *arg, dmu_tx_t *tx)
734 732 {
735 733 dsl_props_set_arg_t *dpsa = arg;
736 734 dsl_pool_t *dp = dmu_tx_pool(tx);
737 735 dsl_dataset_t *ds;
738 736 uint64_t version;
739 737 nvpair_t *elem = NULL;
740 738 int err;
741 739
742 740 err = dsl_dataset_hold(dp, dpsa->dpsa_dsname, FTAG, &ds);
743 741 if (err != 0)
744 742 return (err);
745 743
746 744 version = spa_version(ds->ds_dir->dd_pool->dp_spa);
747 745 while ((elem = nvlist_next_nvpair(dpsa->dpsa_props, elem)) != NULL) {
748 746 if (strlen(nvpair_name(elem)) >= ZAP_MAXNAMELEN) {
749 747 dsl_dataset_rele(ds, FTAG);
750 748 return (SET_ERROR(ENAMETOOLONG));
751 749 }
752 750 if (nvpair_type(elem) == DATA_TYPE_STRING) {
|
↓ open down ↓ |
97 lines elided |
↑ open up ↑ |
753 751 char *valstr = fnvpair_value_string(elem);
754 752 if (strlen(valstr) >= (version <
755 753 SPA_VERSION_STMF_PROP ?
756 754 ZAP_OLDMAXVALUELEN : ZAP_MAXVALUELEN)) {
757 755 dsl_dataset_rele(ds, FTAG);
758 756 return (E2BIG);
759 757 }
760 758 }
761 759 }
762 760
763 - if (dsl_dataset_is_snapshot(ds) && version < SPA_VERSION_SNAP_PROPS) {
761 + if (ds->ds_is_snapshot && version < SPA_VERSION_SNAP_PROPS) {
764 762 dsl_dataset_rele(ds, FTAG);
765 763 return (SET_ERROR(ENOTSUP));
766 764 }
767 765 dsl_dataset_rele(ds, FTAG);
768 766 return (0);
769 767 }
770 768
771 769 void
772 770 dsl_props_set_sync_impl(dsl_dataset_t *ds, zprop_source_t source,
773 771 nvlist_t *props, dmu_tx_t *tx)
774 772 {
775 773 nvpair_t *elem = NULL;
776 774
777 775 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) {
778 776 nvpair_t *pair = elem;
779 777
780 778 if (nvpair_type(pair) == DATA_TYPE_NVLIST) {
781 779 /*
782 780 * dsl_prop_get_all_impl() returns properties in this
783 781 * format.
784 782 */
785 783 nvlist_t *attrs = fnvpair_value_nvlist(pair);
786 784 pair = fnvlist_lookup_nvpair(attrs, ZPROP_VALUE);
787 785 }
788 786
789 787 if (nvpair_type(pair) == DATA_TYPE_STRING) {
790 788 const char *value = fnvpair_value_string(pair);
791 789 dsl_prop_set_sync_impl(ds, nvpair_name(pair),
792 790 source, 1, strlen(value) + 1, value, tx);
793 791 } else if (nvpair_type(pair) == DATA_TYPE_UINT64) {
794 792 uint64_t intval = fnvpair_value_uint64(pair);
795 793 dsl_prop_set_sync_impl(ds, nvpair_name(pair),
796 794 source, sizeof (intval), 1, &intval, tx);
797 795 } else if (nvpair_type(pair) == DATA_TYPE_BOOLEAN) {
798 796 dsl_prop_set_sync_impl(ds, nvpair_name(pair),
799 797 source, 0, 0, NULL, tx);
800 798 } else {
801 799 panic("invalid nvpair type");
802 800 }
803 801 }
804 802 }
805 803
806 804 static void
807 805 dsl_props_set_sync(void *arg, dmu_tx_t *tx)
808 806 {
809 807 dsl_props_set_arg_t *dpsa = arg;
810 808 dsl_pool_t *dp = dmu_tx_pool(tx);
811 809 dsl_dataset_t *ds;
812 810
813 811 VERIFY0(dsl_dataset_hold(dp, dpsa->dpsa_dsname, FTAG, &ds));
814 812 dsl_props_set_sync_impl(ds, dpsa->dpsa_source, dpsa->dpsa_props, tx);
815 813 dsl_dataset_rele(ds, FTAG);
816 814 }
817 815
818 816 /*
819 817 * All-or-nothing; if any prop can't be set, nothing will be modified.
820 818 */
821 819 int
822 820 dsl_props_set(const char *dsname, zprop_source_t source, nvlist_t *props)
823 821 {
824 822 dsl_props_set_arg_t dpsa;
825 823 int nblks = 0;
826 824
827 825 dpsa.dpsa_dsname = dsname;
828 826 dpsa.dpsa_source = source;
829 827 dpsa.dpsa_props = props;
830 828
831 829 /*
832 830 * If the source includes NONE, then we will only be removing entries
833 831 * from the ZAP object. In that case don't check for ENOSPC.
834 832 */
835 833 if ((source & ZPROP_SRC_NONE) == 0)
836 834 nblks = 2 * fnvlist_num_pairs(props);
837 835
838 836 return (dsl_sync_task(dsname, dsl_props_set_check, dsl_props_set_sync,
839 837 &dpsa, nblks, ZFS_SPACE_CHECK_RESERVED));
840 838 }
841 839
842 840 typedef enum dsl_prop_getflags {
843 841 DSL_PROP_GET_INHERITING = 0x1, /* searching parent of target ds */
844 842 DSL_PROP_GET_SNAPSHOT = 0x2, /* snapshot dataset */
845 843 DSL_PROP_GET_LOCAL = 0x4, /* local properties */
846 844 DSL_PROP_GET_RECEIVED = 0x8 /* received properties */
847 845 } dsl_prop_getflags_t;
848 846
849 847 static int
850 848 dsl_prop_get_all_impl(objset_t *mos, uint64_t propobj,
851 849 const char *setpoint, dsl_prop_getflags_t flags, nvlist_t *nv)
852 850 {
853 851 zap_cursor_t zc;
854 852 zap_attribute_t za;
855 853 int err = 0;
856 854
857 855 for (zap_cursor_init(&zc, mos, propobj);
858 856 (err = zap_cursor_retrieve(&zc, &za)) == 0;
859 857 zap_cursor_advance(&zc)) {
860 858 nvlist_t *propval;
861 859 zfs_prop_t prop;
862 860 char buf[ZAP_MAXNAMELEN];
863 861 char *valstr;
864 862 const char *suffix;
865 863 const char *propname;
866 864 const char *source;
867 865
868 866 suffix = strchr(za.za_name, '$');
869 867
870 868 if (suffix == NULL) {
871 869 /*
872 870 * Skip local properties if we only want received
873 871 * properties.
874 872 */
875 873 if (flags & DSL_PROP_GET_RECEIVED)
876 874 continue;
877 875
878 876 propname = za.za_name;
879 877 source = setpoint;
880 878 } else if (strcmp(suffix, ZPROP_INHERIT_SUFFIX) == 0) {
881 879 /* Skip explicitly inherited entries. */
882 880 continue;
883 881 } else if (strcmp(suffix, ZPROP_RECVD_SUFFIX) == 0) {
884 882 if (flags & DSL_PROP_GET_LOCAL)
885 883 continue;
886 884
887 885 (void) strncpy(buf, za.za_name, (suffix - za.za_name));
888 886 buf[suffix - za.za_name] = '\0';
889 887 propname = buf;
890 888
891 889 if (!(flags & DSL_PROP_GET_RECEIVED)) {
892 890 /* Skip if locally overridden. */
893 891 err = zap_contains(mos, propobj, propname);
894 892 if (err == 0)
895 893 continue;
896 894 if (err != ENOENT)
897 895 break;
898 896
899 897 /* Skip if explicitly inherited. */
900 898 valstr = kmem_asprintf("%s%s", propname,
901 899 ZPROP_INHERIT_SUFFIX);
902 900 err = zap_contains(mos, propobj, valstr);
903 901 strfree(valstr);
904 902 if (err == 0)
905 903 continue;
906 904 if (err != ENOENT)
907 905 break;
908 906 }
909 907
910 908 source = ((flags & DSL_PROP_GET_INHERITING) ?
911 909 setpoint : ZPROP_SOURCE_VAL_RECVD);
912 910 } else {
913 911 /*
914 912 * For backward compatibility, skip suffixes we don't
915 913 * recognize.
916 914 */
917 915 continue;
918 916 }
919 917
920 918 prop = zfs_name_to_prop(propname);
921 919
922 920 /* Skip non-inheritable properties. */
923 921 if ((flags & DSL_PROP_GET_INHERITING) && prop != ZPROP_INVAL &&
924 922 !zfs_prop_inheritable(prop))
925 923 continue;
926 924
927 925 /* Skip properties not valid for this type. */
928 926 if ((flags & DSL_PROP_GET_SNAPSHOT) && prop != ZPROP_INVAL &&
929 927 !zfs_prop_valid_for_type(prop, ZFS_TYPE_SNAPSHOT))
930 928 continue;
931 929
932 930 /* Skip properties already defined. */
933 931 if (nvlist_exists(nv, propname))
934 932 continue;
935 933
936 934 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
937 935 if (za.za_integer_length == 1) {
938 936 /*
939 937 * String property
940 938 */
941 939 char *tmp = kmem_alloc(za.za_num_integers,
942 940 KM_SLEEP);
943 941 err = zap_lookup(mos, propobj,
944 942 za.za_name, 1, za.za_num_integers, tmp);
945 943 if (err != 0) {
946 944 kmem_free(tmp, za.za_num_integers);
947 945 break;
948 946 }
949 947 VERIFY(nvlist_add_string(propval, ZPROP_VALUE,
950 948 tmp) == 0);
951 949 kmem_free(tmp, za.za_num_integers);
952 950 } else {
953 951 /*
954 952 * Integer property
955 953 */
956 954 ASSERT(za.za_integer_length == 8);
957 955 (void) nvlist_add_uint64(propval, ZPROP_VALUE,
958 956 za.za_first_integer);
959 957 }
960 958
961 959 VERIFY(nvlist_add_string(propval, ZPROP_SOURCE, source) == 0);
962 960 VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
963 961 nvlist_free(propval);
964 962 }
965 963 zap_cursor_fini(&zc);
966 964 if (err == ENOENT)
967 965 err = 0;
968 966 return (err);
969 967 }
970 968
971 969 /*
972 970 * Iterate over all properties for this dataset and return them in an nvlist.
973 971 */
974 972 static int
975 973 dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
|
↓ open down ↓ |
202 lines elided |
↑ open up ↑ |
976 974 dsl_prop_getflags_t flags)
977 975 {
978 976 dsl_dir_t *dd = ds->ds_dir;
979 977 dsl_pool_t *dp = dd->dd_pool;
980 978 objset_t *mos = dp->dp_meta_objset;
981 979 int err = 0;
982 980 char setpoint[MAXNAMELEN];
983 981
984 982 VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
985 983
986 - if (dsl_dataset_is_snapshot(ds))
984 + if (ds->ds_is_snapshot)
987 985 flags |= DSL_PROP_GET_SNAPSHOT;
988 986
989 987 ASSERT(dsl_pool_config_held(dp));
990 988
991 989 if (dsl_dataset_phys(ds)->ds_props_obj != 0) {
992 990 ASSERT(flags & DSL_PROP_GET_SNAPSHOT);
993 991 dsl_dataset_name(ds, setpoint);
994 992 err = dsl_prop_get_all_impl(mos,
995 993 dsl_dataset_phys(ds)->ds_props_obj, setpoint, flags, *nvp);
996 994 if (err)
997 995 goto out;
998 996 }
999 997
1000 998 for (; dd != NULL; dd = dd->dd_parent) {
1001 999 if (dd != ds->ds_dir || (flags & DSL_PROP_GET_SNAPSHOT)) {
1002 1000 if (flags & (DSL_PROP_GET_LOCAL |
1003 1001 DSL_PROP_GET_RECEIVED))
1004 1002 break;
1005 1003 flags |= DSL_PROP_GET_INHERITING;
1006 1004 }
1007 1005 dsl_dir_name(dd, setpoint);
1008 1006 err = dsl_prop_get_all_impl(mos,
1009 1007 dsl_dir_phys(dd)->dd_props_zapobj, setpoint, flags, *nvp);
1010 1008 if (err)
1011 1009 break;
1012 1010 }
1013 1011 out:
1014 1012 return (err);
1015 1013 }
1016 1014
1017 1015 boolean_t
1018 1016 dsl_prop_get_hasrecvd(const char *dsname)
1019 1017 {
1020 1018 uint64_t dummy;
1021 1019
1022 1020 return (0 ==
1023 1021 dsl_prop_get_integer(dsname, ZPROP_HAS_RECVD, &dummy, NULL));
1024 1022 }
1025 1023
1026 1024 static int
1027 1025 dsl_prop_set_hasrecvd_impl(const char *dsname, zprop_source_t source)
1028 1026 {
1029 1027 uint64_t version;
1030 1028 spa_t *spa;
1031 1029 int error = 0;
1032 1030
1033 1031 VERIFY0(spa_open(dsname, &spa, FTAG));
1034 1032 version = spa_version(spa);
1035 1033 spa_close(spa, FTAG);
1036 1034
1037 1035 if (version >= SPA_VERSION_RECVD_PROPS)
1038 1036 error = dsl_prop_set_int(dsname, ZPROP_HAS_RECVD, source, 0);
1039 1037 return (error);
1040 1038 }
1041 1039
1042 1040 /*
1043 1041 * Call after successfully receiving properties to ensure that only the first
1044 1042 * receive on or after SPA_VERSION_RECVD_PROPS blows away local properties.
1045 1043 */
1046 1044 int
1047 1045 dsl_prop_set_hasrecvd(const char *dsname)
1048 1046 {
1049 1047 int error = 0;
1050 1048 if (!dsl_prop_get_hasrecvd(dsname))
1051 1049 error = dsl_prop_set_hasrecvd_impl(dsname, ZPROP_SRC_LOCAL);
1052 1050 return (error);
1053 1051 }
1054 1052
1055 1053 void
1056 1054 dsl_prop_unset_hasrecvd(const char *dsname)
1057 1055 {
1058 1056 VERIFY0(dsl_prop_set_hasrecvd_impl(dsname, ZPROP_SRC_NONE));
1059 1057 }
1060 1058
1061 1059 int
1062 1060 dsl_prop_get_all(objset_t *os, nvlist_t **nvp)
1063 1061 {
1064 1062 return (dsl_prop_get_all_ds(os->os_dsl_dataset, nvp, 0));
1065 1063 }
1066 1064
1067 1065 int
1068 1066 dsl_prop_get_received(const char *dsname, nvlist_t **nvp)
1069 1067 {
1070 1068 objset_t *os;
1071 1069 int error;
1072 1070
1073 1071 /*
1074 1072 * Received properties are not distinguishable from local properties
1075 1073 * until the dataset has received properties on or after
1076 1074 * SPA_VERSION_RECVD_PROPS.
1077 1075 */
1078 1076 dsl_prop_getflags_t flags = (dsl_prop_get_hasrecvd(dsname) ?
1079 1077 DSL_PROP_GET_RECEIVED : DSL_PROP_GET_LOCAL);
1080 1078
1081 1079 error = dmu_objset_hold(dsname, FTAG, &os);
1082 1080 if (error != 0)
1083 1081 return (error);
1084 1082 error = dsl_prop_get_all_ds(os->os_dsl_dataset, nvp, flags);
1085 1083 dmu_objset_rele(os, FTAG);
1086 1084 return (error);
1087 1085 }
1088 1086
1089 1087 void
1090 1088 dsl_prop_nvlist_add_uint64(nvlist_t *nv, zfs_prop_t prop, uint64_t value)
1091 1089 {
1092 1090 nvlist_t *propval;
1093 1091 const char *propname = zfs_prop_to_name(prop);
1094 1092 uint64_t default_value;
1095 1093
1096 1094 if (nvlist_lookup_nvlist(nv, propname, &propval) == 0) {
1097 1095 VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, value) == 0);
1098 1096 return;
1099 1097 }
1100 1098
1101 1099 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
1102 1100 VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, value) == 0);
1103 1101 /* Indicate the default source if we can. */
1104 1102 if (dodefault(propname, 8, 1, &default_value) == 0 &&
1105 1103 value == default_value) {
1106 1104 VERIFY(nvlist_add_string(propval, ZPROP_SOURCE, "") == 0);
1107 1105 }
1108 1106 VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
1109 1107 nvlist_free(propval);
1110 1108 }
1111 1109
1112 1110 void
1113 1111 dsl_prop_nvlist_add_string(nvlist_t *nv, zfs_prop_t prop, const char *value)
1114 1112 {
1115 1113 nvlist_t *propval;
1116 1114 const char *propname = zfs_prop_to_name(prop);
1117 1115
1118 1116 if (nvlist_lookup_nvlist(nv, propname, &propval) == 0) {
1119 1117 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, value) == 0);
1120 1118 return;
1121 1119 }
1122 1120
1123 1121 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0);
1124 1122 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, value) == 0);
1125 1123 VERIFY(nvlist_add_nvlist(nv, propname, propval) == 0);
1126 1124 nvlist_free(propval);
1127 1125 }
|
↓ open down ↓ |
131 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX