Print this page
5056 ZFS deadlock on db_mtx and dn_holds
Reviewed by: Will Andrews <willa@spectralogic.com>
Reviewed by: Matt Ahrens <mahrens@delphix.com>
Reviewed by: George Wilson <george.wilson@delphix.com>
Approved by: Dan McDonald <danmcd@omniti.com>

Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/fs/zfs/dsl_dataset.c
          +++ new/usr/src/uts/common/fs/zfs/dsl_dataset.c
↓ open down ↓ 15 lines elided ↑ open up ↑
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  /*
  22   22   * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23   23   * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  24   24   * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  25   25   * Copyright (c) 2014 RackTop Systems.
       26 + * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  26   27   */
  27   28  
  28   29  #include <sys/dmu_objset.h>
  29   30  #include <sys/dsl_dataset.h>
  30   31  #include <sys/dsl_dir.h>
  31   32  #include <sys/dsl_prop.h>
  32   33  #include <sys/dsl_synctask.h>
  33   34  #include <sys/dmu_traverse.h>
  34   35  #include <sys/dmu_impl.h>
  35   36  #include <sys/dmu_tx.h>
↓ open down ↓ 28 lines elided ↑ open up ↑
  64   65  #define SWITCH64(x, y) \
  65   66          { \
  66   67                  uint64_t __tmp = (x); \
  67   68                  (x) = (y); \
  68   69                  (y) = __tmp; \
  69   70          }
  70   71  
  71   72  #define DS_REF_MAX      (1ULL << 62)
  72   73  
  73   74  extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds);
  74      -extern inline boolean_t dsl_dataset_is_snapshot(dsl_dataset_t *ds);
  75   75  
  76   76  /*
  77   77   * Figure out how much of this delta should be propogated to the dsl_dir
  78   78   * layer.  If there's a refreservation, that space has already been
  79   79   * partially accounted for in our ancestors.
  80   80   */
  81   81  static int64_t
  82   82  parent_delta(dsl_dataset_t *ds, int64_t delta)
  83   83  {
  84   84          dsl_dataset_phys_t *ds_phys;
↓ open down ↓ 63 lines elided ↑ open up ↑
 148  148          ASSERT(bp->blk_birth <= tx->tx_txg);
 149  149  
 150  150          if (ds == NULL) {
 151  151                  dsl_free(tx->tx_pool, tx->tx_txg, bp);
 152  152                  dsl_pool_mos_diduse_space(tx->tx_pool,
 153  153                      -used, -compressed, -uncompressed);
 154  154                  return (used);
 155  155          }
 156  156          ASSERT3P(tx->tx_pool, ==, ds->ds_dir->dd_pool);
 157  157  
 158      -        ASSERT(!dsl_dataset_is_snapshot(ds));
      158 +        ASSERT(!ds->ds_is_snapshot);
 159  159          dmu_buf_will_dirty(ds->ds_dbuf, tx);
 160  160  
 161  161          if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 162  162                  int64_t delta;
 163  163  
 164  164                  dprintf_bp(bp, "freeing ds=%llu", ds->ds_object);
 165  165                  dsl_free(tx->tx_pool, tx->tx_txg, bp);
 166  166  
 167  167                  mutex_enter(&ds->ds_lock);
 168  168                  ASSERT(dsl_dataset_phys(ds)->ds_unique_bytes >= used ||
↓ open down ↓ 77 lines elided ↑ open up ↑
 246  246  {
 247  247          if (blk_birth <= dsl_dataset_prev_snap_txg(ds) ||
 248  248              (bp != NULL && BP_IS_HOLE(bp)))
 249  249                  return (B_FALSE);
 250  250  
 251  251          ddt_prefetch(dsl_dataset_get_spa(ds), bp);
 252  252  
 253  253          return (B_TRUE);
 254  254  }
 255  255  
 256      -/* ARGSUSED */
 257  256  static void
 258      -dsl_dataset_evict(dmu_buf_t *db, void *dsv)
      257 +dsl_dataset_evict(void *dbu)
 259  258  {
 260      -        dsl_dataset_t *ds = dsv;
      259 +        dsl_dataset_t *ds = dbu;
 261  260  
 262  261          ASSERT(ds->ds_owner == NULL);
 263  262  
      263 +        ds->ds_dbuf = NULL;
      264 +
 264  265          unique_remove(ds->ds_fsid_guid);
 265  266  
 266  267          if (ds->ds_objset != NULL)
 267  268                  dmu_objset_evict(ds->ds_objset);
 268  269  
 269  270          if (ds->ds_prev) {
 270  271                  dsl_dataset_rele(ds->ds_prev, ds);
 271  272                  ds->ds_prev = NULL;
 272  273          }
 273  274  
 274  275          bplist_destroy(&ds->ds_pending_deadlist);
 275      -        if (dsl_dataset_phys(ds)->ds_deadlist_obj != 0)
      276 +        if (ds->ds_deadlist.dl_os != NULL)
 276  277                  dsl_deadlist_close(&ds->ds_deadlist);
 277  278          if (ds->ds_dir)
 278      -                dsl_dir_rele(ds->ds_dir, ds);
      279 +                dsl_dir_async_rele(ds->ds_dir, ds);
 279  280  
 280  281          ASSERT(!list_link_active(&ds->ds_synced_link));
 281  282  
 282  283          mutex_destroy(&ds->ds_lock);
 283  284          mutex_destroy(&ds->ds_opening_lock);
 284  285          mutex_destroy(&ds->ds_sendstream_lock);
 285  286          refcount_destroy(&ds->ds_longholds);
 286  287  
 287  288          kmem_free(ds, sizeof (dsl_dataset_t));
 288  289  }
↓ open down ↓ 93 lines elided ↑ open up ↑
 382  383                  return (SET_ERROR(EINVAL));
 383  384          }
 384  385  
 385  386          ds = dmu_buf_get_user(dbuf);
 386  387          if (ds == NULL) {
 387  388                  dsl_dataset_t *winner = NULL;
 388  389  
 389  390                  ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
 390  391                  ds->ds_dbuf = dbuf;
 391  392                  ds->ds_object = dsobj;
      393 +                ds->ds_is_snapshot = dsl_dataset_phys(ds)->ds_num_children != 0;
 392  394  
 393  395                  mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
 394  396                  mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
 395  397                  mutex_init(&ds->ds_sendstream_lock, NULL, MUTEX_DEFAULT, NULL);
 396  398                  refcount_create(&ds->ds_longholds);
 397  399  
 398  400                  bplist_create(&ds->ds_pending_deadlist);
 399  401                  dsl_deadlist_open(&ds->ds_deadlist,
 400  402                      mos, dsl_dataset_phys(ds)->ds_deadlist_obj);
 401  403  
↓ open down ↓ 18 lines elided ↑ open up ↑
 420  422                          mutex_destroy(&ds->ds_opening_lock);
 421  423                          mutex_destroy(&ds->ds_sendstream_lock);
 422  424                          refcount_destroy(&ds->ds_longholds);
 423  425                          bplist_destroy(&ds->ds_pending_deadlist);
 424  426                          dsl_deadlist_close(&ds->ds_deadlist);
 425  427                          kmem_free(ds, sizeof (dsl_dataset_t));
 426  428                          dmu_buf_rele(dbuf, tag);
 427  429                          return (err);
 428  430                  }
 429  431  
 430      -                if (!dsl_dataset_is_snapshot(ds)) {
      432 +                if (!ds->ds_is_snapshot) {
 431  433                          ds->ds_snapname[0] = '\0';
 432  434                          if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 433  435                                  err = dsl_dataset_hold_obj(dp,
 434  436                                      dsl_dataset_phys(ds)->ds_prev_snap_obj,
 435  437                                      ds, &ds->ds_prev);
 436  438                          }
 437  439                          if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
 438  440                                  int zaperr = zap_lookup(mos, ds->ds_object,
 439  441                                      DS_FIELD_BOOKMARK_NAMES,
 440  442                                      sizeof (ds->ds_bookmarks), 1,
↓ open down ↓ 6 lines elided ↑ open up ↑
 447  449                                  err = dsl_dataset_get_snapname(ds);
 448  450                          if (err == 0 &&
 449  451                              dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
 450  452                                  err = zap_count(
 451  453                                      ds->ds_dir->dd_pool->dp_meta_objset,
 452  454                                      dsl_dataset_phys(ds)->ds_userrefs_obj,
 453  455                                      &ds->ds_userrefs);
 454  456                          }
 455  457                  }
 456  458  
 457      -                if (err == 0 && !dsl_dataset_is_snapshot(ds)) {
      459 +                if (err == 0 && !ds->ds_is_snapshot) {
 458  460                          err = dsl_prop_get_int_ds(ds,
 459  461                              zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 460  462                              &ds->ds_reserved);
 461  463                          if (err == 0) {
 462  464                                  err = dsl_prop_get_int_ds(ds,
 463  465                                      zfs_prop_to_name(ZFS_PROP_REFQUOTA),
 464  466                                      &ds->ds_quota);
 465  467                          }
 466  468                  } else {
 467  469                          ds->ds_reserved = ds->ds_quota = 0;
 468  470                  }
 469  471  
 470      -                if (err != 0 || (winner = dmu_buf_set_user_ie(dbuf, ds,
 471      -                    dsl_dataset_evict)) != NULL) {
      472 +                dmu_buf_init_user(&ds->ds_dbu, dsl_dataset_evict, &ds->ds_dbuf);
      473 +                if (err == 0)
      474 +                        winner = dmu_buf_set_user_ie(dbuf, &ds->ds_dbu);
      475 +
      476 +                if (err != 0 || winner != NULL) {
 472  477                          bplist_destroy(&ds->ds_pending_deadlist);
 473  478                          dsl_deadlist_close(&ds->ds_deadlist);
 474  479                          if (ds->ds_prev)
 475  480                                  dsl_dataset_rele(ds->ds_prev, ds);
 476  481                          dsl_dir_rele(ds->ds_dir, ds);
 477  482                          mutex_destroy(&ds->ds_lock);
 478  483                          mutex_destroy(&ds->ds_opening_lock);
 479  484                          mutex_destroy(&ds->ds_sendstream_lock);
 480  485                          refcount_destroy(&ds->ds_longholds);
 481  486                          kmem_free(ds, sizeof (dsl_dataset_t));
↓ open down ↓ 359 lines elided ↑ open up ↑
 841  846   * the space in the most recent snapshot still in use, we need to take
 842  847   * the total space used in the snapshot and subtract out the space that
 843  848   * has been freed up since the snapshot was taken.
 844  849   */
 845  850  void
 846  851  dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
 847  852  {
 848  853          uint64_t mrs_used;
 849  854          uint64_t dlused, dlcomp, dluncomp;
 850  855  
 851      -        ASSERT(!dsl_dataset_is_snapshot(ds));
      856 +        ASSERT(!ds->ds_is_snapshot);
 852  857  
 853  858          if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
 854  859                  mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
 855  860          else
 856  861                  mrs_used = 0;
 857  862  
 858  863          dsl_deadlist_space(&ds->ds_deadlist, &dlused, &dlcomp, &dluncomp);
 859  864  
 860  865          ASSERT3U(dlused, <=, mrs_used);
 861  866          dsl_dataset_phys(ds)->ds_unique_bytes =
↓ open down ↓ 722 lines elided ↑ open up ↑
1584 1589          ASSERT(dsl_pool_config_held(dp));
1585 1590  
1586 1591          ratio = dsl_dataset_phys(ds)->ds_compressed_bytes == 0 ? 100 :
1587 1592              (dsl_dataset_phys(ds)->ds_uncompressed_bytes * 100 /
1588 1593              dsl_dataset_phys(ds)->ds_compressed_bytes);
1589 1594  
1590 1595          dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRATIO, ratio);
1591 1596          dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
1592 1597              dsl_dataset_phys(ds)->ds_uncompressed_bytes);
1593 1598  
1594      -        if (dsl_dataset_is_snapshot(ds)) {
     1599 +        if (ds->ds_is_snapshot) {
1595 1600                  dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO, ratio);
1596 1601                  dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
1597 1602                      dsl_dataset_phys(ds)->ds_unique_bytes);
1598 1603                  get_clones_stat(ds, nv);
1599 1604          } else {
1600 1605                  if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) {
1601 1606                          char buf[MAXNAMELEN];
1602 1607                          dsl_dataset_name(ds->ds_prev, buf);
1603 1608                          dsl_prop_nvlist_add_string(nv, ZFS_PROP_PREV_SNAP, buf);
1604 1609                  }
↓ open down ↓ 47 lines elided ↑ open up ↑
1652 1657  dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
1653 1658  {
1654 1659          dsl_pool_t *dp = ds->ds_dir->dd_pool;
1655 1660          ASSERT(dsl_pool_config_held(dp));
1656 1661  
1657 1662          stat->dds_creation_txg = dsl_dataset_phys(ds)->ds_creation_txg;
1658 1663          stat->dds_inconsistent =
1659 1664              dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT;
1660 1665          stat->dds_guid = dsl_dataset_phys(ds)->ds_guid;
1661 1666          stat->dds_origin[0] = '\0';
1662      -        if (dsl_dataset_is_snapshot(ds)) {
     1667 +        if (ds->ds_is_snapshot) {
1663 1668                  stat->dds_is_snapshot = B_TRUE;
1664 1669                  stat->dds_num_clones =
1665 1670                      dsl_dataset_phys(ds)->ds_num_children - 1;
1666 1671          } else {
1667 1672                  stat->dds_is_snapshot = B_FALSE;
1668 1673                  stat->dds_num_clones = 0;
1669 1674  
1670 1675                  if (dsl_dir_is_clone(ds->ds_dir)) {
1671 1676                          dsl_dataset_t *ods;
1672 1677  
↓ open down ↓ 239 lines elided ↑ open up ↑
1912 1917          dsl_pool_t *dp = dmu_tx_pool(tx);
1913 1918          dsl_dataset_t *ds;
1914 1919          int64_t unused_refres_delta;
1915 1920          int error;
1916 1921  
1917 1922          error = dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds);
1918 1923          if (error != 0)
1919 1924                  return (error);
1920 1925  
1921 1926          /* must not be a snapshot */
1922      -        if (dsl_dataset_is_snapshot(ds)) {
     1927 +        if (ds->ds_is_snapshot) {
1923 1928                  dsl_dataset_rele(ds, FTAG);
1924 1929                  return (SET_ERROR(EINVAL));
1925 1930          }
1926 1931  
1927 1932          /* must have a most recent snapshot */
1928 1933          if (dsl_dataset_phys(ds)->ds_prev_snap_txg < TXG_INITIAL) {
1929 1934                  dsl_dataset_rele(ds, FTAG);
1930 1935                  return (SET_ERROR(EINVAL));
1931 1936          }
1932 1937  
↓ open down ↓ 551 lines elided ↑ open up ↑
2484 2489          int error;
2485 2490          dsl_dir_t *dd;
2486 2491          struct promotenode *snap;
2487 2492  
2488 2493          error = dsl_dataset_hold(dp, ddpa->ddpa_clonename, tag,
2489 2494              &ddpa->ddpa_clone);
2490 2495          if (error != 0)
2491 2496                  return (error);
2492 2497          dd = ddpa->ddpa_clone->ds_dir;
2493 2498  
2494      -        if (dsl_dataset_is_snapshot(ddpa->ddpa_clone) ||
     2499 +        if (ddpa->ddpa_clone->ds_is_snapshot ||
2495 2500              !dsl_dir_is_clone(dd)) {
2496 2501                  dsl_dataset_rele(ddpa->ddpa_clone, tag);
2497 2502                  return (SET_ERROR(EINVAL));
2498 2503          }
2499 2504  
2500 2505          error = snaplist_make(dp, 0, dsl_dir_phys(dd)->dd_origin_obj,
2501 2506              &ddpa->shared_snaps, tag);
2502 2507          if (error != 0)
2503 2508                  goto out;
2504 2509  
↓ open down ↓ 71 lines elided ↑ open up ↑
2576 2581              2 + numsnaps, ZFS_SPACE_CHECK_RESERVED));
2577 2582  }
2578 2583  
2579 2584  int
2580 2585  dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
2581 2586      dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx)
2582 2587  {
2583 2588          int64_t unused_refres_delta;
2584 2589  
2585 2590          /* they should both be heads */
2586      -        if (dsl_dataset_is_snapshot(clone) ||
2587      -            dsl_dataset_is_snapshot(origin_head))
     2591 +        if (clone->ds_is_snapshot ||
     2592 +            origin_head->ds_is_snapshot)
2588 2593                  return (SET_ERROR(EINVAL));
2589 2594  
2590 2595          /* if we are not forcing, the branch point should be just before them */
2591 2596          if (!force && clone->ds_prev != origin_head->ds_prev)
2592 2597                  return (SET_ERROR(EINVAL));
2593 2598  
2594 2599          /* clone should be the clone (unless they are unrelated) */
2595 2600          if (clone->ds_prev != NULL &&
2596 2601              clone->ds_prev != clone->ds_dir->dd_pool->dp_origin_snap &&
2597 2602              origin_head->ds_dir != clone->ds_prev->ds_dir)
↓ open down ↓ 258 lines elided ↑ open up ↑
2856 2861          int error;
2857 2862          uint64_t newval;
2858 2863  
2859 2864          if (spa_version(dp->dp_spa) < SPA_VERSION_REFQUOTA)
2860 2865                  return (SET_ERROR(ENOTSUP));
2861 2866  
2862 2867          error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
2863 2868          if (error != 0)
2864 2869                  return (error);
2865 2870  
2866      -        if (dsl_dataset_is_snapshot(ds)) {
     2871 +        if (ds->ds_is_snapshot) {
2867 2872                  dsl_dataset_rele(ds, FTAG);
2868 2873                  return (SET_ERROR(EINVAL));
2869 2874          }
2870 2875  
2871 2876          error = dsl_prop_predict(ds->ds_dir,
2872 2877              zfs_prop_to_name(ZFS_PROP_REFQUOTA),
2873 2878              ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
2874 2879          if (error != 0) {
2875 2880                  dsl_dataset_rele(ds, FTAG);
2876 2881                  return (error);
↓ open down ↓ 62 lines elided ↑ open up ↑
2939 2944          int error;
2940 2945          uint64_t newval, unique;
2941 2946  
2942 2947          if (spa_version(dp->dp_spa) < SPA_VERSION_REFRESERVATION)
2943 2948                  return (SET_ERROR(ENOTSUP));
2944 2949  
2945 2950          error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
2946 2951          if (error != 0)
2947 2952                  return (error);
2948 2953  
2949      -        if (dsl_dataset_is_snapshot(ds)) {
     2954 +        if (ds->ds_is_snapshot) {
2950 2955                  dsl_dataset_rele(ds, FTAG);
2951 2956                  return (SET_ERROR(EINVAL));
2952 2957          }
2953 2958  
2954 2959          error = dsl_prop_predict(ds->ds_dir,
2955 2960              zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
2956 2961              ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
2957 2962          if (error != 0) {
2958 2963                  dsl_dataset_rele(ds, FTAG);
2959 2964                  return (error);
↓ open down ↓ 193 lines elided ↑ open up ↑
3153 3158   */
3154 3159  int
3155 3160  dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
3156 3161      dsl_dataset_t *lastsnap,
3157 3162      uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
3158 3163  {
3159 3164          int err = 0;
3160 3165          uint64_t snapobj;
3161 3166          dsl_pool_t *dp = firstsnap->ds_dir->dd_pool;
3162 3167  
3163      -        ASSERT(dsl_dataset_is_snapshot(firstsnap));
3164      -        ASSERT(dsl_dataset_is_snapshot(lastsnap));
     3168 +        ASSERT(firstsnap->ds_is_snapshot);
     3169 +        ASSERT(lastsnap->ds_is_snapshot);
3165 3170  
3166 3171          /*
3167 3172           * Check that the snapshots are in the same dsl_dir, and firstsnap
3168 3173           * is before lastsnap.
3169 3174           */
3170 3175          if (firstsnap->ds_dir != lastsnap->ds_dir ||
3171 3176              dsl_dataset_phys(firstsnap)->ds_creation_txg >
3172 3177              dsl_dataset_phys(lastsnap)->ds_creation_txg)
3173 3178                  return (SET_ERROR(EINVAL));
3174 3179  
↓ open down ↓ 104 lines elided ↑ open up ↑
3279 3284   */
3280 3285  boolean_t
3281 3286  dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
3282 3287          uint64_t earlier_txg)
3283 3288  {
3284 3289          dsl_pool_t *dp = later->ds_dir->dd_pool;
3285 3290          int error;
3286 3291          boolean_t ret;
3287 3292  
3288 3293          ASSERT(dsl_pool_config_held(dp));
3289      -        ASSERT(dsl_dataset_is_snapshot(earlier) || earlier_txg != 0);
     3294 +        ASSERT(earlier->ds_is_snapshot || earlier_txg != 0);
3290 3295  
3291 3296          if (earlier_txg == 0)
3292 3297                  earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
3293 3298  
3294      -        if (dsl_dataset_is_snapshot(later) &&
     3299 +        if (later->ds_is_snapshot &&
3295 3300              earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
3296 3301                  return (B_FALSE);
3297 3302  
3298 3303          if (later->ds_dir == earlier->ds_dir)
3299 3304                  return (B_TRUE);
3300 3305          if (!dsl_dir_is_clone(later->ds_dir))
3301 3306                  return (B_FALSE);
3302 3307  
3303 3308          if (dsl_dir_phys(later->ds_dir)->dd_origin_obj == earlier->ds_object)
3304 3309                  return (B_TRUE);
↓ open down ↓ 17 lines elided ↑ open up ↑
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX