Print this page
*** NO COMMENTS ***
*** 21,30 ****
--- 21,31 ----
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 by Delphix. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
+ * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
*/
#include <assert.h>
#include <ctype.h>
#include <errno.h>
*** 59,68 ****
--- 60,71 ----
static const zio_cksum_t zero_cksum = { 0 };
typedef struct dedup_arg {
int inputfd;
int outputfd;
+ uint64_t dedup_data_sz;
+ boolean_t sendsize;
libzfs_handle_t *dedup_hdl;
} dedup_arg_t;
typedef struct progress_arg {
zfs_handle_t *pa_zhp;
*** 182,191 ****
--- 185,212 ----
fletcher_4_incremental_native(buf, len, zc);
return (write(outfd, buf, len));
}
/*
+ * the function used by the cksummer thread that needs to know
+ * about the sendsize flag
+ */
+ static int
+ dedup_cksum_and_write(dedup_arg_t *dda, const void *buf, uint64_t len,
+ zio_cksum_t *zc, int outfd)
+ {
+ int ret = len;
+
+ dda->dedup_data_sz += len;
+ fletcher_4_incremental_native(buf, len, zc);
+ if (!dda->sendsize)
+ ret = (write(outfd, buf, len));
+
+ return (ret);
+ }
+
+ /*
* This function is started in a separate thread when the dedup option
* has been requested. The main send thread determines the list of
* snapshots to be included in the send stream and makes the ioctl calls
* for each one. But instead of having the ioctl send the output to the
* the output fd specified by the caller of zfs_send()), the
*** 259,269 ****
fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
fflags |= (DMU_BACKUP_FEATURE_DEDUP |
DMU_BACKUP_FEATURE_DEDUPPROPS);
DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
! if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
DMU_COMPOUNDSTREAM && drr->drr_payloadlen != 0) {
int sz = drr->drr_payloadlen;
--- 280,291 ----
fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
fflags |= (DMU_BACKUP_FEATURE_DEDUP |
DMU_BACKUP_FEATURE_DEDUPPROPS);
DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
! if (dedup_cksum_and_write(dda, drr,
! sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
DMU_COMPOUNDSTREAM && drr->drr_payloadlen != 0) {
int sz = drr->drr_payloadlen;
*** 273,284 ****
buf = malloc(sz);
}
(void) ssread(buf, sz, ofp);
if (ferror(stdin))
perror("fread");
! if (cksum_and_write(buf, sz, &stream_cksum,
! outfd) == -1)
goto out;
}
break;
}
--- 295,306 ----
buf = malloc(sz);
}
(void) ssread(buf, sz, ofp);
if (ferror(stdin))
perror("fread");
! if (dedup_cksum_and_write(dda, buf, sz,
! &stream_cksum, outfd) == -1)
goto out;
}
break;
}
*** 289,333 ****
stream_cksum.zc_word[0], stream_cksum.zc_word[1],
stream_cksum.zc_word[2], stream_cksum.zc_word[3]);
if ((write(outfd, drr,
sizeof (dmu_replay_record_t))) == -1)
goto out;
break;
}
case DRR_OBJECT:
{
! if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
if (drro->drr_bonuslen > 0) {
(void) ssread(buf,
P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
ofp);
! if (cksum_and_write(buf,
P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
&stream_cksum, outfd) == -1)
goto out;
}
break;
}
case DRR_SPILL:
{
! if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
(void) ssread(buf, drrs->drr_length, ofp);
! if (cksum_and_write(buf, drrs->drr_length,
&stream_cksum, outfd) == -1)
goto out;
break;
}
case DRR_FREEOBJECTS:
{
! if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
break;
}
--- 311,359 ----
stream_cksum.zc_word[0], stream_cksum.zc_word[1],
stream_cksum.zc_word[2], stream_cksum.zc_word[3]);
if ((write(outfd, drr,
sizeof (dmu_replay_record_t))) == -1)
goto out;
+ dda->dedup_data_sz += sizeof (dmu_replay_record_t);
break;
}
case DRR_OBJECT:
{
! if (dedup_cksum_and_write(dda, drr,
! sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
if (drro->drr_bonuslen > 0) {
(void) ssread(buf,
P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
ofp);
! if (dedup_cksum_and_write(dda, buf,
P2ROUNDUP((uint64_t)drro->drr_bonuslen, 8),
&stream_cksum, outfd) == -1)
goto out;
}
break;
}
case DRR_SPILL:
{
! if (dedup_cksum_and_write(dda, drr,
! sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
(void) ssread(buf, drrs->drr_length, ofp);
! if (dedup_cksum_and_write(dda, buf, drrs->drr_length,
&stream_cksum, outfd) == -1)
goto out;
break;
}
case DRR_FREEOBJECTS:
{
! if (dedup_cksum_and_write(dda, drr,
! sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
break;
}
*** 388,418 ****
wbr_drrr->drr_key.ddk_cksum =
drrw->drr_key.ddk_cksum;
wbr_drrr->drr_key.ddk_prop =
drrw->drr_key.ddk_prop;
! if (cksum_and_write(&wbr_drr,
sizeof (dmu_replay_record_t), &stream_cksum,
outfd) == -1)
goto out;
} else {
/* block not previously seen */
! if (cksum_and_write(drr,
sizeof (dmu_replay_record_t), &stream_cksum,
outfd) == -1)
goto out;
! if (cksum_and_write(buf,
drrw->drr_length,
&stream_cksum, outfd) == -1)
goto out;
}
break;
}
case DRR_FREE:
{
! if (cksum_and_write(drr, sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
break;
}
--- 414,445 ----
wbr_drrr->drr_key.ddk_cksum =
drrw->drr_key.ddk_cksum;
wbr_drrr->drr_key.ddk_prop =
drrw->drr_key.ddk_prop;
! if (dedup_cksum_and_write(dda, &wbr_drr,
sizeof (dmu_replay_record_t), &stream_cksum,
outfd) == -1)
goto out;
} else {
/* block not previously seen */
! if (dedup_cksum_and_write(dda, drr,
sizeof (dmu_replay_record_t), &stream_cksum,
outfd) == -1)
goto out;
! if (dedup_cksum_and_write(dda, buf,
drrw->drr_length,
&stream_cksum, outfd) == -1)
goto out;
}
break;
}
case DRR_FREE:
{
! if (dedup_cksum_and_write(dda, drr,
! sizeof (dmu_replay_record_t),
&stream_cksum, outfd) == -1)
goto out;
break;
}
*** 787,797 ****
const char *fromsnap;
const char *tosnap;
char prevsnap[ZFS_MAXNAMELEN];
uint64_t prevsnap_obj;
boolean_t seenfrom, seento, replicate, doall, fromorigin;
! boolean_t verbose, dryrun, parsable, progress;
int outfd;
boolean_t err;
nvlist_t *fss;
avl_tree_t *fsavl;
snapfilter_cb_t *filter_cb;
--- 814,827 ----
const char *fromsnap;
const char *tosnap;
char prevsnap[ZFS_MAXNAMELEN];
uint64_t prevsnap_obj;
boolean_t seenfrom, seento, replicate, doall, fromorigin;
! boolean_t verbose, dryrun, dedup, parsable, progress;
! boolean_t sendsize;
! uint32_t hdr_send_sz;
! uint64_t send_sz;
int outfd;
boolean_t err;
nvlist_t *fss;
avl_tree_t *fsavl;
snapfilter_cb_t *filter_cb;
*** 866,876 ****
* Dumps a backup of the given snapshot (incremental from fromsnap if it's not
* NULL) to the file descriptor specified by outfd.
*/
static int
dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, uint64_t fromsnap_obj,
! boolean_t fromorigin, int outfd, nvlist_t *debugnv)
{
zfs_cmd_t zc = { 0 };
libzfs_handle_t *hdl = zhp->zfs_hdl;
nvlist_t *thisdbg;
--- 896,907 ----
* Dumps a backup of the given snapshot (incremental from fromsnap if it's not
* NULL) to the file descriptor specified by outfd.
*/
static int
dump_ioctl(zfs_handle_t *zhp, const char *fromsnap, uint64_t fromsnap_obj,
! boolean_t fromorigin, int outfd, nvlist_t *debugnv,
! boolean_t sendsize, uint64_t *sendcounter)
{
zfs_cmd_t zc = { 0 };
libzfs_handle_t *hdl = zhp->zfs_hdl;
nvlist_t *thisdbg;
*** 880,889 ****
--- 911,922 ----
(void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name));
zc.zc_cookie = outfd;
zc.zc_obj = fromorigin;
zc.zc_sendobj = zfs_prop_get_int(zhp, ZFS_PROP_OBJSETID);
zc.zc_fromobj = fromsnap_obj;
+ zc.zc_sendsize = sendsize;
+ zc.zc_sendcounter = 0;
VERIFY(0 == nvlist_alloc(&thisdbg, NV_UNIQUE_NAME, 0));
if (fromsnap && fromsnap[0] != '\0') {
VERIFY(0 == nvlist_add_string(thisdbg,
"fromsnap", fromsnap));
*** 933,942 ****
--- 966,976 ----
default:
return (zfs_standard_error(hdl, errno, errbuf));
}
}
+ *sendcounter = (uint64_t)zc.zc_sendcounter;
if (debugnv)
VERIFY(0 == nvlist_add_nvlist(debugnv, zhp->zfs_name, thisdbg));
nvlist_free(thisdbg);
return (0);
*** 1115,1129 ****
}
fromorigin = sdd->prevsnap[0] == '\0' &&
(sdd->fromorigin || sdd->replicate);
if (sdd->verbose) {
! uint64_t size;
! err = estimate_ioctl(zhp, sdd->prevsnap_obj,
! fromorigin, &size);
!
if (sdd->parsable) {
if (sdd->prevsnap[0] != '\0') {
(void) fprintf(stderr, "incremental\t%s\t%s",
sdd->prevsnap, zhp->zfs_name);
} else {
--- 1149,1161 ----
}
fromorigin = sdd->prevsnap[0] == '\0' &&
(sdd->fromorigin || sdd->replicate);
+ /* print out to-from and approximate size in verbose mode */
if (sdd->verbose) {
! /* print preamble */
if (sdd->parsable) {
if (sdd->prevsnap[0] != '\0') {
(void) fprintf(stderr, "incremental\t%s\t%s",
sdd->prevsnap, zhp->zfs_name);
} else {
*** 1133,1164 ****
} else {
(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
"send from @%s to %s"),
sdd->prevsnap, zhp->zfs_name);
}
if (err == 0) {
if (sdd->parsable) {
(void) fprintf(stderr, "\t%llu\n",
(longlong_t)size);
} else {
char buf[16];
zfs_nicenum(size, buf, sizeof (buf));
! (void) fprintf(stderr, dgettext(TEXT_DOMAIN,
! " estimated size is %s\n"), buf);
}
sdd->size += size;
} else {
(void) fprintf(stderr, "\n");
}
}
if (!sdd->dryrun) {
/*
* If progress reporting is requested, spawn a new thread to
* poll ZFS_IOC_SEND_PROGRESS at a regular interval.
*/
! if (sdd->progress) {
pa.pa_zhp = zhp;
pa.pa_fd = sdd->outfd;
pa.pa_parsable = sdd->parsable;
if (err = pthread_create(&tid, NULL,
--- 1165,1218 ----
} else {
(void) fprintf(stderr, dgettext(TEXT_DOMAIN,
"send from @%s to %s"),
sdd->prevsnap, zhp->zfs_name);
}
+
+ if (sdd->sendsize) {
+ /*
+ * we are going to print out the exact stream size info,
+ * so skip the estimate
+ */
+ (void) fprintf(stderr, "\n");
+ } else {
+ /*
+ * provide stream size estimate otherwise
+ */
+ uint64_t size;
+ err = estimate_ioctl(zhp, sdd->prevsnap_obj,
+ fromorigin, &size);
+
if (err == 0) {
if (sdd->parsable) {
(void) fprintf(stderr, "\t%llu\n",
(longlong_t)size);
} else {
char buf[16];
zfs_nicenum(size, buf, sizeof (buf));
! (void) fprintf(stderr,
! dgettext(TEXT_DOMAIN,
! " estimated size is %s\n"),
! buf);
}
sdd->size += size;
} else {
+ /* could not estimate */
(void) fprintf(stderr, "\n");
}
}
+ }
if (!sdd->dryrun) {
+ uint64_t sendcounter = 0;
+ boolean_t track_progress = (sdd->progress && !sdd->sendsize);
+ boolean_t sendsize = B_FALSE;
/*
* If progress reporting is requested, spawn a new thread to
* poll ZFS_IOC_SEND_PROGRESS at a regular interval.
*/
! if (track_progress) {
pa.pa_zhp = zhp;
pa.pa_fd = sdd->outfd;
pa.pa_parsable = sdd->parsable;
if (err = pthread_create(&tid, NULL,
*** 1166,1179 ****
zfs_close(zhp);
return (err);
}
}
err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
! fromorigin, sdd->outfd, sdd->debugnv);
! if (sdd->progress) {
(void) pthread_cancel(tid);
(void) pthread_join(tid, NULL);
}
}
--- 1220,1249 ----
zfs_close(zhp);
return (err);
}
}
+
+ /*
+ * We need to reset the sendsize flag being sent to
+ * kernel if sdd->dedup is set. With dedup, the file
+ * descriptor sent to kernel is one end of the pipe,
+ * and we would want the data back in the pipe for
+ * cksummer() to calculate the exact size of the dedup-ed
+ * stream. So reset the sendsize flag such that
+ * kernel writes to the pipe.
+ */
+
+ sendsize = sdd->dedup ? B_FALSE : sdd->sendsize;
+
err = dump_ioctl(zhp, sdd->prevsnap, sdd->prevsnap_obj,
! fromorigin, sdd->outfd, sdd->debugnv,
! sendsize, &sendcounter);
! sdd->send_sz += sendcounter;
!
! if (track_progress) {
(void) pthread_cancel(tid);
(void) pthread_join(tid, NULL);
}
}
*** 1412,1421 ****
--- 1482,1492 ----
errbuf));
}
dda.outputfd = outfd;
dda.inputfd = pipefd[1];
dda.dedup_hdl = zhp->zfs_hdl;
+ dda.sendsize = flags->sendsize;
if (err = pthread_create(&tid, NULL, cksummer, &dda)) {
(void) close(pipefd[0]);
(void) close(pipefd[1]);
zfs_error_aux(zhp->zfs_hdl, strerror(errno));
return (zfs_error(zhp->zfs_hdl,
*** 1472,1486 ****
--- 1543,1559 ----
(void) snprintf(drr.drr_u.drr_begin.drr_toname,
sizeof (drr.drr_u.drr_begin.drr_toname),
"%s@%s", zhp->zfs_name, tosnap);
drr.drr_payloadlen = buflen;
err = cksum_and_write(&drr, sizeof (drr), &zc, outfd);
+ sdd.hdr_send_sz += sizeof (drr);
/* write header nvlist */
if (err != -1 && packbuf != NULL) {
err = cksum_and_write(packbuf, buflen, &zc,
outfd);
+ sdd.hdr_send_sz += buflen;
}
free(packbuf);
if (err == -1) {
fsavl_destroy(fsavl);
nvlist_free(fss);
*** 1491,1500 ****
--- 1564,1574 ----
/* write end record */
bzero(&drr, sizeof (drr));
drr.drr_type = DRR_END;
drr.drr_u.drr_end.drr_checksum = zc;
err = write(outfd, &drr, sizeof (drr));
+ sdd.hdr_send_sz += sizeof (drr);
if (err == -1) {
fsavl_destroy(fsavl);
nvlist_free(fss);
err = errno;
goto stderr_out;
*** 1515,1524 ****
--- 1589,1600 ----
sdd.doall = flags->doall;
sdd.fromorigin = flags->fromorigin;
sdd.fss = fss;
sdd.fsavl = fsavl;
sdd.verbose = flags->verbose;
+ sdd.dedup = flags->dedup;
+ sdd.sendsize = flags->sendsize;
sdd.parsable = flags->parsable;
sdd.progress = flags->progress;
sdd.dryrun = flags->dryrun;
sdd.filter_cb = filter_func;
sdd.filter_cb_arg = cb_arg;
*** 1545,1555 ****
goto stderr_out;
}
} else {
sdd.cleanup_fd = -1;
}
! if (flags->verbose) {
/*
* Do a verbose no-op dry run to get all the verbose output
* before generating any data. Then do a non-verbose real
* run to generate the streams.
*/
--- 1621,1631 ----
goto stderr_out;
}
} else {
sdd.cleanup_fd = -1;
}
! if (flags->verbose && !flags->sendsize) {
/*
* Do a verbose no-op dry run to get all the verbose output
* before generating any data. Then do a non-verbose real
* run to generate the streams.
*/
*** 1572,1581 ****
--- 1648,1658 ----
nvlist_free(fss);
if (flags->dedup) {
(void) close(pipefd[0]);
(void) pthread_join(tid, NULL);
+ sdd.send_sz = dda.dedup_data_sz;
}
if (sdd.cleanup_fd != -1) {
VERIFY(0 == close(sdd.cleanup_fd));
sdd.cleanup_fd = -1;
*** 1592,1602 ****
--- 1669,1694 ----
drr.drr_type = DRR_END;
if (write(outfd, &drr, sizeof (drr)) == -1) {
return (zfs_standard_error(zhp->zfs_hdl,
errno, errbuf));
}
+ sdd.hdr_send_sz += sizeof (drr);
}
+
+ if (flags->sendsize) {
+ if (flags->verbose) {
+ fprintf(stderr, "Send stream header size (bytes): "
+ "%u\n", sdd.hdr_send_sz);
+ fprintf(stderr, "Send stream data size (bytes): "
+ "%llu\n", sdd.send_sz);
+ fprintf(stderr, "Total send stream size (bytes): "
+ "%llu\n", sdd.send_sz + (uint64_t)sdd.hdr_send_sz);
+ } else {
+ fprintf(stderr, "Total send stream size (bytes): "
+ "%llu\n", sdd.send_sz + (uint64_t)sdd.hdr_send_sz);
+ }
+ }
return (err || sdd.err);
stderr_out:
err = zfs_standard_error(zhp->zfs_hdl, err, errbuf);