1 #!/usr/bin/ksh
2 #
3 # nfsvsvrtop - display top NFS v3 & v4 I/O events on a server.
4 #
5 # This is measuring the response time between an incoming NFS operation
6 # and its response. In general, this measures the server's view of how
7 # quickly it can respond to requests. By default, the list shows responses
8 # to each client.
9 #
10 # Top-level fields:
11 # load 1 min load average
12 # read total KB read during sample
13 # swrite total KB sync writes during sample
14 # awrite total KB async writes during sample
15 #
16 # The following per-client and "all" clients fields are shown
17 # Ver NFS version (3 or 4)
18 # Client IP addr of client
19 # NFSOPS NFS operations per second
20 # Reads Read operations per second
21 # SWrites Sync write operations per second
22 # AWrites Async write operations per second
23 # Commits Commits per second
24 # Rd_bw Read KB/sec
25 # SWr_bw Sync write KB/sec
26 # AWr_bw Async write KB/sec
27 # Rd_t Average read time in microseconds
28 # SWr_t Average sync write time in microseconds
29 # AWr_t Average async write time in microseconds
30 # Com_t Average commit time in microseconds
31 # Align% Percentage of read/write operations that have offset aligned to
32 # blocksize (default=4096 bytes)
33 #
34 # Note: NFSv4 compound operations are not measured, per se, but they are
35 # counted in the total operations count.
36 #
37 # Note: dtrace doesn't do floating point. A seemingly zero response or
38 # count can result due to integer division.
39 #
40 #
41 # INSPIRATION: top(1) by William LeFebvre and iotop by Brendan Gregg
42 #
43 # Copyright 2011, Nexenta Systems, Inc. All rights reserved.
44 #
45 # CDDL HEADER START
46 #
47 # The contents of this file are subject to the terms of the
48 # Common Development and Distribution License, Version 1.0 only
49 # (the "License"). You may not use this file except in compliance
50 # with the License.
51 #
52 # You can obtain a copy of the license at Docs/cddl1.txt
53 # or http://www.opensolaris.org/os/licensing.
54 # See the License for the specific language governing permissions
55 # and limitations under the License.
56 #
57 # CDDL HEADER END
58 #
59 # Author: Richard.Elling@RichardElling.com
60 #
61 # Revision:
62 # 1.8 29-Nov-2012
63 #
64 # TODO: running count of nfsd threads (done efficiently)
65 # TODO: mount point filter
66 PATH=/usr/sbin:/usr/bin
67
68 ##############################
69 # check to see if the NFS server module is loaded
70 # if not, then the dtrace probes will fail ungracefully
71 if [ "$(uname -s)" = "SunOS" ]; then
72 modinfo | awk '{print $6}' | grep -q nfssrv
73 if [ $? != 0 ]; then
74 echo "error: NFS server module is not loaded, are you serving NFS?"
75 exit 1
76 fi
77 fi
78
79 ##############################
80 # --- Process Arguments ---
81 #
82
83 ### default variables
84 opt_blocksize=4096 # blocksize for alignment measurements
85 opt_client=0 # set if -c option set
86 opt_clear=1 # set if screen to be cleared
87 opt_json=0 # set if output is JSON
88 opt_top=0 # set if list trimmed to top
89 top=0 # number of lines trimmed
90 opt_vers=0 # set if NFS version restricted
91 vers=3 # version of NFS to restrict
92 interval=10 # default interval
93 count=-1 # number of intervals to show
94
95 ### process options
96 while getopts b:c:Cjn:t: name
97 do
98 case $name in
99 b) opt_blocksize=$OPTARG ;;
100 c) opt_client=1; client_IP=$OPTARG ;;
101 C) opt_clear=0 ;;
102 j) opt_json=1 ;;
103 n) opt_vers=1; vers=$OPTARG ;;
104 t) opt_top=1; top=$OPTARG ;;
105 h|?) cat <<END >&2
106 USAGE: nfssvrtop [-Cj] [-b blocksize] [-c client_IP] [-n vers] [-t top]
107 [interval [count]]
108 -b blocksize # alignment blocksize (default=4096)
109 -c client_IP # trace for this client only
110 -C # don't clear the screen
111 -j # print output in JSON format
112 -n vers # show only NFS version
113 -t top # print top number of entries only
114 examples:
115 nfssvrtop # default output, 10 second samples
116 nfssvrtop -b 1024 # check alignment on 1KB boundary
117 nfssvrtop 1 # 1 second samples
118 nfssvrtop -n 4 # only show NFSv4 traffic
119 nfssvrtop -C 60 # 60 second samples, do not clear screen
120 nfssvrtop -t 20 # print top 20 lines only
121 nfssvrtop 5 12 # print 12 x 5 second samples
122 END
123 exit 1
124 esac
125 done
126
127 shift $(($OPTIND - 1))
128
129 ### option logic
130 if [ ! -z "$1" ]; then
131 interval=$1; shift
132 fi
133 if [ ! -z "$1" ]; then
134 count=$1; shift
135 fi
136 if [ $opt_clear = 1 ]; then
137 clearstr=$(clear)
138 else
139 clearstr=""
140 fi
141
142 #################################
143 # --- Main Program, DTrace ---
144 #
145 /usr/sbin/dtrace -Cn '
146 /*
147 * Command line arguments
148 */
149 inline int OPT_blocksize = '$opt_blocksize';
150 inline int OPT_client = '$opt_client';
151 inline int OPT_clear = '$opt_clear';
152 inline int OPT_top = '$opt_top';
153 inline int OPT_json = '$opt_json';
154 inline int OPT_vers = '$opt_vers';
155 inline int INTERVAL = '$interval';
156 inline int COUNTER = '$count';
157 inline int TOP = '$top';
158 inline string CLIENT = "'$client_IP'";
159 inline int VERS = '$vers';
160 inline string CLEAR = "'$clearstr'";
161
162 #pragma D option quiet
163
164 /*
165 * increase dynvarsize if you get "dynamic variable drops"
166 */
167 #pragma D option dynvarsize=12m
168
169 /*
170 * Print header
171 */
172 dtrace:::BEGIN
173 {
174 /* starting values */
175 counts = COUNTER;
176 secs = INTERVAL;
177 total_read_b = 0;
178 total_swrite_b = 0;
179 total_awrite_b = 0;
180
181 OPT_json ? 1 : printf("Tracing... Please wait.\n");
182 }
183
184 /*
185 * Filter as needed, based on starts
186 */
187 nfsv3:nfssrv::op-access-start,
188 nfsv3:nfssrv::op-create-start,
189 nfsv3:nfssrv::op-commit-start,
190 nfsv3:nfssrv::op-fsinfo-start,
191 nfsv3:nfssrv::op-fsstat-start,
192 nfsv3:nfssrv::op-getattr-start,
193 nfsv3:nfssrv::op-link-start,
194 nfsv3:nfssrv::op-lookup-start,
195 nfsv3:nfssrv::op-mkdir-start,
196 nfsv3:nfssrv::op-mknod-start,
197 nfsv3:nfssrv::op-null-start,
198 nfsv3:nfssrv::op-pathconf-start,
199 nfsv3:nfssrv::op-read-start,
200 nfsv3:nfssrv::op-readdir-start,
201 nfsv3:nfssrv::op-readdirplus-start,
202 nfsv3:nfssrv::op-readlink-start,
203 nfsv3:nfssrv::op-remove-start,
204 nfsv3:nfssrv::op-rename-start,
205 nfsv3:nfssrv::op-rmdir-start,
206 nfsv3:nfssrv::op-setattr-start,
207 nfsv3:nfssrv::op-symlink-start,
208 nfsv3:nfssrv::op-write-start
209 /OPT_client == 0 || CLIENT == args[0]->ci_remote/
210 {
211 self->vers = "3";
212 @c_nfsops[self->vers, args[0]->ci_remote] = count();
213 OPT_client == 0 ? @c_nfsops[self->vers, "all"] = count() : 1;
214 }
215
216 nfsv4:nfssrv::cb-recall-start,
217 nfsv4:nfssrv::compound-start,
218 nfsv4:nfssrv::null-start,
219 nfsv4:nfssrv::op-access-start,
220 nfsv4:nfssrv::op-close-start,
221 nfsv4:nfssrv::op-commit-start,
222 nfsv4:nfssrv::op-create-start,
223 nfsv4:nfssrv::op-delegpurge-start,
224 nfsv4:nfssrv::op-delegreturn-start,
225 nfsv4:nfssrv::op-getattr-start,
226 nfsv4:nfssrv::op-getfh-start,
227 nfsv4:nfssrv::op-link-start,
228 nfsv4:nfssrv::op-lock-start,
229 nfsv4:nfssrv::op-lockt-start,
230 nfsv4:nfssrv::op-locku-start,
231 nfsv4:nfssrv::op-lookup-start,
232 nfsv4:nfssrv::op-lookupp-start,
233 nfsv4:nfssrv::op-nverify-start,
234 nfsv4:nfssrv::op-open-confirm-start,
235 nfsv4:nfssrv::op-open-downgrade-start,
236 nfsv4:nfssrv::op-open-start,
237 nfsv4:nfssrv::op-openattr-start,
238 nfsv4:nfssrv::op-putfh-start,
239 nfsv4:nfssrv::op-putpubfh-start,
240 nfsv4:nfssrv::op-putrootfh-start,
241 nfsv4:nfssrv::op-read-start,
242 nfsv4:nfssrv::op-readdir-start,
243 nfsv4:nfssrv::op-readlink-start,
244 nfsv4:nfssrv::op-release-lockowner-start,
245 nfsv4:nfssrv::op-remove-start,
246 nfsv4:nfssrv::op-rename-start,
247 nfsv4:nfssrv::op-renew-start,
248 nfsv4:nfssrv::op-restorefh-start,
249 nfsv4:nfssrv::op-savefh-start,
250 nfsv4:nfssrv::op-secinfo-start,
251 nfsv4:nfssrv::op-setattr-start,
252 nfsv4:nfssrv::op-setclientid-confirm-start,
253 nfsv4:nfssrv::op-setclientid-start,
254 nfsv4:nfssrv::op-verify-start,
255 nfsv4:nfssrv::op-write-start
256 /OPT_client == 0 || CLIENT == args[0]->ci_remote/
257 {
258 self->vers = "4";
259 @c_nfsops[self->vers, args[0]->ci_remote] = count();
260 OPT_client == 0 ? @c_nfsops[self->vers, "all"] = count() : 1;
261 }
262
263 /* measure response time for commits, reads, and writes */
264 nfsv3:nfssrv::op-commit-start,
265 nfsv3:nfssrv::op-read-start,
266 nfsv3:nfssrv::op-write-start,
267 nfsv4:nfssrv::op-commit-start,
268 nfsv4:nfssrv::op-read-start,
269 nfsv4:nfssrv::op-write-start
270 /OPT_client == 0 || CLIENT == args[0]->ci_remote/
271 {
272 self->startts = timestamp;
273 }
274
275
276 /*
277 * commit
278 */
279 nfsv3:nfssrv::op-commit-start,
280 nfsv4:nfssrv::op-commit-start
281 /self->startts/
282 {
283 @c_commit_client[self->vers, args[0]->ci_remote] = count();
284 OPT_client == 0 ? @c_commit_client[self->vers, "all"] = count() : 1;
285 }
286
287 nfsv3:nfssrv::op-commit-done,
288 nfsv4:nfssrv::op-commit-done
289 /self->startts/
290 {
291 t = timestamp - self->startts;
292 @avgtime_commit[self->vers, args[0]->ci_remote] = avg(t);
293 OPT_client == 0 ? @avgtime_commit[self->vers, "all"] = avg(t) : 1;
294 self->startts = 0;
295 }
296
297 /*
298 * read
299 */
300 nfsv3:nfssrv::op-read-start,
301 nfsv4:nfssrv::op-read-start
302 /self->startts/
303 {
304 @c_read_client[self->vers, args[0]->ci_remote] = count();
305 OPT_client == 0 ? @c_read_client[self->vers, "all"] = count() : 1;
306 @read_b[self->vers, args[0]->ci_remote] = sum(args[2]->count);
307 OPT_client == 0 ? @read_b[self->vers, "all"] = sum(args[2]->count) : 1;
308 total_read_b += args[2]->count;
309 @avg_aligned[self->vers, args[0]->ci_remote] =
310 avg((args[2]->offset % OPT_blocksize) ? 0 : 100);
311 @avg_aligned[self->vers, "all"] =
312 avg((args[2]->offset % OPT_blocksize) ? 0 : 100);
313 }
314
315 nfsv3:nfssrv::op-read-done,
316 nfsv4:nfssrv::op-read-done
317 /self->startts/
318 {
319 t = timestamp - self->startts;
320 @avgtime_read[self->vers, args[0]->ci_remote] = avg(t);
321 OPT_client == 0 ? @avgtime_read[self->vers, "all"] = avg(t) : 1;
322 self->startts = 0;
323 }
324
325 /*
326 * write (sync)
327 */
328 nfsv3:nfssrv::op-write-start,
329 nfsv4:nfssrv::op-write-start
330 /self->startts/
331 {
332 @avg_aligned[self->vers, args[0]->ci_remote] =
333 avg((args[2]->offset % OPT_blocksize) ? 0 : 100);
334 @avg_aligned[self->vers, "all"] =
335 avg((args[2]->offset % OPT_blocksize) ? 0 : 100);
336 }
337
338 nfsv3:nfssrv::op-write-start
339 /self->startts && args[2]->stable/
340 {
341 self->issync = 1;
342 data_len = args[2]->data.data_len;
343 @c_swrite_client[self->vers, args[0]->ci_remote] = count();
344 OPT_client == 0 ? @c_swrite_client[self->vers, "all"] = count() : 1;
345 @swrite_b[self->vers, args[0]->ci_remote] = sum(data_len);
346 OPT_client == 0 ? @swrite_b[self->vers, "all"] = sum(data_len) : 1;
347 total_swrite_b += data_len;
348 }
349
350 nfsv4:nfssrv::op-write-start
351 /self->startts && args[2]->stable/
352 {
353 self->issync = 1;
354 data_len = args[2]->data_len;
355 @c_swrite_client[self->vers, args[0]->ci_remote] = count();
356 OPT_client == 0 ? @c_swrite_client[self->vers, "all"] = count() : 1;
357 @swrite_b[self->vers, args[0]->ci_remote] = sum(data_len);
358 OPT_client == 0 ? @swrite_b[self->vers, "all"] = sum(data_len) : 1;
359 total_swrite_b += data_len;
360 }
361
362 nfsv3:nfssrv::op-write-done,
363 nfsv4:nfssrv::op-write-done
364 /self->startts && self->issync/
365 {
366 t = timestamp - self->startts;
367 @avgtime_swrite[self->vers, args[0]->ci_remote] = avg(t);
368 OPT_client == 0 ? @avgtime_swrite[self->vers, "all"] = avg(t) : 1;
369 self->startts = 0;
370 }
371
372 /*
373 * write (async)
374 */
375 nfsv3:nfssrv::op-write-start
376 /self->startts && !args[2]->stable/
377 {
378 self->issync = 0;
379 data_len = args[2]->data.data_len;
380 @c_awrite_client[self->vers, args[0]->ci_remote] = count();
381 OPT_client == 0 ? @c_awrite_client[self->vers, "all"] = count() : 1;
382 @awrite_b[self->vers, args[0]->ci_remote] = sum(data_len);
383 OPT_client == 0 ? @awrite_b[self->vers, "all"] = sum(data_len) : 1;
384 total_awrite_b += data_len;
385 }
386
387 nfsv4:nfssrv::op-write-start
388 /self->startts && !args[2]->stable/
389 {
390 self->issync = 0;
391 data_len = args[2]->data_len;
392 @c_awrite_client[self->vers, args[0]->ci_remote] = count();
393 OPT_client == 0 ? @c_awrite_client[self->vers, "all"] = count() : 1;
394 @awrite_b[self->vers, args[0]->ci_remote] = sum(data_len);
395 OPT_client == 0 ? @awrite_b[self->vers, "all"] = sum(data_len) : 1;
396 total_awrite_b += data_len;
397 }
398
399 nfsv3:nfssrv::op-write-done,
400 nfsv4:nfssrv::op-write-done
401 /self->startts && !self->issync/
402 {
403 t = timestamp - self->startts;
404 @avgtime_awrite[self->vers, args[0]->ci_remote] = avg(t);
405 OPT_client == 0 ? @avgtime_awrite[self->vers, "all"] = avg(t) : 1;
406 self->startts = 0;
407 }
408
409 /*
410 * timer
411 */
412 profile:::tick-1sec
413 {
414 secs--;
415 }
416
417 /*
418 * Print report
419 */
420 profile:::tick-1sec
421 /secs == 0/
422 {
423 /* fetch 1 min load average */
424 self->load1a = `hp_avenrun[0] / 65536;
425 self->load1b = ((`hp_avenrun[0] % 65536) * 100) / 65536;
426
427 /* convert counters to Kbytes */
428 total_read_b /= 1024;
429 total_swrite_b /= 1024;
430 total_awrite_b /= 1024;
431
432 /* normalize to seconds giving a rate */
433 /* todo: this should be measured, not based on the INTERVAL */
434 normalize(@c_nfsops, INTERVAL);
435 normalize(@c_read_client, INTERVAL);
436 normalize(@c_swrite_client, INTERVAL);
437 normalize(@c_awrite_client, INTERVAL);
438 normalize(@c_commit_client, INTERVAL);
439
440 /* normalize to KB per second */
441 normalize(@read_b, 1024 * INTERVAL);
442 normalize(@awrite_b, 1024 * INTERVAL);
443 normalize(@swrite_b, 1024 * INTERVAL);
444
445 /* normalize average to microseconds */
446 normalize(@avgtime_read, 1000);
447 normalize(@avgtime_swrite, 1000);
448 normalize(@avgtime_awrite, 1000);
449 normalize(@avgtime_commit, 1000);
450
451 /* print status */
452 OPT_clear && !OPT_json ? printf("%s", CLEAR) : 1;
453
454 OPT_json ?
455 printf("{ \"collector\": \"nfssvrtop\", \"time\": \"%Y\", \"timestamp\": %d, \"interval\": %d, \"load\": %d.%02d, \"read_KB_int\": %d, \"sync_write_KB_int\": %d, \"async_write_KB_int\": %d, \"clientdata\": [",
456 walltimestamp, walltimestamp, INTERVAL,
457 self->load1a, self->load1b,
458 total_read_b, total_swrite_b, total_awrite_b)
459 :
460 printf("%Y, load: %d.%02d, read: %-8d KB, swrite: %-8d KB, awrite: %-8d KB\n",
461 walltimestamp, self->load1a, self->load1b,
462 total_read_b, total_swrite_b, total_awrite_b);
463
464 /* print headers */
465 OPT_json ? 1 :
466 printf("%s\t%-15s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\t%7s\n",
467 "Ver", "Client", "NFSOPS",
468 "Reads", "SWrites", "AWrites", "Commits",
469 "Rd_bw", "SWr_bw", "AWr_bw",
470 "Rd_t", "SWr_t", "AWr_t", "Com_t", "Align%");
471
472 /* truncate to top lines if needed */
473 OPT_top ? trunc(@c_nfsops, TOP) : 1;
474
475 OPT_json ?
476 printa("{\"version\": \"%s\", \"address\": \"%s\", \"NFSOPS\": %@d, \"reads\": %@d, \"sync_writes\": %@d, \"async_writes\": %@d, \"commits\": %@d, \"read_bw\": %@d, \"sync_write_bw\": %@d, \"async_write_bw\": %@d, \"avg_read_t\": %@d, \"avg_sync_write_t\": %@d, \"avg_async_write_t\": %@d, \"avg_commit_t\": %@d, \"aligned_pct\": %@d },",
477 @c_nfsops, @c_read_client, @c_swrite_client, @c_awrite_client,
478 @c_commit_client, @read_b, @swrite_b, @awrite_b,
479 @avgtime_read, @avgtime_swrite, @avgtime_awrite,
480 @avgtime_commit, @avg_aligned)
481 :
482 printa("%s\t%-15s\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\t%7@d\n",
483 @c_nfsops, @c_read_client, @c_swrite_client, @c_awrite_client,
484 @c_commit_client, @read_b, @swrite_b, @awrite_b,
485 @avgtime_read, @avgtime_swrite, @avgtime_awrite,
486 @avgtime_commit, @avg_aligned);
487
488 OPT_json ? printf("{}]}\n") : 1;
489
490 /* clear data */
491 trunc(@c_nfsops); trunc(@c_read_client); trunc(@c_swrite_client);
492 trunc(@c_awrite_client); trunc(@c_commit_client);
493 trunc(@read_b); trunc(@awrite_b); trunc(@swrite_b);
494 trunc(@avgtime_read); trunc(@avgtime_swrite); trunc(@avgtime_awrite);
495 trunc(@avgtime_commit); trunc(@avg_aligned);
496 total_read_b = 0;
497 total_swrite_b = 0;
498 total_awrite_b = 0;
499 secs = INTERVAL;
500 counts--;
501 }
502
503 /*
504 * end of program
505 */
506 profile:::tick-1sec
507 /counts == 0/
508 {
509 exit(0);
510 }
511
512 /*
513 * clean up when interrupted
514 */
515 dtrace:::END
516 {
517 trunc(@c_nfsops); trunc(@c_read_client); trunc(@c_swrite_client);
518 trunc(@c_awrite_client); trunc(@c_commit_client);
519 trunc(@read_b); trunc(@awrite_b); trunc(@swrite_b);
520 trunc(@avgtime_read); trunc(@avgtime_swrite); trunc(@avgtime_awrite);
521 trunc(@avgtime_commit); trunc(@avg_aligned);
522 }
523 '