194
195 list_insert_head(&zf->zf_stream, zs);
196 }
197
198 /*
199 * This is the predictive prefetch entry point. It associates dnode access
200 * specified with blkid and nblks arguments with prefetch stream, predicts
201 * further accesses based on that stats and initiates speculative prefetch.
202 * fetch_data argument specifies whether actual data blocks should be fetched:
203 * FALSE -- prefetch only indirect blocks for predicted data blocks;
204 * TRUE -- prefetch predicted data blocks plus following indirect blocks.
205 */
206 void
207 dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data)
208 {
209 zstream_t *zs;
210 int64_t pf_start, ipf_start, ipf_istart, ipf_iend;
211 int64_t pf_ahead_blks, max_blks;
212 int epbs, max_dist_blks, pf_nblks, ipf_nblks;
213 uint64_t end_of_access_blkid = blkid + nblks;
214 spa_t *spa = zf->zf_dnode->dn_objset->os_spa;
215
216 if (zfs_prefetch_disable)
217 return;
218
219 /*
220 * If we haven't yet loaded the indirect vdevs' mappings, we
221 * can only read from blocks that we carefully ensure are on
222 * concrete vdevs (or previously-loaded indirect vdevs). So we
223 * can't allow the predictive prefetcher to attempt reads of other
224 * blocks (e.g. of the MOS's dnode obejct).
225 */
226 if (!spa_indirect_vdevs_loaded(spa))
227 return;
228
229 /*
230 * As a fast path for small (single-block) files, ignore access
231 * to the first block.
232 */
233 if (blkid == 0)
234 return;
235
236 rw_enter(&zf->zf_rwlock, RW_READER);
237
238 /*
239 * Find matching prefetch stream. Depending on whether the accesses
240 * are block-aligned, first block of the new access may either follow
241 * the last block of the previous access, or be equal to it.
242 */
243 for (zs = list_head(&zf->zf_stream); zs != NULL;
244 zs = list_next(&zf->zf_stream, zs)) {
245 if (blkid == zs->zs_blkid || blkid + 1 == zs->zs_blkid) {
246 mutex_enter(&zs->zs_lock);
247 /*
248 * zs_blkid could have changed before we
249 * acquired zs_lock; re-check them here.
250 */
251 if (blkid == zs->zs_blkid) {
252 break;
253 } else if (blkid + 1 == zs->zs_blkid) {
254 blkid++;
255 nblks--;
256 if (nblks == 0) {
257 /* Already prefetched this before. */
258 mutex_exit(&zs->zs_lock);
259 rw_exit(&zf->zf_rwlock);
260 return;
261 }
262 break;
263 }
264 mutex_exit(&zs->zs_lock);
265 }
266 }
267
268 if (zs == NULL) {
269 /*
270 * This access is not part of any existing stream. Create
271 * a new stream for it.
272 */
273 ZFETCHSTAT_BUMP(zfetchstat_misses);
274 if (rw_tryupgrade(&zf->zf_rwlock))
275 dmu_zfetch_stream_create(zf, end_of_access_blkid);
276 rw_exit(&zf->zf_rwlock);
277 return;
278 }
279
280 /*
281 * This access was to a block that we issued a prefetch for on
282 * behalf of this stream. Issue further prefetches for this stream.
283 *
284 * Normally, we start prefetching where we stopped
285 * prefetching last (zs_pf_blkid). But when we get our first
286 * hit on this stream, zs_pf_blkid == zs_blkid, we don't
|
194
195 list_insert_head(&zf->zf_stream, zs);
196 }
197
198 /*
199 * This is the predictive prefetch entry point. It associates dnode access
200 * specified with blkid and nblks arguments with prefetch stream, predicts
201 * further accesses based on that stats and initiates speculative prefetch.
202 * fetch_data argument specifies whether actual data blocks should be fetched:
203 * FALSE -- prefetch only indirect blocks for predicted data blocks;
204 * TRUE -- prefetch predicted data blocks plus following indirect blocks.
205 */
206 void
207 dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data)
208 {
209 zstream_t *zs;
210 int64_t pf_start, ipf_start, ipf_istart, ipf_iend;
211 int64_t pf_ahead_blks, max_blks;
212 int epbs, max_dist_blks, pf_nblks, ipf_nblks;
213 uint64_t end_of_access_blkid = blkid + nblks;
214
215 if (zfs_prefetch_disable)
216 return;
217
218 /*
219 * As a fast path for small (single-block) files, ignore access
220 * to the first block.
221 */
222 if (blkid == 0)
223 return;
224
225 rw_enter(&zf->zf_rwlock, RW_READER);
226
227 for (zs = list_head(&zf->zf_stream); zs != NULL;
228 zs = list_next(&zf->zf_stream, zs)) {
229 if (blkid == zs->zs_blkid) {
230 mutex_enter(&zs->zs_lock);
231 /*
232 * zs_blkid could have changed before we
233 * acquired zs_lock; re-check them here.
234 */
235 if (blkid != zs->zs_blkid) {
236 mutex_exit(&zs->zs_lock);
237 continue;
238 }
239 break;
240 }
241 }
242
243 if (zs == NULL) {
244 /*
245 * This access is not part of any existing stream. Create
246 * a new stream for it.
247 */
248 ZFETCHSTAT_BUMP(zfetchstat_misses);
249 if (rw_tryupgrade(&zf->zf_rwlock))
250 dmu_zfetch_stream_create(zf, end_of_access_blkid);
251 rw_exit(&zf->zf_rwlock);
252 return;
253 }
254
255 /*
256 * This access was to a block that we issued a prefetch for on
257 * behalf of this stream. Issue further prefetches for this stream.
258 *
259 * Normally, we start prefetching where we stopped
260 * prefetching last (zs_pf_blkid). But when we get our first
261 * hit on this stream, zs_pf_blkid == zs_blkid, we don't
|