1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
14 */
15
16 /*
17 * Allocation and release DataBlocks.
18 * Two types of allocation of data-block:
19 * - from preallocated list (all datablock are preallocated at startup)
20 * - by using kmem_cache logic on the fly
21 */
22
23 #include "krrp_svc.h"
24 #include "krrp_dblk.h"
25
26
27 static int krrp_dblk_engine_configure(krrp_dblk_engine_t *dblk_engine,
28 boolean_t prealloc, krrp_error_t *error);
29 static int krrp_dblk_init_kmem_alloc_logic(krrp_dblk_engine_t *,
30 size_t, krrp_error_t *);
31 static void krrp_dblk_fini_kmem_alloc_logic(krrp_dblk_engine_t *);
32 static void krrp_dblk_free_dblk_list(krrp_dblk_list_t *);
33 static int krrp_dblk_constructor(void *, void *, int);
34 static void krrp_dblk_constructor_int(krrp_dblk_t *, krrp_dblk_engine_t *);
35 static void krrp_dblk_free_cb(caddr_t);
36
37 static void krrp_dblk_alloc_by_kmem_alloc(krrp_dblk_engine_t *,
38 krrp_dblk_t **, size_t);
39 static void krrp_dblk_alloc_by_kmem_cache_alloc(krrp_dblk_engine_t *,
40 krrp_dblk_t **, size_t);
41
42
43 int
44 krrp_dblk_engine_create(krrp_dblk_engine_t **result_engine,
45 boolean_t prealloc, size_t max_dblk_cnt, size_t dblk_head_sz,
46 size_t dblk_data_sz, size_t notify_free_value,
47 krrp_dblk_free_notify_cb_t *notify_free_cb, void *notify_free_cb_arg,
48 krrp_error_t *error)
49 {
50 krrp_dblk_engine_t *dblk_engine;
51
52 VERIFY(result_engine != NULL && *result_engine == NULL);
53 VERIFY(max_dblk_cnt != 0);
54 VERIFY(dblk_data_sz != 0);
55 VERIFY(notify_free_value != 0);
56 VERIFY(notify_free_cb != NULL);
57
58 dblk_engine = kmem_zalloc(sizeof (krrp_dblk_engine_t), KM_SLEEP);
59
60 dblk_engine->dblk_head_sz = dblk_head_sz;
61 dblk_engine->dblk_data_sz = dblk_data_sz;
62
63 dblk_engine->max_dblk_cnt = max_dblk_cnt;
64 dblk_engine->cur_dblk_cnt = 0;
65
66 dblk_engine->notify_free.cb = notify_free_cb;
67 dblk_engine->notify_free.cb_arg = notify_free_cb_arg;
68 dblk_engine->notify_free.init_value = notify_free_value;
69 dblk_engine->notify_free.cnt = notify_free_value;
70
71 if (krrp_dblk_engine_configure(dblk_engine, prealloc, error) != 0) {
72 kmem_free(dblk_engine, sizeof (krrp_dblk_engine_t));
73 return (-1);
74 }
75
76 mutex_init(&dblk_engine->mtx, NULL, MUTEX_DEFAULT, NULL);
77 cv_init(&dblk_engine->cv, NULL, CV_DEFAULT, NULL);
78
79 *result_engine = dblk_engine;
80 return (0);
81 }
82
83 static void
84 krrp_dblk_engine_destroy_impl(void *arg)
85 {
86 krrp_dblk_engine_t *engine = arg;
87
88 mutex_enter(&engine->mtx);
89
90 while (engine->cur_dblk_cnt != 0)
91 cv_wait(&engine->cv, &engine->mtx);
92
93 switch (engine->type) {
94 case KRRP_DET_KMEM_ALLOC:
95 krrp_dblk_fini_kmem_alloc_logic(engine);
96 break;
97 case KRRP_DET_KMEM_CACHE:
98 kmem_cache_destroy(engine->dblk_cache);
99 break;
100 }
101
102 cv_destroy(&engine->cv);
103
104 mutex_exit(&engine->mtx);
105 mutex_destroy(&engine->mtx);
106
107 kmem_free(engine, sizeof (krrp_dblk_engine_t));
108 }
109
110 void
111 krrp_dblk_engine_destroy(krrp_dblk_engine_t *engine)
112 {
113 mutex_enter(&engine->mtx);
114 engine->destroying = B_TRUE;
115 mutex_exit(&engine->mtx);
116
117 /*
118 * Destroy DBLK Engine asynchronously, because TCP/IP
119 * stack might hold our dblks for a long time
120 */
121 krrp_svc_dispatch_task(krrp_dblk_engine_destroy_impl, engine);
122 }
123
124 static int
125 krrp_dblk_engine_configure(krrp_dblk_engine_t *engine, boolean_t prealloc,
126 krrp_error_t *error)
127 {
128 size_t total_dblk_sz;
129 int rc = 0;
130
131 total_dblk_sz = sizeof (krrp_dblk_t) + engine->dblk_head_sz +
132 engine->dblk_data_sz;
133
134 if (prealloc) {
135 engine->type = KRRP_DET_KMEM_ALLOC;
136 engine->alloc_func = &krrp_dblk_alloc_by_kmem_alloc;
137 rc = krrp_dblk_init_kmem_alloc_logic(engine,
138 total_dblk_sz, error);
139 } else {
140 char kmem_cache_name[KSTAT_STRLEN];
141
142 (void) snprintf(kmem_cache_name, KSTAT_STRLEN,
143 "krrp_dec_%p", (void *)engine);
144
145 engine->type = KRRP_DET_KMEM_CACHE;
146 engine->alloc_func = &krrp_dblk_alloc_by_kmem_cache_alloc;
147 engine->dblk_cache = kmem_cache_create(kmem_cache_name,
148 total_dblk_sz, 8, &krrp_dblk_constructor, NULL, NULL,
149 engine, NULL, KM_SLEEP);
150 }
151
152 return (rc);
153 }
154
155 void
156 krrp_dblk_alloc(krrp_dblk_engine_t *dblk_engine, krrp_dblk_t **dblk,
157 size_t number)
158 {
159 dblk_engine->alloc_func(dblk_engine, dblk, number);
160 }
161
162 void
163 krrp_dblk_rele(krrp_dblk_t *dblk)
164 {
165 krrp_dblk_t *dblk_next;
166
167 while (dblk != NULL) {
168 dblk_next = dblk->next;
169 dblk->free_rtns.free_func(dblk->free_rtns.free_arg);
170 dblk = dblk_next;
171 }
172 }
173
174 int
175 krrp_dblk_get_data(krrp_dblk_t *dblk, void *buf, size_t buf_sz)
176 {
177 size_t buf_remain_sz = buf_sz;
178 size_t offset = 0;
179
180 while (dblk != NULL) {
181 size_t copy_sz = buf_remain_sz > dblk->cur_data_sz ?
182 dblk->cur_data_sz : buf_remain_sz;
183 bcopy(dblk->data, (caddr_t)buf + offset, copy_sz);
184 buf_remain_sz -= copy_sz;
185 offset += copy_sz;
186 if (buf_remain_sz == 0 && dblk->next != NULL)
187 return (EINVAL);
188
189 dblk = dblk->next;
190 }
191
192 return (0);
193 }
194
195 /*
196 * Here we use KM_NOSLEEP to be sure,
197 * that the system is not under mem-pressure
198 */
199 static int
200 krrp_dblk_init_kmem_alloc_logic(krrp_dblk_engine_t *dblk_engine,
201 size_t dblk_sz, krrp_error_t *error)
202 {
203 size_t i;
204 krrp_dblk_list_t *free_dblks;
205
206 free_dblks = &dblk_engine->free_dblks;
207
208 for (i = 0; i < dblk_engine->max_dblk_cnt; i++) {
209 krrp_dblk_t *dblk = kmem_alloc(dblk_sz, KM_NOSLEEP);
210 if (dblk == NULL) {
211 krrp_error_set(error, KRRP_ERRNO_NOMEM, 0);
212 krrp_dblk_fini_kmem_alloc_logic(dblk_engine);
213 return (-1);
214 }
215
216 krrp_dblk_constructor_int(dblk, dblk_engine);
217 if (free_dblks->head == NULL) {
218 free_dblks->head = dblk;
219 } else {
220 free_dblks->tail->next = dblk;
221 }
222
223 free_dblks->tail = dblk;
224 free_dblks->cnt++;
225 }
226
227 return (0);
228 }
229
230 static void
231 krrp_dblk_fini_kmem_alloc_logic(krrp_dblk_engine_t *dblk_engine)
232 {
233 krrp_dblk_free_dblk_list(&dblk_engine->free_dblks);
234 }
235
236 static void
237 krrp_dblk_free_dblk_list(krrp_dblk_list_t *dblk_list)
238 {
239 size_t cnt = 0;
240 krrp_dblk_t *dblk, *dblk_next;
241
242 dblk = dblk_list->head;
243 while (dblk != NULL) {
244 dblk_next = dblk->next;
245 kmem_free(dblk, dblk->total_sz + sizeof (krrp_dblk_t));
246 dblk = dblk_next;
247 cnt++;
248 }
249
250 VERIFY(dblk_list->cnt == cnt);
251 dblk_list->cnt = 0;
252 }
253
254 /*
255 * DBLK Allocator: uses preallocated list of dblks
256 *
257 * allocates requested number of dblks or nothing
258 */
259 static void
260 krrp_dblk_alloc_by_kmem_alloc(krrp_dblk_engine_t *dblk_engine,
261 krrp_dblk_t **result_dblk, size_t number)
262 {
263 krrp_dblk_t *dblk_head = NULL, *dblk_prev = NULL, *dblk;
264 size_t cnt = 0;
265
266 krrp_dblk_list_t *free_dblks = &dblk_engine->free_dblks;
267
268 mutex_enter(&dblk_engine->mtx);
269 if (free_dblks->cnt < number) {
270 mutex_exit(&dblk_engine->mtx);
271 return;
272 }
273
274 dblk_head = free_dblks->head;
275 dblk = dblk_head;
276
277 while (cnt < number) {
278 dblk_prev = dblk;
279 dblk = dblk->next;
280 cnt++;
281 }
282
283 dblk_prev->next = NULL;
284 free_dblks->cnt -= number;
285 free_dblks->head = dblk;
286 if (free_dblks->head == NULL)
287 free_dblks->tail = NULL;
288
289 dblk_engine->cur_dblk_cnt += number;
290 mutex_exit(&dblk_engine->mtx);
291
292 *result_dblk = dblk_head;
293 }
294
295 /*
296 * DBLK Allocator: uses kmem_cache
297 *
298 * allocates requested number of dblks or nothing
299 */
300 static void
301 krrp_dblk_alloc_by_kmem_cache_alloc(krrp_dblk_engine_t *engine,
302 krrp_dblk_t **result_dblk, size_t number)
303 {
304 krrp_dblk_t *dblk_head = NULL, *dblk_next, *dblk = NULL;
305 size_t cnt = 0;
306 size_t available_cnt;
307
308 mutex_enter(&engine->mtx);
309 available_cnt = engine->max_dblk_cnt - engine->cur_dblk_cnt;
310 if (available_cnt < number) {
311 mutex_exit(&engine->mtx);
312 return;
313 }
314
315 while (cnt < number) {
316 engine->cur_dblk_cnt++;
317 dblk_next = kmem_cache_alloc(engine->dblk_cache, KM_SLEEP);
318
319 mutex_exit(&engine->mtx);
320
321 if (dblk_head == NULL)
322 dblk_head = dblk_next;
323 else
324 dblk->next = dblk_next;
325
326 dblk = dblk_next;
327 cnt++;
328
329 mutex_enter(&engine->mtx);
330 }
331
332 mutex_exit(&engine->mtx);
333
334 *result_dblk = dblk_head;
335 }
336
337 /* ARGSUSED */
338 static int
339 krrp_dblk_constructor(void *void_dblk, void *void_dblk_engine, int km_flags)
340 {
341 krrp_dblk_constructor_int(void_dblk, void_dblk_engine);
342
343 return (0);
344 }
345
346 static void
347 krrp_dblk_constructor_int(krrp_dblk_t *dblk, krrp_dblk_engine_t *dblk_engine)
348 {
349 dblk->engine = dblk_engine;
350
351 dblk->free_rtns.free_func = &krrp_dblk_free_cb;
352 dblk->free_rtns.free_arg = (caddr_t)dblk;
353
354 dblk->max_data_sz = dblk_engine->dblk_data_sz;
355 dblk->cur_data_sz = 0;
356 dblk->head = (((caddr_t)dblk) + sizeof (krrp_dblk_t));
357 dblk->data = ((caddr_t)dblk->head + dblk_engine->dblk_head_sz);
358
359 dblk->total_sz = dblk_engine->dblk_head_sz + dblk_engine->dblk_data_sz;
360 dblk->next = NULL;
361 }
362
363 static void
364 krrp_dblk_free_cb(caddr_t void_dblk)
365 {
366 krrp_dblk_t *dblk = (krrp_dblk_t *)void_dblk;
367 krrp_dblk_engine_t *dblk_engine = dblk->engine;
368 krrp_dblk_list_t *free_dblks;
369
370 mutex_enter(&dblk_engine->mtx);
371
372 dblk->cur_data_sz = 0;
373 dblk->next = NULL;
374
375 switch (dblk_engine->type) {
376 case KRRP_DET_KMEM_ALLOC:
377 free_dblks = &dblk_engine->free_dblks;
378
379 if (free_dblks->head == NULL)
380 free_dblks->head = dblk;
381 else
382 free_dblks->tail->next = dblk;
383
384 free_dblks->tail = dblk;
385 free_dblks->cnt++;
386 break;
387 case KRRP_DET_KMEM_CACHE:
388 kmem_cache_free(dblk_engine->dblk_cache, dblk);
389 break;
390 }
391
392 dblk_engine->cur_dblk_cnt--;
393 dblk_engine->notify_free.cnt--;
394
395 if (dblk_engine->notify_free.cnt == 0) {
396 dblk_engine->notify_free.cnt =
397 dblk_engine->notify_free.init_value;
398
399 if (!dblk_engine->destroying) {
400 dblk_engine->notify_free.cb(
401 dblk_engine->notify_free.cb_arg);
402 }
403 }
404
405 cv_signal(&dblk_engine->cv);
406 mutex_exit(&dblk_engine->mtx);
407 }