Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0-only */
2 : /*
3 : * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
4 : */
5 :
6 : #include "bpfilter/cgen/program.h"
7 :
8 : #include <linux/bpf.h>
9 : #include <linux/bpf_common.h>
10 : #include <linux/limits.h>
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <stddef.h>
16 : #include <stdint.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <string.h>
20 : #include <unistd.h>
21 :
22 : #include "bpfilter/cgen/cgroup.h"
23 : #include "bpfilter/cgen/dump.h"
24 : #include "bpfilter/cgen/fixup.h"
25 : #include "bpfilter/cgen/jmp.h"
26 : #include "bpfilter/cgen/matcher/icmp.h"
27 : #include "bpfilter/cgen/matcher/ip4.h"
28 : #include "bpfilter/cgen/matcher/ip6.h"
29 : #include "bpfilter/cgen/matcher/meta.h"
30 : #include "bpfilter/cgen/matcher/set.h"
31 : #include "bpfilter/cgen/matcher/tcp.h"
32 : #include "bpfilter/cgen/matcher/udp.h"
33 : #include "bpfilter/cgen/nf.h"
34 : #include "bpfilter/cgen/printer.h"
35 : #include "bpfilter/cgen/prog/link.h"
36 : #include "bpfilter/cgen/prog/map.h"
37 : #include "bpfilter/cgen/stub.h"
38 : #include "bpfilter/cgen/tc.h"
39 : #include "bpfilter/cgen/xdp.h"
40 : #include "bpfilter/ctx.h"
41 : #include "bpfilter/opts.h"
42 : #include "core/bpf.h"
43 : #include "core/btf.h"
44 : #include "core/chain.h"
45 : #include "core/counter.h"
46 : #include "core/dump.h"
47 : #include "core/flavor.h"
48 : #include "core/helper.h"
49 : #include "core/hook.h"
50 : #include "core/io.h"
51 : #include "core/list.h"
52 : #include "core/logger.h"
53 : #include "core/marsh.h"
54 : #include "core/matcher.h"
55 : #include "core/rule.h"
56 : #include "core/set.h"
57 : #include "core/verdict.h"
58 :
59 : #include "external/filter.h"
60 :
61 : #define _BF_LOG_BUF_SIZE \
62 : (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */
63 : #define _BF_PROGRAM_DEFAULT_IMG_SIZE (1 << 6)
64 : #define _BF_LOG_MAP_N_ENTRIES 1000
65 : #define _BF_LOG_MAP_SIZE \
66 : _bf_round_next_power_of_2(sizeof(struct bf_log) * _BF_LOG_MAP_N_ENTRIES)
67 :
68 0 : static inline size_t _bf_round_next_power_of_2(size_t value)
69 : {
70 0 : value--;
71 0 : value |= value >> 1;
72 0 : value |= value >> 2;
73 0 : value |= value >> 4;
74 0 : value |= value >> 8;
75 0 : value |= value >> 16;
76 :
77 0 : return ++value;
78 : }
79 :
80 7 : static const struct bf_flavor_ops *bf_flavor_ops_get(enum bf_flavor flavor)
81 : {
82 : static const struct bf_flavor_ops *flavor_ops[] = {
83 : [BF_FLAVOR_TC] = &bf_flavor_ops_tc,
84 : [BF_FLAVOR_NF] = &bf_flavor_ops_nf,
85 : [BF_FLAVOR_XDP] = &bf_flavor_ops_xdp,
86 : [BF_FLAVOR_CGROUP] = &bf_flavor_ops_cgroup,
87 : };
88 :
89 : static_assert(ARRAY_SIZE(flavor_ops) == _BF_FLAVOR_MAX,
90 : "missing entries in bf_flavor_ops array");
91 :
92 7 : return flavor_ops[flavor];
93 : }
94 :
95 3 : int bf_program_new(struct bf_program **program, const struct bf_chain *chain)
96 : {
97 3 : _free_bf_program_ struct bf_program *_program = NULL;
98 : char name[BPF_OBJ_NAME_LEN];
99 : uint32_t set_idx = 0;
100 : int r;
101 :
102 3 : bf_assert(program && chain);
103 :
104 3 : _program = calloc(1, sizeof(*_program));
105 3 : if (!_program)
106 : return -ENOMEM;
107 :
108 3 : _program->flavor = bf_hook_to_flavor(chain->hook);
109 3 : _program->runtime.prog_fd = -1;
110 3 : _program->runtime.ops = bf_flavor_ops_get(_program->flavor);
111 3 : _program->runtime.chain = chain;
112 :
113 3 : (void)snprintf(_program->prog_name, BPF_OBJ_NAME_LEN, "%s", "bf_prog");
114 :
115 3 : r = bf_map_new(&_program->cmap, "counters_map", BF_MAP_TYPE_COUNTERS,
116 : sizeof(uint32_t), sizeof(struct bf_counter), 1);
117 3 : if (r < 0)
118 0 : return bf_err_r(r, "failed to create the counters bf_map object");
119 :
120 3 : r = bf_map_new(&_program->pmap, "printer_map", BF_MAP_TYPE_PRINTER,
121 : sizeof(uint32_t), BF_MAP_VALUE_SIZE_UNKNOWN, 1);
122 3 : if (r < 0)
123 0 : return bf_err_r(r, "failed to create the printer bf_map object");
124 :
125 3 : r = bf_map_new(&_program->lmap, "log_map", BF_MAP_TYPE_LOG, 0, 0,
126 : _BF_LOG_MAP_SIZE);
127 3 : if (r < 0)
128 0 : return bf_err_r(r, "failed to create the log bf_map object");
129 :
130 3 : _program->sets = bf_map_list();
131 6 : bf_list_foreach (&chain->sets, set_node) {
132 0 : struct bf_set *set = bf_list_node_get_data(set_node);
133 0 : _free_bf_map_ struct bf_map *map = NULL;
134 :
135 0 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "set_%04x", (uint8_t)set_idx++);
136 0 : r = bf_map_new_from_set(&map, name, set);
137 0 : if (r < 0)
138 : return r;
139 :
140 0 : r = bf_list_add_tail(&_program->sets, map);
141 0 : if (r < 0)
142 : return r;
143 0 : TAKE_PTR(map);
144 : };
145 :
146 3 : r = bf_link_new(&_program->link, "bf_link");
147 3 : if (r)
148 : return r;
149 :
150 3 : r = bf_printer_new(&_program->printer);
151 3 : if (r)
152 : return r;
153 :
154 3 : bf_list_init(&_program->fixups,
155 3 : (bf_list_ops[]) {{.free = (bf_list_ops_free)bf_fixup_free}});
156 :
157 3 : *program = TAKE_PTR(_program);
158 :
159 3 : return 0;
160 : }
161 :
162 15 : void bf_program_free(struct bf_program **program)
163 : {
164 15 : if (!*program)
165 : return;
166 :
167 3 : bf_list_clean(&(*program)->fixups);
168 3 : free((*program)->img);
169 :
170 : /* Close the file descriptors if they are still open. If --transient is
171 : * used, then the file descriptors are already closed (as
172 : * bf_program_unload() has been called). Otherwise, bf_program_unload()
173 : * won't be called, but the programs are pinned, so they can be closed
174 : * safely. */
175 3 : closep(&(*program)->runtime.prog_fd);
176 :
177 3 : bf_map_free(&(*program)->cmap);
178 3 : bf_map_free(&(*program)->pmap);
179 3 : bf_map_free(&(*program)->lmap);
180 3 : bf_list_clean(&(*program)->sets);
181 3 : bf_link_free(&(*program)->link);
182 3 : bf_printer_free(&(*program)->printer);
183 :
184 3 : free(*program);
185 3 : *program = NULL;
186 : }
187 :
188 0 : int bf_program_marsh(const struct bf_program *program, struct bf_marsh **marsh)
189 : {
190 0 : _free_bf_marsh_ struct bf_marsh *_marsh = NULL;
191 : int r;
192 :
193 0 : bf_assert(program);
194 0 : bf_assert(marsh);
195 :
196 0 : r = bf_marsh_new(&_marsh, NULL, 0);
197 0 : if (r < 0)
198 : return r;
199 :
200 : {
201 : // Serialize bf_program.counters
202 0 : _free_bf_marsh_ struct bf_marsh *counters_elem = NULL;
203 :
204 0 : r = bf_map_marsh(program->cmap, &counters_elem);
205 0 : if (r < 0)
206 : return r;
207 :
208 0 : r = bf_marsh_add_child_obj(&_marsh, counters_elem);
209 0 : if (r < 0)
210 : return r;
211 : }
212 :
213 : {
214 : // Serialize bf_program.pmap
215 0 : _free_bf_marsh_ struct bf_marsh *pmap_elem = NULL;
216 :
217 0 : r = bf_map_marsh(program->pmap, &pmap_elem);
218 0 : if (r < 0)
219 : return r;
220 :
221 0 : r = bf_marsh_add_child_obj(&_marsh, pmap_elem);
222 0 : if (r < 0)
223 : return r;
224 : }
225 :
226 : {
227 : // Serialize bf_program.lmap
228 0 : _free_bf_marsh_ struct bf_marsh *lmap_elem = NULL;
229 :
230 0 : r = bf_map_marsh(program->lmap, &lmap_elem);
231 0 : if (r < 0)
232 : return r;
233 :
234 0 : r = bf_marsh_add_child_obj(&_marsh, lmap_elem);
235 0 : if (r < 0)
236 : return r;
237 : }
238 :
239 : {
240 : // Serialize bf_program.sets
241 0 : _free_bf_marsh_ struct bf_marsh *sets_elem = NULL;
242 :
243 0 : r = bf_list_marsh(&program->sets, &sets_elem);
244 0 : if (r < 0)
245 : return r;
246 :
247 0 : r = bf_marsh_add_child_obj(&_marsh, sets_elem);
248 0 : if (r < 0) {
249 0 : return bf_err_r(
250 : r,
251 : "failed to insert serialized sets into bf_program serialized data");
252 : }
253 : }
254 :
255 : {
256 : // Serialize bf_program.links
257 0 : _free_bf_marsh_ struct bf_marsh *links_elem = NULL;
258 :
259 0 : r = bf_link_marsh(program->link, &links_elem);
260 0 : if (r)
261 0 : return bf_err_r(r, "failed to serialize bf_program.link");
262 :
263 0 : r = bf_marsh_add_child_obj(&_marsh, links_elem);
264 0 : if (r) {
265 0 : return bf_err_r(
266 : r,
267 : "failed to insert serialized link into bf_program serialized data");
268 : }
269 : }
270 :
271 : {
272 : // Serialise bf_program.printer
273 0 : _free_bf_marsh_ struct bf_marsh *child = NULL;
274 :
275 0 : r = bf_printer_marsh(program->printer, &child);
276 0 : if (r)
277 0 : return bf_err_r(r, "failed to marsh bf_printer object");
278 :
279 0 : r = bf_marsh_add_child_obj(&_marsh, child);
280 0 : if (r)
281 0 : return bf_err_r(r, "failed to append object to marsh");
282 : }
283 :
284 0 : r |= bf_marsh_add_child_raw(&_marsh, program->img,
285 0 : program->img_size * sizeof(struct bpf_insn));
286 0 : if (r)
287 0 : return bf_err_r(r, "Failed to serialize program");
288 :
289 0 : *marsh = TAKE_PTR(_marsh);
290 :
291 0 : return 0;
292 : }
293 :
294 0 : int bf_program_unmarsh(const struct bf_marsh *marsh,
295 : struct bf_program **program,
296 : const struct bf_chain *chain, int dir_fd)
297 : {
298 0 : _free_bf_program_ struct bf_program *_program = NULL;
299 0 : _free_bf_link_ struct bf_link *link = NULL;
300 : struct bf_marsh *child = NULL;
301 : int r;
302 :
303 0 : bf_assert(marsh && program);
304 :
305 0 : r = bf_program_new(&_program, chain);
306 0 : if (r < 0)
307 : return r;
308 :
309 0 : if (!(child = bf_marsh_next_child(marsh, child)))
310 : return -EINVAL;
311 0 : bf_map_free(&_program->cmap);
312 0 : r = bf_map_new_from_marsh(&_program->cmap, dir_fd, child);
313 0 : if (r < 0)
314 : return r;
315 :
316 0 : if (!(child = bf_marsh_next_child(marsh, child)))
317 : return -EINVAL;
318 0 : bf_map_free(&_program->pmap);
319 0 : r = bf_map_new_from_marsh(&_program->pmap, dir_fd, child);
320 0 : if (r < 0)
321 : return r;
322 :
323 0 : if (!(child = bf_marsh_next_child(marsh, child)))
324 : return -EINVAL;
325 0 : bf_map_free(&_program->lmap);
326 0 : r = bf_map_new_from_marsh(&_program->lmap, dir_fd, child);
327 0 : if (r < 0)
328 : return r;
329 :
330 : /** @todo Avoid creating and filling the list in @ref bf_program_new before
331 : * trashing it all here. Eventually, this function will be replaced with
332 : * @c bf_program_new_from_marsh and this issue could be solved by **not**
333 : * relying on @ref bf_program_new to allocate an initialize @p _program . */
334 0 : bf_list_clean(&_program->sets);
335 0 : _program->sets = bf_map_list();
336 :
337 0 : if (!(child = bf_marsh_next_child(marsh, child)))
338 : return -EINVAL;
339 : {
340 : // Unmarsh bf_program.sets
341 : struct bf_marsh *set_elem = NULL;
342 :
343 0 : while ((set_elem = bf_marsh_next_child(child, set_elem))) {
344 0 : _free_bf_map_ struct bf_map *map = NULL;
345 :
346 0 : r = bf_map_new_from_marsh(&map, dir_fd, set_elem);
347 0 : if (r < 0)
348 : return r;
349 :
350 0 : r = bf_list_add_tail(&_program->sets, map);
351 0 : if (r < 0)
352 : return r;
353 :
354 0 : TAKE_PTR(map);
355 : }
356 : }
357 :
358 : // Unmarsh bf_program.links
359 0 : if (!(child = bf_marsh_next_child(marsh, child)))
360 : return -EINVAL;
361 :
362 : /* Try to restore the link: on success, replace the program's link with the
363 : * restored on. If -ENOENT is returned, the link doesn't exist, meaning the
364 : * program is not attached. Otherwise, return an error. */
365 0 : r = bf_link_new_from_marsh(&link, dir_fd, child);
366 0 : if (!r)
367 0 : bf_swap(_program->link, link);
368 0 : else if (r != -ENOENT)
369 0 : return bf_err_r(r, "failed to restore bf_program.link");
370 :
371 : // Unmarsh bf_program.printer
372 0 : child = bf_marsh_next_child(marsh, child);
373 0 : if (!child)
374 0 : return bf_err_r(-EINVAL, "failed to find valid child");
375 :
376 0 : bf_printer_free(&_program->printer);
377 0 : r = bf_printer_new_from_marsh(&_program->printer, child);
378 0 : if (r)
379 0 : return bf_err_r(r, "failed to restore bf_printer object");
380 :
381 0 : if (!(child = bf_marsh_next_child(marsh, child)))
382 : return -EINVAL;
383 0 : _program->img = bf_memdup(child->data, child->data_len);
384 0 : _program->img_size = child->data_len / sizeof(struct bpf_insn);
385 0 : _program->img_cap = child->data_len / sizeof(struct bpf_insn);
386 :
387 0 : if (bf_marsh_next_child(marsh, child))
388 0 : bf_warn("codegen marsh has more children than expected");
389 :
390 0 : r = bf_bpf_obj_get(_program->prog_name, dir_fd, &_program->runtime.prog_fd);
391 0 : if (r < 0)
392 0 : return bf_err_r(r, "failed to get prog fd");
393 :
394 0 : *program = TAKE_PTR(_program);
395 :
396 0 : return 0;
397 : }
398 :
399 0 : void bf_program_dump(const struct bf_program *program, prefix_t *prefix)
400 : {
401 0 : bf_assert(program);
402 0 : bf_assert(prefix);
403 :
404 0 : DUMP(prefix, "struct bf_program at %p", program);
405 :
406 0 : bf_dump_prefix_push(prefix);
407 :
408 0 : DUMP(prefix, "prog_name: %s", program->prog_name);
409 :
410 0 : DUMP(prefix, "cmap: struct bf_map *");
411 0 : bf_dump_prefix_push(prefix);
412 0 : bf_map_dump(program->cmap, bf_dump_prefix_last(prefix));
413 0 : bf_dump_prefix_pop(prefix);
414 :
415 0 : DUMP(prefix, "pmap: struct bf_map *");
416 0 : bf_dump_prefix_push(prefix);
417 0 : bf_map_dump(program->pmap, bf_dump_prefix_last(prefix));
418 0 : bf_dump_prefix_pop(prefix);
419 :
420 0 : DUMP(prefix, "lmap: struct bf_map *");
421 0 : bf_dump_prefix_push(prefix);
422 0 : bf_map_dump(program->lmap, bf_dump_prefix_last(prefix));
423 0 : bf_dump_prefix_pop(prefix);
424 :
425 0 : DUMP(prefix, "sets: bf_list<bf_map>[%lu]", bf_list_size(&program->sets));
426 0 : bf_dump_prefix_push(prefix);
427 0 : bf_list_foreach (&program->sets, map_node) {
428 0 : struct bf_map *map = bf_list_node_get_data(map_node);
429 :
430 0 : if (bf_list_is_tail(&program->sets, map_node))
431 0 : bf_dump_prefix_last(prefix);
432 :
433 0 : bf_map_dump(map, prefix);
434 : }
435 0 : bf_dump_prefix_pop(prefix);
436 :
437 0 : DUMP(prefix, "link: struct bf_link *");
438 0 : bf_dump_prefix_push(prefix);
439 0 : bf_link_dump(program->link, prefix);
440 0 : bf_dump_prefix_pop(prefix);
441 :
442 0 : DUMP(prefix, "printer: struct bf_printer *");
443 0 : bf_dump_prefix_push(prefix);
444 0 : bf_printer_dump(program->printer, prefix);
445 0 : bf_dump_prefix_pop(prefix);
446 :
447 0 : DUMP(prefix, "img: %p", program->img);
448 0 : DUMP(prefix, "img_size: %lu", program->img_size);
449 0 : DUMP(prefix, "img_cap: %lu", program->img_cap);
450 :
451 0 : DUMP(prefix, "fixups: bf_list<struct bf_fixup>[%lu]",
452 : bf_list_size(&program->fixups));
453 0 : bf_dump_prefix_push(prefix);
454 0 : bf_list_foreach (&program->fixups, fixup_node) {
455 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
456 :
457 0 : if (bf_list_is_tail(&program->fixups, fixup_node))
458 0 : bf_dump_prefix_last(prefix);
459 :
460 0 : bf_fixup_dump(fixup, prefix);
461 : }
462 0 : bf_dump_prefix_pop(prefix);
463 :
464 0 : DUMP(bf_dump_prefix_last(prefix), "runtime: <anonymous>");
465 0 : bf_dump_prefix_push(prefix);
466 0 : DUMP(prefix, "prog_fd: %d", program->runtime.prog_fd);
467 0 : DUMP(bf_dump_prefix_last(prefix), "ops: %p", program->runtime.ops);
468 0 : bf_dump_prefix_pop(prefix);
469 :
470 0 : bf_dump_prefix_pop(prefix);
471 0 : }
472 :
473 3 : int bf_program_grow_img(struct bf_program *program)
474 : {
475 : size_t new_cap = _BF_PROGRAM_DEFAULT_IMG_SIZE;
476 : int r;
477 :
478 3 : bf_assert(program);
479 :
480 3 : if (program->img)
481 0 : new_cap = _bf_round_next_power_of_2(program->img_cap << 1);
482 :
483 3 : r = bf_realloc((void **)&program->img, new_cap * sizeof(struct bpf_insn));
484 3 : if (r < 0) {
485 0 : return bf_err_r(r, "failed to grow program img from %lu to %lu insn",
486 : program->img_cap, new_cap);
487 : }
488 :
489 3 : program->img_cap = new_cap;
490 :
491 3 : return 0;
492 : }
493 :
494 0 : static void _bf_program_fixup_insn(struct bpf_insn *insn,
495 : enum bf_fixup_insn type, int32_t value)
496 : {
497 0 : switch (type) {
498 0 : case BF_FIXUP_INSN_OFF:
499 0 : bf_assert(!insn->off);
500 0 : bf_assert(value < SHRT_MAX);
501 0 : insn->off = (int16_t)value;
502 0 : break;
503 0 : case BF_FIXUP_INSN_IMM:
504 0 : bf_assert(!insn->imm);
505 0 : insn->imm = value;
506 0 : break;
507 0 : default:
508 0 : bf_abort(
509 : "unsupported fixup instruction type, this should not happen: %d",
510 : type);
511 : break;
512 : }
513 0 : }
514 :
515 0 : static int _bf_program_fixup(struct bf_program *program,
516 : enum bf_fixup_type type)
517 : {
518 0 : bf_assert(program);
519 0 : bf_assert(type >= 0 && type < _BF_FIXUP_TYPE_MAX);
520 :
521 0 : bf_list_foreach (&program->fixups, fixup_node) {
522 : enum bf_fixup_insn insn_type = _BF_FIXUP_INSN_MAX;
523 : int32_t value;
524 : size_t offset;
525 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
526 0 : struct bpf_insn *insn = &program->img[fixup->insn];
527 : struct bf_map *map;
528 :
529 0 : if (type != fixup->type)
530 0 : continue;
531 :
532 0 : switch (type) {
533 0 : case BF_FIXUP_TYPE_JMP_NEXT_RULE:
534 : insn_type = BF_FIXUP_INSN_OFF;
535 0 : value = (int)(program->img_size - fixup->insn - 1U);
536 0 : break;
537 0 : case BF_FIXUP_TYPE_COUNTERS_MAP_FD:
538 : insn_type = BF_FIXUP_INSN_IMM;
539 0 : value = program->cmap->fd;
540 0 : break;
541 0 : case BF_FIXUP_TYPE_PRINTER_MAP_FD:
542 : insn_type = BF_FIXUP_INSN_IMM;
543 0 : value = program->pmap->fd;
544 0 : break;
545 0 : case BF_FIXUP_TYPE_LOG_MAP_FD:
546 : insn_type = BF_FIXUP_INSN_IMM;
547 0 : value = program->lmap->fd;
548 0 : break;
549 0 : case BF_FIXUP_TYPE_SET_MAP_FD:
550 0 : map = bf_list_get_at(&program->sets, fixup->attr.set_index);
551 0 : if (!map) {
552 0 : return bf_err_r(-ENOENT, "can't find set map at index %lu",
553 : fixup->attr.set_index);
554 : }
555 : insn_type = BF_FIXUP_INSN_IMM;
556 0 : value = map->fd;
557 0 : break;
558 0 : case BF_FIXUP_ELFSTUB_CALL:
559 : insn_type = BF_FIXUP_INSN_IMM;
560 0 : offset = program->elfstubs_location[fixup->attr.elfstub_id] -
561 : fixup->insn - 1;
562 0 : if (offset >= INT_MAX)
563 0 : return bf_err_r(-EINVAL, "invalid ELF stub call offset");
564 0 : value = (int32_t)offset;
565 0 : break;
566 0 : default:
567 0 : bf_abort("unsupported fixup type, this should not happen: %d",
568 : type);
569 : break;
570 : }
571 :
572 0 : _bf_program_fixup_insn(insn, insn_type, value);
573 0 : bf_list_delete(&program->fixups, fixup_node);
574 : }
575 :
576 : return 0;
577 : }
578 :
579 0 : static int _bf_program_generate_rule(struct bf_program *program,
580 : struct bf_rule *rule)
581 : {
582 : int r;
583 :
584 0 : bf_assert(program);
585 0 : bf_assert(rule);
586 :
587 0 : bf_list_foreach (&rule->matchers, matcher_node) {
588 0 : struct bf_matcher *matcher = bf_list_node_get_data(matcher_node);
589 :
590 0 : switch (matcher->type) {
591 0 : case BF_MATCHER_META_IFACE:
592 : case BF_MATCHER_META_L3_PROTO:
593 : case BF_MATCHER_META_L4_PROTO:
594 : case BF_MATCHER_META_PROBABILITY:
595 : case BF_MATCHER_META_SPORT:
596 : case BF_MATCHER_META_DPORT:
597 0 : r = bf_matcher_generate_meta(program, matcher);
598 0 : if (r)
599 : return r;
600 : break;
601 0 : case BF_MATCHER_IP4_SADDR:
602 : case BF_MATCHER_IP4_SNET:
603 : case BF_MATCHER_IP4_DADDR:
604 : case BF_MATCHER_IP4_DNET:
605 : case BF_MATCHER_IP4_PROTO:
606 0 : r = bf_matcher_generate_ip4(program, matcher);
607 0 : if (r)
608 : return r;
609 : break;
610 0 : case BF_MATCHER_IP6_SADDR:
611 : case BF_MATCHER_IP6_SNET:
612 : case BF_MATCHER_IP6_DADDR:
613 : case BF_MATCHER_IP6_DNET:
614 : case BF_MATCHER_IP6_NEXTHDR:
615 0 : r = bf_matcher_generate_ip6(program, matcher);
616 0 : if (r)
617 : return r;
618 : break;
619 0 : case BF_MATCHER_TCP_SPORT:
620 : case BF_MATCHER_TCP_DPORT:
621 : case BF_MATCHER_TCP_FLAGS:
622 0 : r = bf_matcher_generate_tcp(program, matcher);
623 0 : if (r)
624 : return r;
625 : break;
626 0 : case BF_MATCHER_UDP_SPORT:
627 : case BF_MATCHER_UDP_DPORT:
628 0 : r = bf_matcher_generate_udp(program, matcher);
629 0 : if (r)
630 : return r;
631 : break;
632 0 : case BF_MATCHER_ICMP_TYPE:
633 : case BF_MATCHER_ICMP_CODE:
634 : case BF_MATCHER_ICMPV6_TYPE:
635 : case BF_MATCHER_ICMPV6_CODE:
636 0 : r = bf_matcher_generate_icmp(program, matcher);
637 0 : if (r)
638 : return r;
639 : break;
640 0 : case BF_MATCHER_SET:
641 0 : r = bf_matcher_generate_set(program, matcher);
642 0 : if (r)
643 : return r;
644 : break;
645 0 : default:
646 0 : return bf_err_r(-EINVAL, "unknown matcher type %d", matcher->type);
647 : };
648 : }
649 :
650 0 : if (rule->log) {
651 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
652 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
653 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, rule->index));
654 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_3, rule->log));
655 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_4, BPF_REG_7));
656 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_5, BPF_REG_8));
657 :
658 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_LOG);
659 : }
660 :
661 0 : if (rule->counters) {
662 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
663 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
664 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
665 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_3, rule->index));
666 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
667 : }
668 :
669 0 : switch (rule->verdict) {
670 0 : case BF_VERDICT_ACCEPT:
671 : case BF_VERDICT_DROP:
672 0 : EMIT(program,
673 : BPF_MOV64_IMM(BPF_REG_0,
674 : program->runtime.ops->get_verdict(rule->verdict)));
675 0 : EMIT(program, BPF_EXIT_INSN());
676 0 : break;
677 : case BF_VERDICT_CONTINUE:
678 : // Fall through to next rule or default chain policy.
679 : break;
680 0 : default:
681 0 : bf_abort("unsupported verdict, this should not happen: %d",
682 : rule->verdict);
683 : break;
684 : }
685 :
686 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_JMP_NEXT_RULE);
687 0 : if (r)
688 0 : return bf_err_r(r, "failed to generate next rule fixups");
689 :
690 : return 0;
691 : }
692 :
693 0 : static int _bf_program_generate_elfstubs(struct bf_program *program)
694 : {
695 : const struct bf_elfstub *elfstub;
696 : size_t start_at;
697 : int r;
698 :
699 0 : bf_assert(program);
700 :
701 0 : bf_list_foreach (&program->fixups, fixup_node) {
702 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
703 0 : size_t off = program->img_size;
704 :
705 0 : if (fixup->type != BF_FIXUP_ELFSTUB_CALL)
706 0 : continue;
707 :
708 : // Only generate each ELF stub once
709 0 : if (program->elfstubs_location[fixup->attr.elfstub_id])
710 0 : continue;
711 :
712 0 : bf_dbg("generate ELF stub for ID %d", fixup->attr.elfstub_id);
713 :
714 0 : elfstub = bf_ctx_get_elfstub(fixup->attr.elfstub_id);
715 0 : if (!elfstub) {
716 0 : return bf_err_r(-ENOENT, "no ELF stub found for ID %d",
717 : fixup->attr.elfstub_id);
718 : }
719 :
720 0 : start_at = program->img_size;
721 :
722 0 : for (size_t i = 0; i < elfstub->ninsns; ++i) {
723 0 : r = bf_program_emit(program, elfstub->insns[i]);
724 0 : if (r)
725 0 : return bf_err_r(r, "failed to insert ELF stub instruction");
726 : }
727 :
728 0 : bf_list_foreach (&elfstub->strs, pstr_node) {
729 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
730 0 : struct bf_printk_str *pstr = bf_list_node_get_data(pstr_node);
731 0 : size_t insn_idx = start_at + pstr->insn_idx;
732 : const struct bf_printer_msg *msg =
733 0 : bf_printer_add_msg(program->printer, pstr->str);
734 : struct bpf_insn ld_insn[2] = {
735 : BPF_LD_MAP_FD(BPF_REG_1, 0),
736 : };
737 :
738 : ld_insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
739 0 : ld_insn[1].imm = (int)bf_printer_msg_offset(msg);
740 :
741 0 : program->img[insn_idx] = ld_insn[0];
742 0 : program->img[insn_idx + 1] = ld_insn[1];
743 :
744 0 : r = bf_fixup_new(&fixup, BF_FIXUP_TYPE_PRINTER_MAP_FD, insn_idx,
745 : NULL);
746 0 : if (r)
747 : return r;
748 :
749 0 : r = bf_list_add_tail(&program->fixups, fixup);
750 0 : if (r)
751 : return r;
752 :
753 0 : TAKE_PTR(fixup);
754 : }
755 :
756 0 : program->elfstubs_location[fixup->attr.elfstub_id] = off;
757 : }
758 :
759 : return 0;
760 : }
761 :
762 40 : int bf_program_emit(struct bf_program *program, struct bpf_insn insn)
763 : {
764 : int r;
765 :
766 40 : bf_assert(program);
767 :
768 40 : if (program->img_size == program->img_cap) {
769 3 : r = bf_program_grow_img(program);
770 3 : if (r)
771 : return r;
772 : }
773 :
774 40 : program->img[program->img_size++] = insn;
775 :
776 40 : return 0;
777 : }
778 :
779 0 : int bf_program_emit_kfunc_call(struct bf_program *program, const char *name)
780 : {
781 : int r;
782 :
783 0 : bf_assert(program);
784 0 : bf_assert(name);
785 :
786 0 : r = bf_btf_get_id(name);
787 0 : if (r < 0)
788 : return r;
789 :
790 0 : EMIT(program, ((struct bpf_insn) {.code = BPF_JMP | BPF_CALL,
791 : .dst_reg = 0,
792 : .src_reg = BPF_PSEUDO_KFUNC_CALL,
793 : .off = 0,
794 : .imm = r}));
795 :
796 0 : return 0;
797 : }
798 :
799 0 : int bf_program_emit_fixup(struct bf_program *program, enum bf_fixup_type type,
800 : struct bpf_insn insn, const union bf_fixup_attr *attr)
801 : {
802 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
803 : int r;
804 :
805 0 : bf_assert(program);
806 :
807 0 : if (program->img_size == program->img_cap) {
808 0 : r = bf_program_grow_img(program);
809 0 : if (r)
810 : return r;
811 : }
812 :
813 0 : r = bf_fixup_new(&fixup, type, program->img_size, attr);
814 0 : if (r)
815 : return r;
816 :
817 0 : r = bf_list_add_tail(&program->fixups, fixup);
818 0 : if (r)
819 : return r;
820 :
821 0 : TAKE_PTR(fixup);
822 :
823 : /* This call could fail and return an error, in which case it is not
824 : * properly handled. However, this shouldn't be an issue as we previously
825 : * test whether enough room is available in cgen.img, which is currently
826 : * the only reason for EMIT() to fail. */
827 0 : EMIT(program, insn);
828 :
829 : return 0;
830 : }
831 :
832 0 : int bf_program_emit_fixup_elfstub(struct bf_program *program,
833 : enum bf_elfstub_id id)
834 : {
835 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
836 : int r;
837 :
838 0 : bf_assert(program);
839 :
840 0 : if (program->img_size == program->img_cap) {
841 0 : r = bf_program_grow_img(program);
842 0 : if (r)
843 : return r;
844 : }
845 :
846 0 : r = bf_fixup_new(&fixup, BF_FIXUP_ELFSTUB_CALL, program->img_size, NULL);
847 0 : if (r)
848 : return r;
849 :
850 0 : fixup->attr.elfstub_id = id;
851 :
852 0 : r = bf_list_add_tail(&program->fixups, fixup);
853 0 : if (r)
854 : return r;
855 :
856 0 : TAKE_PTR(fixup);
857 :
858 0 : EMIT(program, BPF_CALL_REL(0));
859 :
860 0 : return 0;
861 : }
862 :
863 0 : int bf_program_generate(struct bf_program *program)
864 : {
865 0 : const struct bf_chain *chain = program->runtime.chain;
866 : int r;
867 :
868 : // Save the program's argument into the context.
869 0 : EMIT(program,
870 : BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
871 :
872 : // Reset the protocol ID registers
873 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, 0));
874 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_8, 0));
875 :
876 : // If at least one rule logs the matched packets, populate ctx->log_map
877 0 : if (program->runtime.chain->flags & BF_FLAG(BF_CHAIN_LOG)) {
878 0 : EMIT_LOAD_LOG_FD_FIXUP(program, BPF_REG_2);
879 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2,
880 : BF_PROG_CTX_OFF(log_map)));
881 : }
882 :
883 : // Zeroing IPv6 extension headers
884 0 : if (program->runtime.chain->flags & BF_FLAG(BF_CHAIN_STORE_NEXTHDR)) {
885 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7,
886 : BF_PROG_CTX_OFF(ipv6_eh)));
887 : }
888 :
889 0 : r = program->runtime.ops->gen_inline_prologue(program);
890 0 : if (r)
891 : return r;
892 :
893 0 : bf_list_foreach (&chain->rules, rule_node) {
894 0 : r = _bf_program_generate_rule(program,
895 0 : bf_list_node_get_data(rule_node));
896 0 : if (r)
897 : return r;
898 : }
899 :
900 0 : r = program->runtime.ops->gen_inline_epilogue(program);
901 0 : if (r)
902 : return r;
903 :
904 : // Call the update counters function
905 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
906 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
907 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
908 0 : EMIT(program,
909 : BPF_MOV32_IMM(BPF_REG_3, bf_program_chain_counter_idx(program)));
910 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
911 :
912 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_0, program->runtime.ops->get_verdict(
913 : chain->policy)));
914 0 : EMIT(program, BPF_EXIT_INSN());
915 :
916 0 : r = _bf_program_generate_elfstubs(program);
917 0 : if (r)
918 : return r;
919 :
920 0 : r = _bf_program_fixup(program, BF_FIXUP_ELFSTUB_CALL);
921 0 : if (r)
922 0 : return bf_err_r(r, "failed to generate ELF stub call fixups");
923 :
924 : return 0;
925 : }
926 :
927 0 : static int _bf_program_load_printer_map(struct bf_program *program)
928 : {
929 0 : _cleanup_free_ void *pstr = NULL;
930 : size_t pstr_len;
931 0 : uint32_t key = 0;
932 : int r;
933 :
934 0 : bf_assert(program);
935 :
936 0 : r = bf_printer_assemble(program->printer, &pstr, &pstr_len);
937 0 : if (r)
938 0 : return bf_err_r(r, "failed to assemble printer map string");
939 :
940 0 : r = bf_map_set_value_size(program->pmap, pstr_len);
941 0 : if (r < 0)
942 : return r;
943 :
944 0 : r = bf_map_create(program->pmap, 0);
945 0 : if (r < 0)
946 : return r;
947 :
948 0 : r = bf_map_set_elem(program->pmap, &key, pstr);
949 0 : if (r)
950 : return r;
951 :
952 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_PRINTER_MAP_FD);
953 0 : if (r) {
954 0 : bf_map_destroy(program->pmap);
955 0 : return bf_err_r(r, "failed to fixup printer map FD");
956 : }
957 :
958 : return 0;
959 : }
960 :
961 0 : static int _bf_program_load_counters_map(struct bf_program *program)
962 : {
963 0 : _cleanup_close_ int _fd = -1;
964 : int r;
965 :
966 0 : bf_assert(program);
967 :
968 0 : r = bf_map_set_n_elems(program->cmap,
969 0 : bf_list_size(&program->runtime.chain->rules) + 2);
970 0 : if (r < 0)
971 : return r;
972 :
973 0 : r = bf_map_create(program->cmap, 0);
974 0 : if (r < 0)
975 : return r;
976 :
977 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_COUNTERS_MAP_FD);
978 0 : if (r < 0) {
979 0 : bf_map_destroy(program->cmap);
980 0 : return bf_err_r(r, "failed to fixup counters map FD");
981 : }
982 :
983 : return 0;
984 : }
985 :
986 0 : static int _bf_program_load_log_map(struct bf_program *program)
987 : {
988 0 : _cleanup_close_ int _fd = -1;
989 : int r;
990 :
991 0 : bf_assert(program);
992 :
993 0 : r = bf_map_create(program->lmap, 0);
994 0 : if (r < 0)
995 : return r;
996 :
997 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_LOG_MAP_FD);
998 0 : if (r < 0) {
999 0 : bf_map_destroy(program->lmap);
1000 0 : return bf_err_r(r, "failed to fixup log map FD");
1001 : }
1002 :
1003 : return 0;
1004 : }
1005 :
1006 0 : static int _bf_program_load_sets_maps(struct bf_program *new_prog)
1007 : {
1008 : const bf_list_node *set_node;
1009 : const bf_list_node *map_node;
1010 : int r;
1011 :
1012 0 : bf_assert(new_prog);
1013 :
1014 0 : set_node = bf_list_get_head(&new_prog->runtime.chain->sets);
1015 0 : map_node = bf_list_get_head(&new_prog->sets);
1016 :
1017 : // Fill the bf_map with the sets content
1018 0 : while (set_node && map_node) {
1019 : _cleanup_free_ uint8_t *values = NULL;
1020 : _cleanup_free_ uint8_t *keys = NULL;
1021 0 : struct bf_set *set = bf_list_node_get_data(set_node);
1022 0 : struct bf_map *map = bf_list_node_get_data(map_node);
1023 0 : size_t nelems = bf_list_size(&set->elems);
1024 0 : union bpf_attr attr = {};
1025 : size_t idx = 0;
1026 :
1027 0 : r = bf_map_create(map, 0);
1028 0 : if (r < 0) {
1029 0 : r = bf_err_r(r, "failed to create BPF map for set");
1030 0 : goto err_destroy_maps;
1031 : }
1032 :
1033 0 : values = malloc(nelems);
1034 0 : if (!values) {
1035 0 : r = bf_err_r(errno, "failed to allocate map values");
1036 0 : goto err_destroy_maps;
1037 : }
1038 :
1039 0 : keys = malloc(set->elem_size * nelems);
1040 0 : if (!keys) {
1041 0 : r = bf_err_r(errno, "failed to allocate map keys");
1042 0 : goto err_destroy_maps;
1043 : }
1044 :
1045 0 : bf_list_foreach (&set->elems, elem_node) {
1046 0 : void *elem = bf_list_node_get_data(elem_node);
1047 :
1048 0 : memcpy(keys + (idx * set->elem_size), elem, set->elem_size);
1049 0 : values[idx] = 1;
1050 0 : ++idx;
1051 : }
1052 :
1053 0 : attr.batch.map_fd = map->fd;
1054 0 : attr.batch.keys = (unsigned long long)keys;
1055 0 : attr.batch.values = (unsigned long long)values;
1056 0 : attr.batch.count = nelems;
1057 0 : attr.batch.flags = BPF_ANY;
1058 :
1059 0 : r = bf_bpf(BPF_MAP_UPDATE_BATCH, &attr);
1060 0 : if (r < 0) {
1061 0 : bf_err_r(r, "failed to add set elements to the map");
1062 0 : goto err_destroy_maps;
1063 : }
1064 :
1065 0 : set_node = bf_list_node_next(set_node);
1066 0 : map_node = bf_list_node_next(map_node);
1067 : }
1068 :
1069 0 : r = _bf_program_fixup(new_prog, BF_FIXUP_TYPE_SET_MAP_FD);
1070 0 : if (r < 0)
1071 0 : goto err_destroy_maps;
1072 :
1073 : return 0;
1074 :
1075 0 : err_destroy_maps:
1076 0 : bf_list_foreach (&new_prog->sets, map_node)
1077 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1078 : return r;
1079 : }
1080 :
1081 0 : int bf_program_load(struct bf_program *prog)
1082 : {
1083 : _cleanup_free_ char *log_buf = NULL;
1084 : int r;
1085 :
1086 0 : bf_assert(prog && prog->img);
1087 :
1088 0 : r = _bf_program_load_sets_maps(prog);
1089 0 : if (r)
1090 : return r;
1091 :
1092 0 : r = _bf_program_load_counters_map(prog);
1093 0 : if (r)
1094 : return r;
1095 :
1096 0 : r = _bf_program_load_printer_map(prog);
1097 0 : if (r)
1098 : return r;
1099 :
1100 0 : r = _bf_program_load_log_map(prog);
1101 0 : if (r)
1102 : return r;
1103 :
1104 0 : if (bf_opts_is_verbose(BF_VERBOSE_DEBUG)) {
1105 0 : log_buf = malloc(_BF_LOG_BUF_SIZE);
1106 0 : if (!log_buf) {
1107 0 : return bf_err_r(-ENOMEM,
1108 : "failed to allocate BPF_PROG_LOAD logs buffer");
1109 : }
1110 : }
1111 :
1112 0 : if (bf_opts_is_verbose(BF_VERBOSE_BYTECODE))
1113 0 : bf_program_dump_bytecode(prog);
1114 :
1115 0 : r = bf_bpf_prog_load(
1116 0 : prog->prog_name, bf_hook_to_bpf_prog_type(prog->runtime.chain->hook),
1117 0 : prog->img, prog->img_size,
1118 0 : bf_hook_to_bpf_attach_type(prog->runtime.chain->hook), log_buf,
1119 : log_buf ? _BF_LOG_BUF_SIZE : 0, bf_ctx_token(), &prog->runtime.prog_fd);
1120 0 : if (r) {
1121 0 : return bf_err_r(r, "failed to load bf_program (%lu bytes):\n%s\nerrno:",
1122 : prog->img_size, log_buf ? log_buf : "<NO LOG BUFFER>");
1123 : }
1124 :
1125 : return r;
1126 : }
1127 :
1128 0 : int bf_program_attach(struct bf_program *prog, struct bf_hookopts **hookopts)
1129 : {
1130 : int r;
1131 :
1132 0 : bf_assert(prog && hookopts);
1133 :
1134 0 : r = bf_link_attach(prog->link, prog->runtime.chain->hook, hookopts,
1135 : prog->runtime.prog_fd);
1136 0 : if (r) {
1137 0 : return bf_err_r(r, "failed to attach bf_link for %s program",
1138 : bf_flavor_to_str(prog->flavor));
1139 : }
1140 :
1141 : return r;
1142 : }
1143 :
1144 0 : void bf_program_detach(struct bf_program *prog)
1145 : {
1146 0 : bf_assert(prog);
1147 :
1148 0 : bf_link_detach(prog->link);
1149 0 : }
1150 :
1151 0 : void bf_program_unload(struct bf_program *prog)
1152 : {
1153 0 : bf_assert(prog);
1154 :
1155 0 : closep(&prog->runtime.prog_fd);
1156 0 : bf_link_detach(prog->link);
1157 0 : bf_map_destroy(prog->cmap);
1158 0 : bf_map_destroy(prog->pmap);
1159 0 : bf_map_destroy(prog->lmap);
1160 0 : bf_list_foreach (&prog->sets, map_node)
1161 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1162 0 : }
1163 :
1164 0 : int bf_program_get_counter(const struct bf_program *program,
1165 : uint32_t counter_idx, struct bf_counter *counter)
1166 : {
1167 0 : bf_assert(program);
1168 0 : bf_assert(counter);
1169 :
1170 : int r;
1171 :
1172 0 : r = bf_bpf_map_lookup_elem(program->cmap->fd, &counter_idx, counter);
1173 0 : if (r < 0)
1174 0 : return bf_err_r(errno, "failed to lookup counters map");
1175 :
1176 : return 0;
1177 : }
1178 :
1179 0 : int bf_cgen_set_counters(struct bf_program *program,
1180 : const struct bf_counter *counters)
1181 : {
1182 : UNUSED(program);
1183 : UNUSED(counters);
1184 :
1185 0 : return -ENOTSUP;
1186 : }
1187 :
1188 0 : int bf_program_pin(struct bf_program *prog, int dir_fd)
1189 : {
1190 : const char *name;
1191 : int r;
1192 :
1193 0 : bf_assert(prog);
1194 :
1195 0 : name = prog->runtime.chain->name;
1196 :
1197 0 : r = bf_bpf_obj_pin(prog->prog_name, prog->runtime.prog_fd, dir_fd);
1198 0 : if (r) {
1199 0 : bf_err_r(r, "failed to pin BPF program for '%s'", name);
1200 0 : goto err_unpin_all;
1201 : }
1202 :
1203 0 : r = bf_map_pin(prog->cmap, dir_fd);
1204 0 : if (r) {
1205 0 : bf_err_r(r, "failed to pin BPF counters map for '%s'", name);
1206 0 : goto err_unpin_all;
1207 : }
1208 :
1209 0 : r = bf_map_pin(prog->pmap, dir_fd);
1210 0 : if (r) {
1211 0 : bf_err_r(r, "failed to pin BPF printer map for '%s'", name);
1212 0 : goto err_unpin_all;
1213 : }
1214 :
1215 0 : r = bf_map_pin(prog->lmap, dir_fd);
1216 0 : if (r) {
1217 0 : bf_err_r(r, "failed to pin BPF log map for '%s'", name);
1218 0 : goto err_unpin_all;
1219 : }
1220 :
1221 0 : bf_list_foreach (&prog->sets, set_node) {
1222 0 : r = bf_map_pin(bf_list_node_get_data(set_node), dir_fd);
1223 0 : if (r) {
1224 0 : bf_err_r(r, "failed to pin BPF set map for '%s'", name);
1225 0 : goto err_unpin_all;
1226 : }
1227 : }
1228 :
1229 : // If a link exists, pin it too.
1230 0 : if (prog->link->hookopts) {
1231 0 : r = bf_link_pin(prog->link, dir_fd);
1232 0 : if (r) {
1233 0 : bf_err_r(r, "failed to pin BPF link for '%s'", name);
1234 0 : goto err_unpin_all;
1235 : }
1236 : }
1237 :
1238 : return 0;
1239 :
1240 0 : err_unpin_all:
1241 0 : bf_program_unpin(prog, dir_fd);
1242 0 : return r;
1243 : }
1244 :
1245 0 : void bf_program_unpin(struct bf_program *prog, int dir_fd)
1246 : {
1247 0 : bf_assert(prog);
1248 :
1249 0 : bf_map_unpin(prog->cmap, dir_fd);
1250 0 : bf_map_unpin(prog->pmap, dir_fd);
1251 0 : bf_map_unpin(prog->lmap, dir_fd);
1252 :
1253 0 : bf_list_foreach (&prog->sets, set_node)
1254 0 : bf_map_unpin(bf_list_node_get_data(set_node), dir_fd);
1255 :
1256 0 : bf_link_unpin(prog->link, dir_fd);
1257 :
1258 0 : unlinkat(dir_fd, prog->prog_name, 0);
1259 0 : }
1260 :
1261 0 : size_t bf_program_chain_counter_idx(const struct bf_program *program)
1262 : {
1263 0 : return bf_list_size(&program->runtime.chain->rules);
1264 : }
1265 :
1266 0 : size_t bf_program_error_counter_idx(const struct bf_program *program)
1267 : {
1268 0 : return bf_list_size(&program->runtime.chain->rules) + 1;
1269 : }
|
