Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0-only */
2 : /*
3 : * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
4 : */
5 :
6 : #include "bpfilter/cgen/program.h"
7 :
8 : #include <linux/bpf.h>
9 : #include <linux/bpf_common.h>
10 : #include <linux/limits.h>
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <stddef.h>
16 : #include <stdint.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <string.h>
20 : #include <unistd.h>
21 :
22 : #include "bpfilter/cgen/cgroup.h"
23 : #include "bpfilter/cgen/dump.h"
24 : #include "bpfilter/cgen/fixup.h"
25 : #include "bpfilter/cgen/jmp.h"
26 : #include "bpfilter/cgen/matcher/icmp.h"
27 : #include "bpfilter/cgen/matcher/ip4.h"
28 : #include "bpfilter/cgen/matcher/ip6.h"
29 : #include "bpfilter/cgen/matcher/meta.h"
30 : #include "bpfilter/cgen/matcher/set.h"
31 : #include "bpfilter/cgen/matcher/tcp.h"
32 : #include "bpfilter/cgen/matcher/udp.h"
33 : #include "bpfilter/cgen/nf.h"
34 : #include "bpfilter/cgen/printer.h"
35 : #include "bpfilter/cgen/prog/link.h"
36 : #include "bpfilter/cgen/prog/map.h"
37 : #include "bpfilter/cgen/stub.h"
38 : #include "bpfilter/cgen/tc.h"
39 : #include "bpfilter/cgen/xdp.h"
40 : #include "bpfilter/ctx.h"
41 : #include "bpfilter/opts.h"
42 : #include "core/bpf.h"
43 : #include "core/btf.h"
44 : #include "core/chain.h"
45 : #include "core/counter.h"
46 : #include "core/dump.h"
47 : #include "core/flavor.h"
48 : #include "core/helper.h"
49 : #include "core/hook.h"
50 : #include "core/io.h"
51 : #include "core/list.h"
52 : #include "core/logger.h"
53 : #include "core/marsh.h"
54 : #include "core/matcher.h"
55 : #include "core/rule.h"
56 : #include "core/set.h"
57 : #include "core/verdict.h"
58 :
59 : #include "external/filter.h"
60 :
61 : #define _BF_LOG_BUF_SIZE \
62 : (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */
63 : #define _BF_PROGRAM_DEFAULT_IMG_SIZE (1 << 6)
64 :
65 7 : static const struct bf_flavor_ops *bf_flavor_ops_get(enum bf_flavor flavor)
66 : {
67 : static const struct bf_flavor_ops *flavor_ops[] = {
68 : [BF_FLAVOR_TC] = &bf_flavor_ops_tc,
69 : [BF_FLAVOR_NF] = &bf_flavor_ops_nf,
70 : [BF_FLAVOR_XDP] = &bf_flavor_ops_xdp,
71 : [BF_FLAVOR_CGROUP] = &bf_flavor_ops_cgroup,
72 : };
73 :
74 : static_assert(ARRAY_SIZE(flavor_ops) == _BF_FLAVOR_MAX,
75 : "missing entries in bf_flavor_ops array");
76 :
77 7 : return flavor_ops[flavor];
78 : }
79 :
80 3 : int bf_program_new(struct bf_program **program, const struct bf_chain *chain)
81 : {
82 3 : _free_bf_program_ struct bf_program *_program = NULL;
83 : char name[BPF_OBJ_NAME_LEN];
84 : uint32_t set_idx = 0;
85 : int r;
86 :
87 3 : bf_assert(program && chain);
88 :
89 3 : _program = calloc(1, sizeof(*_program));
90 3 : if (!_program)
91 : return -ENOMEM;
92 :
93 3 : _program->flavor = bf_hook_to_flavor(chain->hook);
94 3 : _program->runtime.prog_fd = -1;
95 3 : _program->runtime.ops = bf_flavor_ops_get(_program->flavor);
96 3 : _program->runtime.chain = chain;
97 :
98 3 : (void)snprintf(_program->prog_name, BPF_OBJ_NAME_LEN, "%s", "bf_prog");
99 :
100 3 : r = bf_map_new(&_program->cmap, "counters_map", BF_MAP_TYPE_COUNTERS,
101 : BF_MAP_BPF_TYPE_ARRAY, sizeof(uint32_t),
102 : sizeof(struct bf_counter), 1);
103 3 : if (r < 0)
104 0 : return bf_err_r(r, "failed to create the counters bf_map object");
105 :
106 3 : r = bf_map_new(&_program->pmap, "printer_map", BF_MAP_TYPE_PRINTER,
107 : BF_MAP_BPF_TYPE_ARRAY, sizeof(uint32_t),
108 : BF_MAP_VALUE_SIZE_UNKNOWN, 1);
109 3 : if (r < 0)
110 0 : return bf_err_r(r, "failed to create the printer bf_map object");
111 :
112 3 : _program->sets = bf_map_list();
113 6 : bf_list_foreach (&chain->sets, set_node) {
114 0 : struct bf_set *set = bf_list_node_get_data(set_node);
115 0 : _free_bf_map_ struct bf_map *map = NULL;
116 :
117 0 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "set_%04x", (uint8_t)set_idx++);
118 0 : r = bf_map_new(&map, name, BF_MAP_TYPE_SET,
119 : bf_set_type_to_map_bpf_type(set->type), set->elem_size,
120 0 : 1, bf_list_size(&set->elems));
121 0 : if (r < 0)
122 : return r;
123 :
124 0 : r = bf_list_add_tail(&_program->sets, map);
125 0 : if (r < 0)
126 : return r;
127 0 : TAKE_PTR(map);
128 : };
129 :
130 3 : r = bf_link_new(&_program->link, "bf_link");
131 3 : if (r)
132 : return r;
133 :
134 3 : r = bf_printer_new(&_program->printer);
135 3 : if (r)
136 : return r;
137 :
138 3 : bf_list_init(&_program->fixups,
139 3 : (bf_list_ops[]) {{.free = (bf_list_ops_free)bf_fixup_free}});
140 :
141 3 : *program = TAKE_PTR(_program);
142 :
143 3 : return 0;
144 : }
145 :
146 15 : void bf_program_free(struct bf_program **program)
147 : {
148 15 : if (!*program)
149 : return;
150 :
151 3 : bf_list_clean(&(*program)->fixups);
152 3 : free((*program)->img);
153 :
154 : /* Close the file descriptors if they are still open. If --transient is
155 : * used, then the file descriptors are already closed (as
156 : * bf_program_unload() has been called). Otherwise, bf_program_unload()
157 : * won't be called, but the programs are pinned, so they can be closed
158 : * safely. */
159 3 : closep(&(*program)->runtime.prog_fd);
160 :
161 3 : bf_map_free(&(*program)->cmap);
162 3 : bf_map_free(&(*program)->pmap);
163 3 : bf_list_clean(&(*program)->sets);
164 3 : bf_link_free(&(*program)->link);
165 3 : bf_printer_free(&(*program)->printer);
166 :
167 3 : free(*program);
168 3 : *program = NULL;
169 : }
170 :
171 0 : int bf_program_marsh(const struct bf_program *program, struct bf_marsh **marsh)
172 : {
173 0 : _free_bf_marsh_ struct bf_marsh *_marsh = NULL;
174 : int r;
175 :
176 0 : bf_assert(program);
177 0 : bf_assert(marsh);
178 :
179 0 : r = bf_marsh_new(&_marsh, NULL, 0);
180 0 : if (r < 0)
181 : return r;
182 :
183 : {
184 : // Serialize bf_program.counters
185 0 : _free_bf_marsh_ struct bf_marsh *counters_elem = NULL;
186 :
187 0 : r = bf_map_marsh(program->cmap, &counters_elem);
188 0 : if (r < 0)
189 : return r;
190 :
191 0 : r = bf_marsh_add_child_obj(&_marsh, counters_elem);
192 0 : if (r < 0)
193 : return r;
194 : }
195 :
196 : {
197 : // Serialize bf_program.pmap
198 0 : _free_bf_marsh_ struct bf_marsh *pmap_elem = NULL;
199 :
200 0 : r = bf_map_marsh(program->pmap, &pmap_elem);
201 0 : if (r < 0)
202 : return r;
203 :
204 0 : r = bf_marsh_add_child_obj(&_marsh, pmap_elem);
205 0 : if (r < 0)
206 : return r;
207 : }
208 :
209 : {
210 : // Serialize bf_program.sets
211 0 : _free_bf_marsh_ struct bf_marsh *sets_elem = NULL;
212 :
213 0 : r = bf_list_marsh(&program->sets, &sets_elem);
214 0 : if (r < 0)
215 : return r;
216 :
217 0 : r = bf_marsh_add_child_obj(&_marsh, sets_elem);
218 0 : if (r < 0) {
219 0 : return bf_err_r(
220 : r,
221 : "failed to insert serialized sets into bf_program serialized data");
222 : }
223 : }
224 :
225 : {
226 : // Serialize bf_program.links
227 0 : _free_bf_marsh_ struct bf_marsh *links_elem = NULL;
228 :
229 0 : r = bf_link_marsh(program->link, &links_elem);
230 0 : if (r)
231 0 : return bf_err_r(r, "failed to serialize bf_program.link");
232 :
233 0 : r = bf_marsh_add_child_obj(&_marsh, links_elem);
234 0 : if (r) {
235 0 : return bf_err_r(
236 : r,
237 : "failed to insert serialized link into bf_program serialized data");
238 : }
239 : }
240 :
241 : {
242 : // Serialise bf_program.printer
243 0 : _free_bf_marsh_ struct bf_marsh *child = NULL;
244 :
245 0 : r = bf_printer_marsh(program->printer, &child);
246 0 : if (r)
247 0 : return bf_err_r(r, "failed to marsh bf_printer object");
248 :
249 0 : r = bf_marsh_add_child_obj(&_marsh, child);
250 0 : if (r)
251 0 : return bf_err_r(r, "failed to append object to marsh");
252 : }
253 :
254 0 : r |= bf_marsh_add_child_raw(&_marsh, program->img,
255 0 : program->img_size * sizeof(struct bpf_insn));
256 0 : if (r)
257 0 : return bf_err_r(r, "Failed to serialize program");
258 :
259 0 : *marsh = TAKE_PTR(_marsh);
260 :
261 0 : return 0;
262 : }
263 :
264 0 : int bf_program_unmarsh(const struct bf_marsh *marsh,
265 : struct bf_program **program,
266 : const struct bf_chain *chain, int dir_fd)
267 : {
268 0 : _free_bf_program_ struct bf_program *_program = NULL;
269 0 : _free_bf_link_ struct bf_link *link = NULL;
270 : struct bf_marsh *child = NULL;
271 : int r;
272 :
273 0 : bf_assert(marsh && program);
274 :
275 0 : r = bf_program_new(&_program, chain);
276 0 : if (r < 0)
277 : return r;
278 :
279 0 : if (!(child = bf_marsh_next_child(marsh, child)))
280 : return -EINVAL;
281 0 : bf_map_free(&_program->cmap);
282 0 : r = bf_map_new_from_marsh(&_program->cmap, dir_fd, child);
283 0 : if (r < 0)
284 : return r;
285 :
286 0 : if (!(child = bf_marsh_next_child(marsh, child)))
287 : return -EINVAL;
288 0 : bf_map_free(&_program->pmap);
289 0 : r = bf_map_new_from_marsh(&_program->pmap, dir_fd, child);
290 0 : if (r < 0)
291 : return r;
292 :
293 : /** @todo Avoid creating and filling the list in @ref bf_program_new before
294 : * trashing it all here. Eventually, this function will be replaced with
295 : * @c bf_program_new_from_marsh and this issue could be solved by **not**
296 : * relying on @ref bf_program_new to allocate an initialize @p _program . */
297 0 : bf_list_clean(&_program->sets);
298 0 : _program->sets = bf_map_list();
299 :
300 0 : if (!(child = bf_marsh_next_child(marsh, child)))
301 : return -EINVAL;
302 : {
303 : // Unmarsh bf_program.sets
304 : struct bf_marsh *set_elem = NULL;
305 :
306 0 : while ((set_elem = bf_marsh_next_child(child, set_elem))) {
307 0 : _free_bf_map_ struct bf_map *map = NULL;
308 :
309 0 : r = bf_map_new_from_marsh(&map, dir_fd, set_elem);
310 0 : if (r < 0)
311 : return r;
312 :
313 0 : r = bf_list_add_tail(&_program->sets, map);
314 0 : if (r < 0)
315 : return r;
316 :
317 0 : TAKE_PTR(map);
318 : }
319 : }
320 :
321 : // Unmarsh bf_program.links
322 0 : if (!(child = bf_marsh_next_child(marsh, child)))
323 : return -EINVAL;
324 :
325 : /* Try to restore the link: on success, replace the program's link with the
326 : * restored on. If -ENOENT is returned, the link doesn't exist, meaning the
327 : * program is not attached. Otherwise, return an error. */
328 0 : r = bf_link_new_from_marsh(&link, dir_fd, child);
329 0 : if (!r)
330 0 : bf_swap(_program->link, link);
331 0 : else if (r != -ENOENT)
332 0 : return bf_err_r(r, "failed to restore bf_program.link");
333 :
334 : // Unmarsh bf_program.printer
335 0 : child = bf_marsh_next_child(marsh, child);
336 0 : if (!child)
337 0 : return bf_err_r(-EINVAL, "failed to find valid child");
338 :
339 0 : bf_printer_free(&_program->printer);
340 0 : r = bf_printer_new_from_marsh(&_program->printer, child);
341 0 : if (r)
342 0 : return bf_err_r(r, "failed to restore bf_printer object");
343 :
344 0 : if (!(child = bf_marsh_next_child(marsh, child)))
345 : return -EINVAL;
346 0 : _program->img = bf_memdup(child->data, child->data_len);
347 0 : _program->img_size = child->data_len / sizeof(struct bpf_insn);
348 0 : _program->img_cap = child->data_len / sizeof(struct bpf_insn);
349 :
350 0 : if (bf_marsh_next_child(marsh, child))
351 0 : bf_warn("codegen marsh has more children than expected");
352 :
353 0 : r = bf_bpf_obj_get(_program->prog_name, dir_fd, &_program->runtime.prog_fd);
354 0 : if (r < 0)
355 0 : return bf_err_r(r, "failed to get prog fd");
356 :
357 0 : *program = TAKE_PTR(_program);
358 :
359 0 : return 0;
360 : }
361 :
362 0 : void bf_program_dump(const struct bf_program *program, prefix_t *prefix)
363 : {
364 0 : bf_assert(program);
365 0 : bf_assert(prefix);
366 :
367 0 : DUMP(prefix, "struct bf_program at %p", program);
368 :
369 0 : bf_dump_prefix_push(prefix);
370 :
371 0 : DUMP(prefix, "prog_name: %s", program->prog_name);
372 :
373 0 : DUMP(prefix, "cmap: struct bf_map *");
374 0 : bf_dump_prefix_push(prefix);
375 0 : bf_map_dump(program->cmap, bf_dump_prefix_last(prefix));
376 0 : bf_dump_prefix_pop(prefix);
377 :
378 0 : DUMP(prefix, "pmap: struct bf_map *");
379 0 : bf_dump_prefix_push(prefix);
380 0 : bf_map_dump(program->pmap, bf_dump_prefix_last(prefix));
381 0 : bf_dump_prefix_pop(prefix);
382 :
383 0 : DUMP(prefix, "sets: bf_list<bf_map>[%lu]", bf_list_size(&program->sets));
384 0 : bf_dump_prefix_push(prefix);
385 0 : bf_list_foreach (&program->sets, map_node) {
386 0 : struct bf_map *map = bf_list_node_get_data(map_node);
387 :
388 0 : if (bf_list_is_tail(&program->sets, map_node))
389 0 : bf_dump_prefix_last(prefix);
390 :
391 0 : bf_map_dump(map, prefix);
392 : }
393 0 : bf_dump_prefix_pop(prefix);
394 :
395 0 : DUMP(prefix, "link: struct bf_link *");
396 0 : bf_dump_prefix_push(prefix);
397 0 : bf_link_dump(program->link, prefix);
398 0 : bf_dump_prefix_pop(prefix);
399 :
400 0 : DUMP(prefix, "printer: struct bf_printer *");
401 0 : bf_dump_prefix_push(prefix);
402 0 : bf_printer_dump(program->printer, prefix);
403 0 : bf_dump_prefix_pop(prefix);
404 :
405 0 : DUMP(prefix, "img: %p", program->img);
406 0 : DUMP(prefix, "img_size: %lu", program->img_size);
407 0 : DUMP(prefix, "img_cap: %lu", program->img_cap);
408 :
409 0 : DUMP(prefix, "fixups: bf_list<struct bf_fixup>[%lu]",
410 : bf_list_size(&program->fixups));
411 0 : bf_dump_prefix_push(prefix);
412 0 : bf_list_foreach (&program->fixups, fixup_node) {
413 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
414 :
415 0 : if (bf_list_is_tail(&program->fixups, fixup_node))
416 0 : bf_dump_prefix_last(prefix);
417 :
418 0 : bf_fixup_dump(fixup, prefix);
419 : }
420 0 : bf_dump_prefix_pop(prefix);
421 :
422 0 : DUMP(bf_dump_prefix_last(prefix), "runtime: <anonymous>");
423 0 : bf_dump_prefix_push(prefix);
424 0 : DUMP(prefix, "prog_fd: %d", program->runtime.prog_fd);
425 0 : DUMP(bf_dump_prefix_last(prefix), "ops: %p", program->runtime.ops);
426 0 : bf_dump_prefix_pop(prefix);
427 :
428 0 : bf_dump_prefix_pop(prefix);
429 0 : }
430 :
431 0 : static inline size_t _bf_round_next_power_of_2(size_t value)
432 : {
433 0 : value--;
434 0 : value |= value >> 1;
435 0 : value |= value >> 2;
436 0 : value |= value >> 4;
437 0 : value |= value >> 8;
438 0 : value |= value >> 16;
439 :
440 0 : return ++value;
441 : }
442 :
443 3 : int bf_program_grow_img(struct bf_program *program)
444 : {
445 : size_t new_cap = _BF_PROGRAM_DEFAULT_IMG_SIZE;
446 : int r;
447 :
448 3 : bf_assert(program);
449 :
450 3 : if (program->img)
451 0 : new_cap = _bf_round_next_power_of_2(program->img_cap << 1);
452 :
453 3 : r = bf_realloc((void **)&program->img, new_cap * sizeof(struct bpf_insn));
454 3 : if (r < 0) {
455 0 : return bf_err_r(r, "failed to grow program img from %lu to %lu insn",
456 : program->img_cap, new_cap);
457 : }
458 :
459 3 : program->img_cap = new_cap;
460 :
461 3 : return 0;
462 : }
463 :
464 0 : static void _bf_program_fixup_insn(struct bpf_insn *insn,
465 : enum bf_fixup_insn type, int32_t value)
466 : {
467 0 : switch (type) {
468 0 : case BF_FIXUP_INSN_OFF:
469 0 : bf_assert(!insn->off);
470 0 : bf_assert(value < SHRT_MAX);
471 0 : insn->off = (int16_t)value;
472 0 : break;
473 0 : case BF_FIXUP_INSN_IMM:
474 0 : bf_assert(!insn->imm);
475 0 : insn->imm = value;
476 0 : break;
477 0 : default:
478 0 : bf_abort(
479 : "unsupported fixup instruction type, this should not happen: %d",
480 : type);
481 : break;
482 : }
483 0 : }
484 :
485 0 : static int _bf_program_fixup(struct bf_program *program,
486 : enum bf_fixup_type type)
487 : {
488 0 : bf_assert(program);
489 0 : bf_assert(type >= 0 && type < _BF_FIXUP_TYPE_MAX);
490 :
491 0 : bf_list_foreach (&program->fixups, fixup_node) {
492 : enum bf_fixup_insn insn_type = _BF_FIXUP_INSN_MAX;
493 : int32_t value;
494 : size_t offset;
495 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
496 0 : struct bpf_insn *insn = &program->img[fixup->insn];
497 : struct bf_map *map;
498 :
499 0 : if (type != fixup->type)
500 0 : continue;
501 :
502 0 : switch (type) {
503 0 : case BF_FIXUP_TYPE_JMP_NEXT_RULE:
504 : insn_type = BF_FIXUP_INSN_OFF;
505 0 : value = (int)(program->img_size - fixup->insn - 1U);
506 0 : break;
507 0 : case BF_FIXUP_TYPE_COUNTERS_MAP_FD:
508 : insn_type = BF_FIXUP_INSN_IMM;
509 0 : value = program->cmap->fd;
510 0 : break;
511 0 : case BF_FIXUP_TYPE_PRINTER_MAP_FD:
512 : insn_type = BF_FIXUP_INSN_IMM;
513 0 : value = program->pmap->fd;
514 0 : break;
515 0 : case BF_FIXUP_TYPE_SET_MAP_FD:
516 0 : map = bf_list_get_at(&program->sets, fixup->attr.set_index);
517 0 : if (!map) {
518 0 : return bf_err_r(-ENOENT, "can't find set map at index %lu",
519 : fixup->attr.set_index);
520 : }
521 : insn_type = BF_FIXUP_INSN_IMM;
522 0 : value = map->fd;
523 0 : break;
524 0 : case BF_FIXUP_ELFSTUB_CALL:
525 : insn_type = BF_FIXUP_INSN_IMM;
526 0 : offset = program->elfstubs_location[fixup->attr.elfstub_id] -
527 : fixup->insn - 1;
528 0 : if (offset >= INT_MAX)
529 0 : return bf_err_r(-EINVAL, "invalid ELF stub call offset");
530 0 : value = (int32_t)offset;
531 0 : break;
532 0 : default:
533 0 : bf_abort("unsupported fixup type, this should not happen: %d",
534 : type);
535 : break;
536 : }
537 :
538 0 : _bf_program_fixup_insn(insn, insn_type, value);
539 0 : bf_list_delete(&program->fixups, fixup_node);
540 : }
541 :
542 : return 0;
543 : }
544 :
545 0 : static int _bf_program_generate_rule(struct bf_program *program,
546 : struct bf_rule *rule)
547 : {
548 : int r;
549 :
550 0 : bf_assert(program);
551 0 : bf_assert(rule);
552 :
553 0 : bf_list_foreach (&rule->matchers, matcher_node) {
554 0 : struct bf_matcher *matcher = bf_list_node_get_data(matcher_node);
555 :
556 0 : switch (matcher->type) {
557 0 : case BF_MATCHER_META_IFINDEX:
558 : case BF_MATCHER_META_L3_PROTO:
559 : case BF_MATCHER_META_L4_PROTO:
560 : case BF_MATCHER_META_PROBABILITY:
561 : case BF_MATCHER_META_SPORT:
562 : case BF_MATCHER_META_DPORT:
563 0 : r = bf_matcher_generate_meta(program, matcher);
564 0 : if (r)
565 : return r;
566 : break;
567 0 : case BF_MATCHER_IP4_SADDR:
568 : case BF_MATCHER_IP4_SNET:
569 : case BF_MATCHER_IP4_DADDR:
570 : case BF_MATCHER_IP4_DNET:
571 : case BF_MATCHER_IP4_PROTO:
572 0 : r = bf_matcher_generate_ip4(program, matcher);
573 0 : if (r)
574 : return r;
575 : break;
576 0 : case BF_MATCHER_IP6_SADDR:
577 : case BF_MATCHER_IP6_SNET:
578 : case BF_MATCHER_IP6_DADDR:
579 : case BF_MATCHER_IP6_DNET:
580 0 : r = bf_matcher_generate_ip6(program, matcher);
581 0 : if (r)
582 : return r;
583 : break;
584 0 : case BF_MATCHER_TCP_SPORT:
585 : case BF_MATCHER_TCP_DPORT:
586 : case BF_MATCHER_TCP_FLAGS:
587 0 : r = bf_matcher_generate_tcp(program, matcher);
588 0 : if (r)
589 : return r;
590 : break;
591 0 : case BF_MATCHER_UDP_SPORT:
592 : case BF_MATCHER_UDP_DPORT:
593 0 : r = bf_matcher_generate_udp(program, matcher);
594 0 : if (r)
595 : return r;
596 : break;
597 0 : case BF_MATCHER_SET_SRCIP6PORT:
598 : case BF_MATCHER_SET_SRCIP6:
599 0 : r = bf_matcher_generate_set(program, matcher);
600 0 : if (r)
601 : return r;
602 : break;
603 0 : case BF_MATCHER_ICMP_TYPE:
604 : case BF_MATCHER_ICMP_CODE:
605 : case BF_MATCHER_ICMPV6_TYPE:
606 : case BF_MATCHER_ICMPV6_CODE:
607 0 : r = bf_matcher_generate_icmp(program, matcher);
608 0 : if (r)
609 : return r;
610 : break;
611 0 : default:
612 0 : return bf_err_r(-EINVAL, "unknown matcher type %d", matcher->type);
613 : };
614 : }
615 :
616 0 : if (rule->counters) {
617 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
618 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
619 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
620 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_3, rule->index));
621 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
622 : }
623 :
624 0 : switch (rule->verdict) {
625 0 : case BF_VERDICT_ACCEPT:
626 : case BF_VERDICT_DROP:
627 0 : EMIT(program,
628 : BPF_MOV64_IMM(BPF_REG_0,
629 : program->runtime.ops->get_verdict(rule->verdict)));
630 0 : EMIT(program, BPF_EXIT_INSN());
631 0 : break;
632 : case BF_VERDICT_CONTINUE:
633 : // Fall through to next rule or default chain policy.
634 : break;
635 0 : default:
636 0 : bf_abort("unsupported verdict, this should not happen: %d",
637 : rule->verdict);
638 : break;
639 : }
640 :
641 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_JMP_NEXT_RULE);
642 0 : if (r)
643 0 : return bf_err_r(r, "failed to generate next rule fixups");
644 :
645 : return 0;
646 : }
647 :
648 0 : static int _bf_program_generate_elfstubs(struct bf_program *program)
649 : {
650 : const struct bf_elfstub *elfstub;
651 : size_t start_at;
652 : int r;
653 :
654 0 : bf_assert(program);
655 :
656 0 : bf_list_foreach (&program->fixups, fixup_node) {
657 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
658 0 : size_t off = program->img_size;
659 :
660 0 : if (fixup->type != BF_FIXUP_ELFSTUB_CALL)
661 0 : continue;
662 :
663 : // Only generate each ELF stub once
664 0 : if (program->elfstubs_location[fixup->attr.elfstub_id])
665 0 : continue;
666 :
667 0 : bf_dbg("generate ELF stub for ID %d", fixup->attr.elfstub_id);
668 :
669 0 : elfstub = bf_ctx_get_elfstub(fixup->attr.elfstub_id);
670 0 : if (!elfstub) {
671 0 : return bf_err_r(-ENOENT, "no ELF stub found for ID %d",
672 : fixup->attr.elfstub_id);
673 : }
674 :
675 0 : start_at = program->img_size;
676 :
677 0 : for (size_t i = 0; i < elfstub->ninsns; ++i) {
678 0 : r = bf_program_emit(program, elfstub->insns[i]);
679 0 : if (r)
680 0 : return bf_err_r(r, "failed to insert ELF stub instruction");
681 : }
682 :
683 0 : bf_list_foreach (&elfstub->strs, pstr_node) {
684 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
685 0 : struct bf_printk_str *pstr = bf_list_node_get_data(pstr_node);
686 0 : size_t insn_idx = start_at + pstr->insn_idx;
687 : const struct bf_printer_msg *msg =
688 0 : bf_printer_add_msg(program->printer, pstr->str);
689 : struct bpf_insn ld_insn[2] = {
690 : BPF_LD_MAP_FD(BPF_REG_1, 0),
691 : };
692 :
693 : ld_insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
694 0 : ld_insn[1].imm = (int)bf_printer_msg_offset(msg);
695 :
696 0 : program->img[insn_idx] = ld_insn[0];
697 0 : program->img[insn_idx + 1] = ld_insn[1];
698 :
699 0 : r = bf_fixup_new(&fixup, BF_FIXUP_TYPE_PRINTER_MAP_FD, insn_idx,
700 : NULL);
701 0 : if (r)
702 : return r;
703 :
704 0 : r = bf_list_add_tail(&program->fixups, fixup);
705 0 : if (r)
706 : return r;
707 :
708 0 : TAKE_PTR(fixup);
709 : }
710 :
711 0 : program->elfstubs_location[fixup->attr.elfstub_id] = off;
712 : }
713 :
714 : return 0;
715 : }
716 :
717 40 : int bf_program_emit(struct bf_program *program, struct bpf_insn insn)
718 : {
719 : int r;
720 :
721 40 : bf_assert(program);
722 :
723 40 : if (program->img_size == program->img_cap) {
724 3 : r = bf_program_grow_img(program);
725 3 : if (r)
726 : return r;
727 : }
728 :
729 40 : program->img[program->img_size++] = insn;
730 :
731 40 : return 0;
732 : }
733 :
734 0 : int bf_program_emit_kfunc_call(struct bf_program *program, const char *name)
735 : {
736 : int r;
737 :
738 0 : bf_assert(program);
739 0 : bf_assert(name);
740 :
741 0 : r = bf_btf_get_id(name);
742 0 : if (r < 0)
743 : return r;
744 :
745 0 : EMIT(program, ((struct bpf_insn) {.code = BPF_JMP | BPF_CALL,
746 : .dst_reg = 0,
747 : .src_reg = BPF_PSEUDO_KFUNC_CALL,
748 : .off = 0,
749 : .imm = r}));
750 :
751 0 : return 0;
752 : }
753 :
754 0 : int bf_program_emit_fixup(struct bf_program *program, enum bf_fixup_type type,
755 : struct bpf_insn insn, const union bf_fixup_attr *attr)
756 : {
757 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
758 : int r;
759 :
760 0 : bf_assert(program);
761 :
762 0 : if (program->img_size == program->img_cap) {
763 0 : r = bf_program_grow_img(program);
764 0 : if (r)
765 : return r;
766 : }
767 :
768 0 : r = bf_fixup_new(&fixup, type, program->img_size, attr);
769 0 : if (r)
770 : return r;
771 :
772 0 : r = bf_list_add_tail(&program->fixups, fixup);
773 0 : if (r)
774 : return r;
775 :
776 0 : TAKE_PTR(fixup);
777 :
778 : /* This call could fail and return an error, in which case it is not
779 : * properly handled. However, this shouldn't be an issue as we previously
780 : * test whether enough room is available in cgen.img, which is currently
781 : * the only reason for EMIT() to fail. */
782 0 : EMIT(program, insn);
783 :
784 : return 0;
785 : }
786 :
787 0 : int bf_program_emit_fixup_elfstub(struct bf_program *program,
788 : enum bf_elfstub_id id)
789 : {
790 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
791 : int r;
792 :
793 0 : bf_assert(program);
794 :
795 0 : if (program->img_size == program->img_cap) {
796 0 : r = bf_program_grow_img(program);
797 0 : if (r)
798 : return r;
799 : }
800 :
801 0 : r = bf_fixup_new(&fixup, BF_FIXUP_ELFSTUB_CALL, program->img_size, NULL);
802 0 : if (r)
803 : return r;
804 :
805 0 : fixup->attr.elfstub_id = id;
806 :
807 0 : r = bf_list_add_tail(&program->fixups, fixup);
808 0 : if (r)
809 : return r;
810 :
811 0 : TAKE_PTR(fixup);
812 :
813 0 : EMIT(program, BPF_CALL_REL(0));
814 :
815 0 : return 0;
816 : }
817 :
818 0 : int bf_program_generate(struct bf_program *program)
819 : {
820 0 : const struct bf_chain *chain = program->runtime.chain;
821 : int r;
822 :
823 : // Save the program's argument into the context.
824 0 : EMIT(program,
825 : BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
826 :
827 : // Reset the protocol ID registers
828 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, 0));
829 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_8, 0));
830 :
831 0 : r = program->runtime.ops->gen_inline_prologue(program);
832 0 : if (r)
833 : return r;
834 :
835 0 : bf_list_foreach (&chain->rules, rule_node) {
836 0 : r = _bf_program_generate_rule(program,
837 0 : bf_list_node_get_data(rule_node));
838 0 : if (r)
839 : return r;
840 : }
841 :
842 0 : r = program->runtime.ops->gen_inline_epilogue(program);
843 0 : if (r)
844 : return r;
845 :
846 : // Call the update counters function
847 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
848 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
849 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
850 0 : EMIT(program,
851 : BPF_MOV32_IMM(BPF_REG_3, bf_program_chain_counter_idx(program)));
852 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
853 :
854 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_0, program->runtime.ops->get_verdict(
855 : chain->policy)));
856 0 : EMIT(program, BPF_EXIT_INSN());
857 :
858 0 : r = _bf_program_generate_elfstubs(program);
859 0 : if (r)
860 : return r;
861 :
862 0 : r = _bf_program_fixup(program, BF_FIXUP_ELFSTUB_CALL);
863 0 : if (r)
864 0 : return bf_err_r(r, "failed to generate ELF stub call fixups");
865 :
866 : return 0;
867 : }
868 :
869 0 : static int _bf_program_load_printer_map(struct bf_program *program)
870 : {
871 0 : _cleanup_free_ void *pstr = NULL;
872 : size_t pstr_len;
873 0 : uint32_t key = 0;
874 : int r;
875 :
876 0 : bf_assert(program);
877 :
878 0 : r = bf_printer_assemble(program->printer, &pstr, &pstr_len);
879 0 : if (r)
880 0 : return bf_err_r(r, "failed to assemble printer map string");
881 :
882 0 : r = bf_map_set_value_size(program->pmap, pstr_len);
883 0 : if (r < 0)
884 : return r;
885 :
886 0 : r = bf_map_create(program->pmap, 0);
887 0 : if (r < 0)
888 : return r;
889 :
890 0 : r = bf_map_set_elem(program->pmap, &key, pstr);
891 0 : if (r)
892 : return r;
893 :
894 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_PRINTER_MAP_FD);
895 0 : if (r) {
896 0 : bf_map_destroy(program->pmap);
897 0 : return bf_err_r(r, "failed to fixup printer map FD");
898 : }
899 :
900 : return 0;
901 : }
902 :
903 0 : static int _bf_program_load_counters_map(struct bf_program *program)
904 : {
905 0 : _cleanup_close_ int _fd = -1;
906 : int r;
907 :
908 0 : bf_assert(program);
909 :
910 0 : r = bf_map_set_n_elems(program->cmap,
911 0 : bf_list_size(&program->runtime.chain->rules) + 2);
912 0 : if (r < 0)
913 : return r;
914 :
915 0 : r = bf_map_create(program->cmap, 0);
916 0 : if (r < 0)
917 : return r;
918 :
919 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_COUNTERS_MAP_FD);
920 0 : if (r < 0) {
921 0 : bf_map_destroy(program->cmap);
922 0 : return bf_err_r(r, "failed to fixup counters map FD");
923 : }
924 :
925 : return 0;
926 : }
927 :
928 0 : static int _bf_program_load_sets_maps(struct bf_program *new_prog)
929 : {
930 : const bf_list_node *set_node;
931 : const bf_list_node *map_node;
932 : int r;
933 :
934 0 : bf_assert(new_prog);
935 :
936 0 : set_node = bf_list_get_head(&new_prog->runtime.chain->sets);
937 0 : map_node = bf_list_get_head(&new_prog->sets);
938 :
939 : // Fill the bf_map with the sets content
940 0 : while (set_node && map_node) {
941 : _cleanup_free_ uint8_t *values = NULL;
942 : _cleanup_free_ uint8_t *keys = NULL;
943 0 : struct bf_set *set = bf_list_node_get_data(set_node);
944 0 : struct bf_map *map = bf_list_node_get_data(map_node);
945 0 : size_t nelems = bf_list_size(&set->elems);
946 0 : union bpf_attr attr = {};
947 : size_t idx = 0;
948 :
949 0 : r = bf_map_create(map, 0);
950 0 : if (r < 0) {
951 0 : r = bf_err_r(r, "failed to create BPF map for set");
952 0 : goto err_destroy_maps;
953 : }
954 :
955 0 : values = malloc(nelems);
956 0 : if (!values) {
957 0 : r = bf_err_r(errno, "failed to allocate map values");
958 0 : goto err_destroy_maps;
959 : }
960 :
961 0 : keys = malloc(set->elem_size * nelems);
962 0 : if (!keys) {
963 0 : r = bf_err_r(errno, "failed to allocate map keys");
964 0 : goto err_destroy_maps;
965 : }
966 :
967 0 : bf_list_foreach (&set->elems, elem_node) {
968 0 : void *elem = bf_list_node_get_data(elem_node);
969 :
970 0 : memcpy(keys + (idx * set->elem_size), elem, set->elem_size);
971 0 : values[idx] = 1;
972 0 : ++idx;
973 : }
974 :
975 0 : attr.batch.map_fd = map->fd;
976 0 : attr.batch.keys = (unsigned long long)keys;
977 0 : attr.batch.values = (unsigned long long)values;
978 0 : attr.batch.count = nelems;
979 0 : attr.batch.flags = BPF_ANY;
980 :
981 0 : r = bf_bpf(BPF_MAP_UPDATE_BATCH, &attr);
982 0 : if (r < 0) {
983 0 : bf_err_r(r, "failed to add set elements to the map");
984 0 : goto err_destroy_maps;
985 : }
986 :
987 0 : set_node = bf_list_node_next(set_node);
988 0 : map_node = bf_list_node_next(map_node);
989 : }
990 :
991 0 : r = _bf_program_fixup(new_prog, BF_FIXUP_TYPE_SET_MAP_FD);
992 0 : if (r < 0)
993 0 : goto err_destroy_maps;
994 :
995 : return 0;
996 :
997 0 : err_destroy_maps:
998 0 : bf_list_foreach (&new_prog->sets, map_node)
999 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1000 : return r;
1001 : }
1002 :
1003 0 : int bf_program_load(struct bf_program *prog)
1004 : {
1005 : _cleanup_free_ char *log_buf = NULL;
1006 : int r;
1007 :
1008 0 : bf_assert(prog && prog->img);
1009 :
1010 0 : r = _bf_program_load_sets_maps(prog);
1011 0 : if (r)
1012 : return r;
1013 :
1014 0 : r = _bf_program_load_counters_map(prog);
1015 0 : if (r)
1016 : return r;
1017 :
1018 0 : r = _bf_program_load_printer_map(prog);
1019 0 : if (r)
1020 : return r;
1021 :
1022 0 : if (bf_opts_is_verbose(BF_VERBOSE_DEBUG)) {
1023 0 : log_buf = malloc(_BF_LOG_BUF_SIZE);
1024 0 : if (!log_buf) {
1025 0 : return bf_err_r(-ENOMEM,
1026 : "failed to allocate BPF_PROG_LOAD logs buffer");
1027 : }
1028 : }
1029 :
1030 0 : if (bf_opts_is_verbose(BF_VERBOSE_BYTECODE))
1031 0 : bf_program_dump_bytecode(prog);
1032 :
1033 0 : r = bf_bpf_prog_load(
1034 0 : prog->prog_name, bf_hook_to_bpf_prog_type(prog->runtime.chain->hook),
1035 0 : prog->img, prog->img_size,
1036 0 : bf_hook_to_bpf_attach_type(prog->runtime.chain->hook), log_buf,
1037 : log_buf ? _BF_LOG_BUF_SIZE : 0, bf_ctx_token(), &prog->runtime.prog_fd);
1038 0 : if (r) {
1039 0 : return bf_err_r(r, "failed to load bf_program (%lu bytes):\n%s\nerrno:",
1040 : prog->img_size, log_buf ? log_buf : "<NO LOG BUFFER>");
1041 : }
1042 :
1043 : return r;
1044 : }
1045 :
1046 0 : int bf_program_attach(struct bf_program *prog, struct bf_hookopts **hookopts)
1047 : {
1048 : int r;
1049 :
1050 0 : bf_assert(prog && hookopts);
1051 :
1052 0 : r = bf_link_attach(prog->link, prog->runtime.chain->hook, hookopts,
1053 : prog->runtime.prog_fd);
1054 0 : if (r) {
1055 0 : return bf_err_r(r, "failed to attach bf_link for %s program",
1056 : bf_flavor_to_str(prog->flavor));
1057 : }
1058 :
1059 : return r;
1060 : }
1061 :
1062 0 : void bf_program_detach(struct bf_program *prog)
1063 : {
1064 0 : bf_assert(prog);
1065 :
1066 0 : bf_link_detach(prog->link);
1067 0 : }
1068 :
1069 0 : void bf_program_unload(struct bf_program *prog)
1070 : {
1071 0 : bf_assert(prog);
1072 :
1073 0 : closep(&prog->runtime.prog_fd);
1074 0 : bf_link_detach(prog->link);
1075 0 : bf_map_destroy(prog->cmap);
1076 0 : bf_map_destroy(prog->pmap);
1077 0 : bf_list_foreach (&prog->sets, map_node)
1078 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1079 0 : }
1080 :
1081 0 : int bf_program_get_counter(const struct bf_program *program,
1082 : uint32_t counter_idx, struct bf_counter *counter)
1083 : {
1084 0 : bf_assert(program);
1085 0 : bf_assert(counter);
1086 :
1087 : int r;
1088 :
1089 0 : r = bf_bpf_map_lookup_elem(program->cmap->fd, &counter_idx, counter);
1090 0 : if (r < 0)
1091 0 : return bf_err_r(errno, "failed to lookup counters map");
1092 :
1093 : return 0;
1094 : }
1095 :
1096 0 : int bf_cgen_set_counters(struct bf_program *program,
1097 : const struct bf_counter *counters)
1098 : {
1099 : UNUSED(program);
1100 : UNUSED(counters);
1101 :
1102 0 : return -ENOTSUP;
1103 : }
1104 :
1105 0 : int bf_program_pin(struct bf_program *prog, int dir_fd)
1106 : {
1107 : const char *name;
1108 : int r;
1109 :
1110 0 : bf_assert(prog);
1111 :
1112 0 : name = prog->runtime.chain->name;
1113 :
1114 0 : r = bf_bpf_obj_pin(prog->prog_name, prog->runtime.prog_fd, dir_fd);
1115 0 : if (r) {
1116 0 : bf_err_r(r, "failed to pin BPF program for '%s'", name);
1117 0 : goto err_unpin_all;
1118 : }
1119 :
1120 0 : r = bf_map_pin(prog->cmap, dir_fd);
1121 0 : if (r) {
1122 0 : bf_err_r(r, "failed to pin BPF counters map for '%s'", name);
1123 0 : goto err_unpin_all;
1124 : }
1125 :
1126 0 : r = bf_map_pin(prog->pmap, dir_fd);
1127 0 : if (r) {
1128 0 : bf_err_r(r, "failed to pin BPF printer map for '%s'", name);
1129 0 : goto err_unpin_all;
1130 : }
1131 :
1132 0 : bf_list_foreach (&prog->sets, set_node) {
1133 0 : r = bf_map_pin(bf_list_node_get_data(set_node), dir_fd);
1134 0 : if (r) {
1135 0 : bf_err_r(r, "failed to pin BPF set map for '%s'", name);
1136 0 : goto err_unpin_all;
1137 : }
1138 : }
1139 :
1140 : // If a link exists, pin it too.
1141 0 : if (prog->link->hookopts) {
1142 0 : r = bf_link_pin(prog->link, dir_fd);
1143 0 : if (r) {
1144 0 : bf_err_r(r, "failed to pin BPF link for '%s'", name);
1145 0 : goto err_unpin_all;
1146 : }
1147 : }
1148 :
1149 : return 0;
1150 :
1151 0 : err_unpin_all:
1152 0 : bf_program_unpin(prog, dir_fd);
1153 0 : return r;
1154 : }
1155 :
1156 0 : void bf_program_unpin(struct bf_program *prog, int dir_fd)
1157 : {
1158 0 : bf_assert(prog);
1159 :
1160 0 : bf_map_unpin(prog->cmap, dir_fd);
1161 0 : bf_map_unpin(prog->pmap, dir_fd);
1162 :
1163 0 : bf_list_foreach (&prog->sets, set_node)
1164 0 : bf_map_unpin(bf_list_node_get_data(set_node), dir_fd);
1165 :
1166 0 : bf_link_unpin(prog->link, dir_fd);
1167 :
1168 0 : unlinkat(dir_fd, prog->prog_name, 0);
1169 0 : }
1170 :
1171 0 : size_t bf_program_chain_counter_idx(const struct bf_program *program)
1172 : {
1173 0 : return bf_list_size(&program->runtime.chain->rules);
1174 : }
1175 :
1176 0 : size_t bf_program_error_counter_idx(const struct bf_program *program)
1177 : {
1178 0 : return bf_list_size(&program->runtime.chain->rules) + 1;
1179 : }
|
