Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0-only */
2 : /*
3 : * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
4 : */
5 :
6 : #include "cgen/program.h"
7 :
8 : #include <linux/bpf.h>
9 : #include <linux/bpf_common.h>
10 : #include <linux/limits.h>
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <stddef.h>
16 : #include <stdint.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <string.h>
20 : #include <unistd.h>
21 :
22 : #include <bpfilter/bpf.h>
23 : #include <bpfilter/btf.h>
24 : #include <bpfilter/chain.h>
25 : #include <bpfilter/counter.h>
26 : #include <bpfilter/dump.h>
27 : #include <bpfilter/flavor.h>
28 : #include <bpfilter/helper.h>
29 : #include <bpfilter/hook.h>
30 : #include <bpfilter/io.h>
31 : #include <bpfilter/list.h>
32 : #include <bpfilter/logger.h>
33 : #include <bpfilter/matcher.h>
34 : #include <bpfilter/pack.h>
35 : #include <bpfilter/rule.h>
36 : #include <bpfilter/set.h>
37 : #include <bpfilter/verdict.h>
38 :
39 : #include "cgen/cgroup.h"
40 : #include "cgen/dump.h"
41 : #include "cgen/fixup.h"
42 : #include "cgen/jmp.h"
43 : #include "cgen/matcher/icmp.h"
44 : #include "cgen/matcher/ip4.h"
45 : #include "cgen/matcher/ip6.h"
46 : #include "cgen/matcher/meta.h"
47 : #include "cgen/matcher/set.h"
48 : #include "cgen/matcher/tcp.h"
49 : #include "cgen/matcher/udp.h"
50 : #include "cgen/nf.h"
51 : #include "cgen/printer.h"
52 : #include "cgen/prog/link.h"
53 : #include "cgen/prog/map.h"
54 : #include "cgen/stub.h"
55 : #include "cgen/tc.h"
56 : #include "cgen/xdp.h"
57 : #include "ctx.h"
58 : #include "filter.h"
59 : #include "opts.h"
60 :
61 : #define _BF_LOG_BUF_SIZE \
62 : (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */
63 : #define _BF_PROGRAM_DEFAULT_IMG_SIZE (1 << 6)
64 : #define _BF_LOG_MAP_N_ENTRIES 1000
65 : #define _BF_LOG_MAP_SIZE \
66 : _bf_round_next_power_of_2(sizeof(struct bf_log) * _BF_LOG_MAP_N_ENTRIES)
67 :
68 0 : static inline size_t _bf_round_next_power_of_2(size_t value)
69 : {
70 0 : value--;
71 0 : value |= value >> 1;
72 0 : value |= value >> 2;
73 0 : value |= value >> 4;
74 0 : value |= value >> 8;
75 0 : value |= value >> 16;
76 :
77 0 : return ++value;
78 : }
79 :
80 7 : static const struct bf_flavor_ops *bf_flavor_ops_get(enum bf_flavor flavor)
81 : {
82 : static const struct bf_flavor_ops *flavor_ops[] = {
83 : [BF_FLAVOR_TC] = &bf_flavor_ops_tc,
84 : [BF_FLAVOR_NF] = &bf_flavor_ops_nf,
85 : [BF_FLAVOR_XDP] = &bf_flavor_ops_xdp,
86 : [BF_FLAVOR_CGROUP] = &bf_flavor_ops_cgroup,
87 : };
88 :
89 : static_assert(ARRAY_SIZE(flavor_ops) == _BF_FLAVOR_MAX,
90 : "missing entries in bf_flavor_ops array");
91 :
92 7 : return flavor_ops[flavor];
93 : }
94 :
95 3 : int bf_program_new(struct bf_program **program, const struct bf_chain *chain)
96 : {
97 3 : _free_bf_program_ struct bf_program *_program = NULL;
98 : char name[BPF_OBJ_NAME_LEN];
99 : uint32_t set_idx = 0;
100 : int r;
101 :
102 3 : bf_assert(program && chain);
103 :
104 3 : _program = calloc(1, sizeof(*_program));
105 3 : if (!_program)
106 : return -ENOMEM;
107 :
108 3 : _program->flavor = bf_hook_to_flavor(chain->hook);
109 3 : _program->runtime.prog_fd = -1;
110 3 : _program->runtime.ops = bf_flavor_ops_get(_program->flavor);
111 3 : _program->runtime.chain = chain;
112 :
113 3 : (void)snprintf(_program->prog_name, BPF_OBJ_NAME_LEN, "%s", "bf_prog");
114 :
115 3 : r = bf_map_new(&_program->cmap, "counters_map", BF_MAP_TYPE_COUNTERS,
116 : sizeof(uint32_t), sizeof(struct bf_counter), 1);
117 3 : if (r < 0)
118 0 : return bf_err_r(r, "failed to create the counters bf_map object");
119 :
120 3 : r = bf_map_new(&_program->pmap, "printer_map", BF_MAP_TYPE_PRINTER,
121 : sizeof(uint32_t), BF_MAP_VALUE_SIZE_UNKNOWN, 1);
122 3 : if (r < 0)
123 0 : return bf_err_r(r, "failed to create the printer bf_map object");
124 :
125 3 : r = bf_map_new(&_program->lmap, "log_map", BF_MAP_TYPE_LOG, 0, 0,
126 : _BF_LOG_MAP_SIZE);
127 3 : if (r < 0)
128 0 : return bf_err_r(r, "failed to create the log bf_map object");
129 :
130 3 : _program->sets = bf_list_default(bf_map_free, bf_map_pack);
131 6 : bf_list_foreach (&chain->sets, set_node) {
132 0 : struct bf_set *set = bf_list_node_get_data(set_node);
133 0 : _free_bf_map_ struct bf_map *map = NULL;
134 :
135 0 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "set_%04x", (uint8_t)set_idx++);
136 0 : r = bf_map_new_from_set(&map, name, set);
137 0 : if (r < 0)
138 : return r;
139 :
140 0 : r = bf_list_add_tail(&_program->sets, map);
141 0 : if (r < 0)
142 : return r;
143 0 : TAKE_PTR(map);
144 : };
145 :
146 3 : r = bf_link_new(&_program->link, "bf_link");
147 3 : if (r)
148 : return r;
149 :
150 3 : r = bf_printer_new(&_program->printer);
151 3 : if (r)
152 : return r;
153 :
154 3 : bf_list_init(&_program->fixups,
155 3 : (bf_list_ops[]) {{.free = (bf_list_ops_free)bf_fixup_free}});
156 :
157 3 : *program = TAKE_PTR(_program);
158 :
159 3 : return 0;
160 : }
161 :
162 0 : int bf_program_new_from_pack(struct bf_program **program,
163 : const struct bf_chain *chain, int dir_fd,
164 : bf_rpack_node_t node)
165 : {
166 0 : _free_bf_program_ struct bf_program *_program = NULL;
167 0 : _free_bf_link_ struct bf_link *link = NULL;
168 : const void *img;
169 : size_t img_len;
170 : bf_rpack_node_t child, array_node;
171 : int r;
172 :
173 0 : bf_assert(program);
174 0 : bf_assert(chain);
175 :
176 0 : r = bf_program_new(&_program, chain);
177 0 : if (r < 0)
178 : return r;
179 :
180 0 : bf_map_free(&_program->cmap);
181 0 : r = bf_rpack_kv_obj(node, "cmap", &child);
182 0 : if (r)
183 0 : return bf_rpack_key_err(r, "bf_program.cmap");
184 0 : r = bf_map_new_from_pack(&_program->cmap, dir_fd, child);
185 0 : if (r)
186 : return r;
187 :
188 0 : bf_map_free(&_program->pmap);
189 0 : r = bf_rpack_kv_obj(node, "pmap", &child);
190 0 : if (r)
191 0 : return bf_rpack_key_err(r, "bf_program.pmap");
192 0 : r = bf_map_new_from_pack(&_program->pmap, dir_fd, child);
193 0 : if (r)
194 : return r;
195 :
196 0 : bf_map_free(&_program->lmap);
197 0 : r = bf_rpack_kv_obj(node, "lmap", &child);
198 0 : if (r)
199 0 : return bf_rpack_key_err(r, "bf_program.lmap");
200 0 : r = bf_map_new_from_pack(&_program->lmap, dir_fd, child);
201 0 : if (r)
202 : return r;
203 :
204 0 : bf_list_clean(&_program->sets);
205 0 : _program->sets = bf_list_default(bf_map_free, bf_map_pack);
206 0 : r = bf_rpack_kv_array(node, "sets", &child);
207 0 : if (r)
208 0 : return bf_rpack_key_err(r, "bf_program.sets");
209 0 : bf_rpack_array_foreach (child, array_node) {
210 0 : _free_bf_map_ struct bf_map *map = NULL;
211 :
212 0 : r = bf_list_emplace(&_program->sets, bf_map_new_from_pack, map, dir_fd,
213 : array_node);
214 : if (r)
215 0 : return bf_err_r(r, "failed to unpack bf_map into bf_program.sets");
216 : }
217 :
218 : /* Try to restore the link: on success, replace the program's link with the
219 : * restored on. If -ENOENT is returned, the link doesn't exist, meaning the
220 : * program is not attached. Otherwise, return an error. */
221 0 : r = bf_rpack_kv_obj(node, "link", &child);
222 0 : if (r)
223 0 : return bf_rpack_key_err(r, "bf_program.link");
224 0 : r = bf_link_new_from_pack(&link, dir_fd, child);
225 0 : if (!r)
226 0 : bf_swap(_program->link, link);
227 0 : else if (r != -ENOENT)
228 0 : return bf_err_r(r, "failed to restore bf_program.link");
229 :
230 0 : bf_printer_free(&_program->printer);
231 0 : r = bf_rpack_kv_obj(node, "printer", &child);
232 0 : if (r)
233 0 : return bf_rpack_key_err(r, "bf_program.printer");
234 0 : r = bf_printer_new_from_pack(&_program->printer, child);
235 0 : if (r)
236 : return r;
237 :
238 0 : r = bf_rpack_kv_bin(node, "img", &img, &img_len);
239 0 : if (r)
240 0 : return bf_rpack_key_err(r, "bf_program.img");
241 0 : _program->img = bf_memdup(img, img_len);
242 0 : if (!_program->img)
243 0 : return bf_rpack_key_err(-ENOMEM, "bf_program.img");
244 0 : _program->img_size = img_len / sizeof(struct bpf_insn);
245 0 : _program->img_cap = _program->img_size;
246 :
247 0 : r = bf_bpf_obj_get(_program->prog_name, dir_fd, &_program->runtime.prog_fd);
248 0 : if (r < 0)
249 0 : return bf_err_r(r, "failed to restore bf_program.fd");
250 :
251 0 : *program = TAKE_PTR(_program);
252 :
253 0 : return 0;
254 : }
255 :
256 14 : void bf_program_free(struct bf_program **program)
257 : {
258 14 : if (!*program)
259 : return;
260 :
261 3 : bf_list_clean(&(*program)->fixups);
262 3 : free((*program)->img);
263 :
264 : /* Close the file descriptors if they are still open. If --transient is
265 : * used, then the file descriptors are already closed (as
266 : * bf_program_unload() has been called). Otherwise, bf_program_unload()
267 : * won't be called, but the programs are pinned, so they can be closed
268 : * safely. */
269 3 : closep(&(*program)->runtime.prog_fd);
270 :
271 3 : bf_map_free(&(*program)->cmap);
272 3 : bf_map_free(&(*program)->pmap);
273 3 : bf_map_free(&(*program)->lmap);
274 3 : bf_list_clean(&(*program)->sets);
275 3 : bf_link_free(&(*program)->link);
276 3 : bf_printer_free(&(*program)->printer);
277 :
278 3 : free(*program);
279 3 : *program = NULL;
280 : }
281 :
282 0 : int bf_program_pack(const struct bf_program *program, bf_wpack_t *pack)
283 : {
284 0 : bf_assert(program);
285 0 : bf_assert(pack);
286 :
287 0 : bf_wpack_open_object(pack, "cmap");
288 0 : bf_map_pack(program->cmap, pack);
289 0 : bf_wpack_close_object(pack);
290 :
291 0 : bf_wpack_open_object(pack, "pmap");
292 0 : bf_map_pack(program->pmap, pack);
293 0 : bf_wpack_close_object(pack);
294 :
295 0 : bf_wpack_open_object(pack, "lmap");
296 0 : bf_map_pack(program->lmap, pack);
297 0 : bf_wpack_close_object(pack);
298 :
299 0 : bf_wpack_kv_list(pack, "sets", &program->sets);
300 :
301 0 : bf_wpack_open_object(pack, "link");
302 0 : bf_link_pack(program->link, pack);
303 0 : bf_wpack_close_object(pack);
304 :
305 0 : bf_wpack_open_object(pack, "printer");
306 0 : bf_printer_pack(program->printer, pack);
307 0 : bf_wpack_close_object(pack);
308 :
309 0 : bf_wpack_kv_bin(pack, "img", program->img,
310 0 : program->img_size * sizeof(struct bpf_insn));
311 :
312 0 : return bf_wpack_is_valid(pack) ? 0 : -EINVAL;
313 : }
314 :
315 0 : void bf_program_dump(const struct bf_program *program, prefix_t *prefix)
316 : {
317 0 : bf_assert(program);
318 0 : bf_assert(prefix);
319 :
320 0 : DUMP(prefix, "struct bf_program at %p", program);
321 :
322 0 : bf_dump_prefix_push(prefix);
323 :
324 0 : DUMP(prefix, "prog_name: %s", program->prog_name);
325 :
326 0 : DUMP(prefix, "cmap: struct bf_map *");
327 0 : bf_dump_prefix_push(prefix);
328 0 : bf_map_dump(program->cmap, bf_dump_prefix_last(prefix));
329 0 : bf_dump_prefix_pop(prefix);
330 :
331 0 : DUMP(prefix, "pmap: struct bf_map *");
332 0 : bf_dump_prefix_push(prefix);
333 0 : bf_map_dump(program->pmap, bf_dump_prefix_last(prefix));
334 0 : bf_dump_prefix_pop(prefix);
335 :
336 0 : DUMP(prefix, "lmap: struct bf_map *");
337 0 : bf_dump_prefix_push(prefix);
338 0 : bf_map_dump(program->lmap, bf_dump_prefix_last(prefix));
339 0 : bf_dump_prefix_pop(prefix);
340 :
341 0 : DUMP(prefix, "sets: bf_list<bf_map>[%lu]", bf_list_size(&program->sets));
342 0 : bf_dump_prefix_push(prefix);
343 0 : bf_list_foreach (&program->sets, map_node) {
344 0 : struct bf_map *map = bf_list_node_get_data(map_node);
345 :
346 0 : if (bf_list_is_tail(&program->sets, map_node))
347 0 : bf_dump_prefix_last(prefix);
348 :
349 0 : bf_map_dump(map, prefix);
350 : }
351 0 : bf_dump_prefix_pop(prefix);
352 :
353 0 : DUMP(prefix, "link: struct bf_link *");
354 0 : bf_dump_prefix_push(prefix);
355 0 : bf_link_dump(program->link, prefix);
356 0 : bf_dump_prefix_pop(prefix);
357 :
358 0 : DUMP(prefix, "printer: struct bf_printer *");
359 0 : bf_dump_prefix_push(prefix);
360 0 : bf_printer_dump(program->printer, prefix);
361 0 : bf_dump_prefix_pop(prefix);
362 :
363 0 : DUMP(prefix, "img: %p", program->img);
364 0 : DUMP(prefix, "img_size: %lu", program->img_size);
365 0 : DUMP(prefix, "img_cap: %lu", program->img_cap);
366 :
367 0 : DUMP(prefix, "fixups: bf_list<struct bf_fixup>[%lu]",
368 : bf_list_size(&program->fixups));
369 0 : bf_dump_prefix_push(prefix);
370 0 : bf_list_foreach (&program->fixups, fixup_node) {
371 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
372 :
373 0 : if (bf_list_is_tail(&program->fixups, fixup_node))
374 0 : bf_dump_prefix_last(prefix);
375 :
376 0 : bf_fixup_dump(fixup, prefix);
377 : }
378 0 : bf_dump_prefix_pop(prefix);
379 :
380 0 : DUMP(bf_dump_prefix_last(prefix), "runtime: <anonymous>");
381 0 : bf_dump_prefix_push(prefix);
382 0 : DUMP(prefix, "prog_fd: %d", program->runtime.prog_fd);
383 0 : DUMP(bf_dump_prefix_last(prefix), "ops: %p", program->runtime.ops);
384 0 : bf_dump_prefix_pop(prefix);
385 :
386 0 : bf_dump_prefix_pop(prefix);
387 0 : }
388 :
389 3 : int bf_program_grow_img(struct bf_program *program)
390 : {
391 : size_t new_cap = _BF_PROGRAM_DEFAULT_IMG_SIZE;
392 : int r;
393 :
394 3 : bf_assert(program);
395 :
396 3 : if (program->img)
397 0 : new_cap = _bf_round_next_power_of_2(program->img_cap << 1);
398 :
399 3 : r = bf_realloc((void **)&program->img, new_cap * sizeof(struct bpf_insn));
400 3 : if (r < 0) {
401 0 : return bf_err_r(r, "failed to grow program img from %lu to %lu insn",
402 : program->img_cap, new_cap);
403 : }
404 :
405 3 : program->img_cap = new_cap;
406 :
407 3 : return 0;
408 : }
409 :
410 0 : static void _bf_program_fixup_insn(struct bpf_insn *insn,
411 : enum bf_fixup_insn type, int32_t value)
412 : {
413 0 : switch (type) {
414 0 : case BF_FIXUP_INSN_OFF:
415 0 : bf_assert(!insn->off);
416 0 : bf_assert(value < SHRT_MAX);
417 0 : insn->off = (int16_t)value;
418 0 : break;
419 0 : case BF_FIXUP_INSN_IMM:
420 0 : bf_assert(!insn->imm);
421 0 : insn->imm = value;
422 0 : break;
423 0 : default:
424 0 : bf_abort(
425 : "unsupported fixup instruction type, this should not happen: %d",
426 : type);
427 : break;
428 : }
429 0 : }
430 :
431 0 : static int _bf_program_fixup(struct bf_program *program,
432 : enum bf_fixup_type type)
433 : {
434 0 : bf_assert(program);
435 0 : bf_assert(type >= 0 && type < _BF_FIXUP_TYPE_MAX);
436 :
437 0 : bf_list_foreach (&program->fixups, fixup_node) {
438 : enum bf_fixup_insn insn_type = _BF_FIXUP_INSN_MAX;
439 : int32_t value;
440 : size_t offset;
441 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
442 0 : struct bpf_insn *insn = &program->img[fixup->insn];
443 : struct bf_map *map;
444 :
445 0 : if (type != fixup->type)
446 0 : continue;
447 :
448 0 : switch (type) {
449 0 : case BF_FIXUP_TYPE_JMP_NEXT_RULE:
450 : insn_type = BF_FIXUP_INSN_OFF;
451 0 : value = (int)(program->img_size - fixup->insn - 1U);
452 0 : break;
453 0 : case BF_FIXUP_TYPE_COUNTERS_MAP_FD:
454 : insn_type = BF_FIXUP_INSN_IMM;
455 0 : value = program->cmap->fd;
456 0 : break;
457 0 : case BF_FIXUP_TYPE_PRINTER_MAP_FD:
458 : insn_type = BF_FIXUP_INSN_IMM;
459 0 : value = program->pmap->fd;
460 0 : break;
461 0 : case BF_FIXUP_TYPE_LOG_MAP_FD:
462 : insn_type = BF_FIXUP_INSN_IMM;
463 0 : value = program->lmap->fd;
464 0 : break;
465 0 : case BF_FIXUP_TYPE_SET_MAP_FD:
466 0 : map = bf_list_get_at(&program->sets, fixup->attr.set_index);
467 0 : if (!map) {
468 0 : return bf_err_r(-ENOENT, "can't find set map at index %lu",
469 : fixup->attr.set_index);
470 : }
471 : insn_type = BF_FIXUP_INSN_IMM;
472 0 : value = map->fd;
473 0 : break;
474 0 : case BF_FIXUP_ELFSTUB_CALL:
475 : insn_type = BF_FIXUP_INSN_IMM;
476 0 : offset = program->elfstubs_location[fixup->attr.elfstub_id] -
477 : fixup->insn - 1;
478 0 : if (offset >= INT_MAX)
479 0 : return bf_err_r(-EINVAL, "invalid ELF stub call offset");
480 0 : value = (int32_t)offset;
481 0 : break;
482 0 : default:
483 0 : bf_abort("unsupported fixup type, this should not happen: %d",
484 : type);
485 : break;
486 : }
487 :
488 0 : _bf_program_fixup_insn(insn, insn_type, value);
489 0 : bf_list_delete(&program->fixups, fixup_node);
490 : }
491 :
492 : return 0;
493 : }
494 :
495 0 : static int _bf_program_generate_rule(struct bf_program *program,
496 : struct bf_rule *rule)
497 : {
498 : int r;
499 :
500 0 : bf_assert(program);
501 0 : bf_assert(rule);
502 :
503 0 : bf_list_foreach (&rule->matchers, matcher_node) {
504 0 : struct bf_matcher *matcher = bf_list_node_get_data(matcher_node);
505 :
506 0 : switch (bf_matcher_get_type(matcher)) {
507 0 : case BF_MATCHER_META_IFACE:
508 : case BF_MATCHER_META_L3_PROTO:
509 : case BF_MATCHER_META_L4_PROTO:
510 : case BF_MATCHER_META_PROBABILITY:
511 : case BF_MATCHER_META_SPORT:
512 : case BF_MATCHER_META_DPORT:
513 : case BF_MATCHER_META_MARK:
514 0 : r = bf_matcher_generate_meta(program, matcher);
515 0 : if (r)
516 : return r;
517 : break;
518 0 : case BF_MATCHER_IP4_SADDR:
519 : case BF_MATCHER_IP4_SNET:
520 : case BF_MATCHER_IP4_DADDR:
521 : case BF_MATCHER_IP4_DNET:
522 : case BF_MATCHER_IP4_PROTO:
523 0 : r = bf_matcher_generate_ip4(program, matcher);
524 0 : if (r)
525 : return r;
526 : break;
527 0 : case BF_MATCHER_IP6_SADDR:
528 : case BF_MATCHER_IP6_SNET:
529 : case BF_MATCHER_IP6_DADDR:
530 : case BF_MATCHER_IP6_DNET:
531 : case BF_MATCHER_IP6_NEXTHDR:
532 0 : r = bf_matcher_generate_ip6(program, matcher);
533 0 : if (r)
534 : return r;
535 : break;
536 0 : case BF_MATCHER_TCP_SPORT:
537 : case BF_MATCHER_TCP_DPORT:
538 : case BF_MATCHER_TCP_FLAGS:
539 0 : r = bf_matcher_generate_tcp(program, matcher);
540 0 : if (r)
541 : return r;
542 : break;
543 0 : case BF_MATCHER_UDP_SPORT:
544 : case BF_MATCHER_UDP_DPORT:
545 0 : r = bf_matcher_generate_udp(program, matcher);
546 0 : if (r)
547 : return r;
548 : break;
549 0 : case BF_MATCHER_ICMP_TYPE:
550 : case BF_MATCHER_ICMP_CODE:
551 : case BF_MATCHER_ICMPV6_TYPE:
552 : case BF_MATCHER_ICMPV6_CODE:
553 0 : r = bf_matcher_generate_icmp(program, matcher);
554 0 : if (r)
555 : return r;
556 : break;
557 0 : case BF_MATCHER_SET:
558 0 : r = bf_matcher_generate_set(program, matcher);
559 0 : if (r)
560 : return r;
561 : break;
562 0 : default:
563 0 : return bf_err_r(-EINVAL, "unknown matcher type %d",
564 : bf_matcher_get_type(matcher));
565 : };
566 : }
567 :
568 0 : if (bf_rule_mark_is_set(rule)) {
569 0 : if (!program->runtime.ops->gen_inline_set_mark) {
570 0 : return bf_err_r(-ENOTSUP, "set mark is not supported by %s",
571 : program->runtime.chain->name);
572 : }
573 :
574 0 : r = program->runtime.ops->gen_inline_set_mark(program,
575 : bf_rule_mark_get(rule));
576 0 : if (r) {
577 0 : return bf_err_r(r,
578 : "failed to generate bytecode to set mark for '%s'",
579 : program->runtime.chain->name);
580 : }
581 : }
582 :
583 0 : if (rule->log) {
584 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
585 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
586 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, rule->index));
587 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_3, rule->log));
588 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_4, BPF_REG_7));
589 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_5, BPF_REG_8));
590 :
591 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_LOG);
592 : }
593 :
594 0 : if (rule->counters) {
595 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
596 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
597 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
598 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_3, rule->index));
599 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
600 : }
601 :
602 0 : switch (rule->verdict) {
603 0 : case BF_VERDICT_ACCEPT:
604 : case BF_VERDICT_DROP:
605 0 : EMIT(program,
606 : BPF_MOV64_IMM(BPF_REG_0,
607 : program->runtime.ops->get_verdict(rule->verdict)));
608 0 : EMIT(program, BPF_EXIT_INSN());
609 0 : break;
610 : case BF_VERDICT_CONTINUE:
611 : // Fall through to next rule or default chain policy.
612 : break;
613 0 : default:
614 0 : bf_abort("unsupported verdict, this should not happen: %d",
615 : rule->verdict);
616 : break;
617 : }
618 :
619 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_JMP_NEXT_RULE);
620 0 : if (r)
621 0 : return bf_err_r(r, "failed to generate next rule fixups");
622 :
623 : return 0;
624 : }
625 :
626 0 : static int _bf_program_generate_elfstubs(struct bf_program *program)
627 : {
628 : const struct bf_elfstub *elfstub;
629 : size_t start_at;
630 : int r;
631 :
632 0 : bf_assert(program);
633 :
634 0 : bf_list_foreach (&program->fixups, fixup_node) {
635 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
636 0 : size_t off = program->img_size;
637 :
638 0 : if (fixup->type != BF_FIXUP_ELFSTUB_CALL)
639 0 : continue;
640 :
641 : // Only generate each ELF stub once
642 0 : if (program->elfstubs_location[fixup->attr.elfstub_id])
643 0 : continue;
644 :
645 0 : bf_dbg("generate ELF stub for ID %d", fixup->attr.elfstub_id);
646 :
647 0 : elfstub = bf_ctx_get_elfstub(fixup->attr.elfstub_id);
648 0 : if (!elfstub) {
649 0 : return bf_err_r(-ENOENT, "no ELF stub found for ID %d",
650 : fixup->attr.elfstub_id);
651 : }
652 :
653 0 : start_at = program->img_size;
654 :
655 0 : for (size_t i = 0; i < elfstub->ninsns; ++i) {
656 0 : r = bf_program_emit(program, elfstub->insns[i]);
657 0 : if (r)
658 0 : return bf_err_r(r, "failed to insert ELF stub instruction");
659 : }
660 :
661 0 : bf_list_foreach (&elfstub->strs, pstr_node) {
662 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
663 0 : struct bf_printk_str *pstr = bf_list_node_get_data(pstr_node);
664 0 : size_t insn_idx = start_at + pstr->insn_idx;
665 : const struct bf_printer_msg *msg =
666 0 : bf_printer_add_msg(program->printer, pstr->str);
667 : struct bpf_insn ld_insn[2] = {
668 : BPF_LD_MAP_FD(BPF_REG_1, 0),
669 : };
670 :
671 : ld_insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
672 0 : ld_insn[1].imm = (int)bf_printer_msg_offset(msg);
673 :
674 0 : program->img[insn_idx] = ld_insn[0];
675 0 : program->img[insn_idx + 1] = ld_insn[1];
676 :
677 0 : r = bf_fixup_new(&fixup, BF_FIXUP_TYPE_PRINTER_MAP_FD, insn_idx,
678 : NULL);
679 0 : if (r)
680 : return r;
681 :
682 0 : r = bf_list_add_tail(&program->fixups, fixup);
683 0 : if (r)
684 : return r;
685 :
686 0 : TAKE_PTR(fixup);
687 : }
688 :
689 0 : program->elfstubs_location[fixup->attr.elfstub_id] = off;
690 : }
691 :
692 : return 0;
693 : }
694 :
695 40 : int bf_program_emit(struct bf_program *program, struct bpf_insn insn)
696 : {
697 : int r;
698 :
699 40 : bf_assert(program);
700 :
701 40 : if (program->img_size == program->img_cap) {
702 3 : r = bf_program_grow_img(program);
703 3 : if (r)
704 : return r;
705 : }
706 :
707 40 : program->img[program->img_size++] = insn;
708 :
709 40 : return 0;
710 : }
711 :
712 0 : int bf_program_emit_kfunc_call(struct bf_program *program, const char *name)
713 : {
714 : int r;
715 :
716 0 : bf_assert(program);
717 0 : bf_assert(name);
718 :
719 0 : r = bf_btf_get_id(name);
720 0 : if (r < 0)
721 : return r;
722 :
723 0 : EMIT(program, ((struct bpf_insn) {.code = BPF_JMP | BPF_CALL,
724 : .dst_reg = 0,
725 : .src_reg = BPF_PSEUDO_KFUNC_CALL,
726 : .off = 0,
727 : .imm = r}));
728 :
729 0 : return 0;
730 : }
731 :
732 0 : int bf_program_emit_fixup(struct bf_program *program, enum bf_fixup_type type,
733 : struct bpf_insn insn, const union bf_fixup_attr *attr)
734 : {
735 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
736 : int r;
737 :
738 0 : bf_assert(program);
739 :
740 0 : if (program->img_size == program->img_cap) {
741 0 : r = bf_program_grow_img(program);
742 0 : if (r)
743 : return r;
744 : }
745 :
746 0 : r = bf_fixup_new(&fixup, type, program->img_size, attr);
747 0 : if (r)
748 : return r;
749 :
750 0 : r = bf_list_add_tail(&program->fixups, fixup);
751 0 : if (r)
752 : return r;
753 :
754 0 : TAKE_PTR(fixup);
755 :
756 : /* This call could fail and return an error, in which case it is not
757 : * properly handled. However, this shouldn't be an issue as we previously
758 : * test whether enough room is available in cgen.img, which is currently
759 : * the only reason for EMIT() to fail. */
760 0 : EMIT(program, insn);
761 :
762 : return 0;
763 : }
764 :
765 0 : int bf_program_emit_fixup_elfstub(struct bf_program *program,
766 : enum bf_elfstub_id id)
767 : {
768 0 : _free_bf_fixup_ struct bf_fixup *fixup = NULL;
769 : int r;
770 :
771 0 : bf_assert(program);
772 :
773 0 : if (program->img_size == program->img_cap) {
774 0 : r = bf_program_grow_img(program);
775 0 : if (r)
776 : return r;
777 : }
778 :
779 0 : r = bf_fixup_new(&fixup, BF_FIXUP_ELFSTUB_CALL, program->img_size, NULL);
780 0 : if (r)
781 : return r;
782 :
783 0 : fixup->attr.elfstub_id = id;
784 :
785 0 : r = bf_list_add_tail(&program->fixups, fixup);
786 0 : if (r)
787 : return r;
788 :
789 0 : TAKE_PTR(fixup);
790 :
791 0 : EMIT(program, BPF_CALL_REL(0));
792 :
793 0 : return 0;
794 : }
795 :
796 0 : int bf_program_generate(struct bf_program *program)
797 : {
798 0 : const struct bf_chain *chain = program->runtime.chain;
799 : int r;
800 :
801 : // Save the program's argument into the context.
802 0 : EMIT(program,
803 : BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
804 :
805 : // Reset the protocol ID registers
806 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, 0));
807 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_8, 0));
808 :
809 : // If at least one rule logs the matched packets, populate ctx->log_map
810 0 : if (program->runtime.chain->flags & BF_FLAG(BF_CHAIN_LOG)) {
811 0 : EMIT_LOAD_LOG_FD_FIXUP(program, BPF_REG_2);
812 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2,
813 : BF_PROG_CTX_OFF(log_map)));
814 : }
815 :
816 : // Zeroing IPv6 extension headers
817 0 : if (program->runtime.chain->flags & BF_FLAG(BF_CHAIN_STORE_NEXTHDR)) {
818 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7,
819 : BF_PROG_CTX_OFF(ipv6_eh)));
820 : }
821 :
822 0 : r = program->runtime.ops->gen_inline_prologue(program);
823 0 : if (r)
824 : return r;
825 :
826 0 : bf_list_foreach (&chain->rules, rule_node) {
827 0 : r = _bf_program_generate_rule(program,
828 0 : bf_list_node_get_data(rule_node));
829 0 : if (r)
830 : return r;
831 : }
832 :
833 0 : r = program->runtime.ops->gen_inline_epilogue(program);
834 0 : if (r)
835 : return r;
836 :
837 : // Call the update counters function
838 : /// @todo Allow chains to have no counters at all.
839 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
840 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
841 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_2);
842 0 : EMIT(program,
843 : BPF_MOV32_IMM(BPF_REG_3, bf_program_chain_counter_idx(program)));
844 0 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_UPDATE_COUNTERS);
845 :
846 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_0, program->runtime.ops->get_verdict(
847 : chain->policy)));
848 0 : EMIT(program, BPF_EXIT_INSN());
849 :
850 0 : r = _bf_program_generate_elfstubs(program);
851 0 : if (r)
852 : return r;
853 :
854 0 : r = _bf_program_fixup(program, BF_FIXUP_ELFSTUB_CALL);
855 0 : if (r)
856 0 : return bf_err_r(r, "failed to generate ELF stub call fixups");
857 :
858 : return 0;
859 : }
860 :
861 0 : static int _bf_program_load_printer_map(struct bf_program *program)
862 : {
863 0 : _cleanup_free_ void *pstr = NULL;
864 : size_t pstr_len;
865 0 : uint32_t key = 0;
866 : int r;
867 :
868 0 : bf_assert(program);
869 :
870 0 : r = bf_printer_assemble(program->printer, &pstr, &pstr_len);
871 0 : if (r)
872 0 : return bf_err_r(r, "failed to assemble printer map string");
873 :
874 0 : r = bf_map_set_value_size(program->pmap, pstr_len);
875 0 : if (r < 0)
876 : return r;
877 :
878 0 : r = bf_map_create(program->pmap);
879 0 : if (r < 0)
880 : return r;
881 :
882 0 : r = bf_map_set_elem(program->pmap, &key, pstr);
883 0 : if (r)
884 : return r;
885 :
886 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_PRINTER_MAP_FD);
887 0 : if (r) {
888 0 : bf_map_destroy(program->pmap);
889 0 : return bf_err_r(r, "failed to fixup printer map FD");
890 : }
891 :
892 : return 0;
893 : }
894 :
895 0 : static int _bf_program_load_counters_map(struct bf_program *program)
896 : {
897 0 : _cleanup_close_ int _fd = -1;
898 : int r;
899 :
900 0 : bf_assert(program);
901 :
902 0 : r = bf_map_set_n_elems(program->cmap,
903 0 : bf_list_size(&program->runtime.chain->rules) + 2);
904 0 : if (r < 0)
905 : return r;
906 :
907 0 : r = bf_map_create(program->cmap);
908 0 : if (r < 0)
909 : return r;
910 :
911 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_COUNTERS_MAP_FD);
912 0 : if (r < 0) {
913 0 : bf_map_destroy(program->cmap);
914 0 : return bf_err_r(r, "failed to fixup counters map FD");
915 : }
916 :
917 : return 0;
918 : }
919 :
920 0 : static int _bf_program_load_log_map(struct bf_program *program)
921 : {
922 0 : _cleanup_close_ int _fd = -1;
923 : int r;
924 :
925 0 : bf_assert(program);
926 :
927 0 : r = bf_map_create(program->lmap);
928 0 : if (r < 0)
929 : return r;
930 :
931 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_LOG_MAP_FD);
932 0 : if (r < 0) {
933 0 : bf_map_destroy(program->lmap);
934 0 : return bf_err_r(r, "failed to fixup log map FD");
935 : }
936 :
937 : return 0;
938 : }
939 :
940 0 : static int _bf_program_load_sets_maps(struct bf_program *new_prog)
941 : {
942 : const bf_list_node *set_node;
943 : const bf_list_node *map_node;
944 : int r;
945 :
946 0 : bf_assert(new_prog);
947 :
948 0 : set_node = bf_list_get_head(&new_prog->runtime.chain->sets);
949 0 : map_node = bf_list_get_head(&new_prog->sets);
950 :
951 : // Fill the bf_map with the sets content
952 0 : while (set_node && map_node) {
953 : _cleanup_free_ uint8_t *values = NULL;
954 : _cleanup_free_ uint8_t *keys = NULL;
955 0 : struct bf_set *set = bf_list_node_get_data(set_node);
956 0 : struct bf_map *map = bf_list_node_get_data(map_node);
957 0 : size_t nelems = bf_list_size(&set->elems);
958 : size_t idx = 0;
959 :
960 0 : r = bf_map_create(map);
961 0 : if (r < 0) {
962 0 : r = bf_err_r(r, "failed to create BPF map for set");
963 0 : goto err_destroy_maps;
964 : }
965 :
966 0 : values = malloc(nelems);
967 0 : if (!values) {
968 0 : r = bf_err_r(errno, "failed to allocate map values");
969 0 : goto err_destroy_maps;
970 : }
971 :
972 0 : keys = malloc(set->elem_size * nelems);
973 0 : if (!keys) {
974 0 : r = bf_err_r(errno, "failed to allocate map keys");
975 0 : goto err_destroy_maps;
976 : }
977 :
978 0 : bf_list_foreach (&set->elems, elem_node) {
979 0 : void *elem = bf_list_node_get_data(elem_node);
980 :
981 0 : memcpy(keys + (idx * set->elem_size), elem, set->elem_size);
982 0 : values[idx] = 1;
983 0 : ++idx;
984 : }
985 :
986 0 : r = bf_bpf_map_update_batch(map->fd, keys, values, nelems, BPF_ANY);
987 0 : if (r < 0) {
988 0 : bf_err_r(r, "failed to add set elements to the map");
989 0 : goto err_destroy_maps;
990 : }
991 :
992 0 : set_node = bf_list_node_next(set_node);
993 0 : map_node = bf_list_node_next(map_node);
994 : }
995 :
996 0 : r = _bf_program_fixup(new_prog, BF_FIXUP_TYPE_SET_MAP_FD);
997 0 : if (r < 0)
998 0 : goto err_destroy_maps;
999 :
1000 : return 0;
1001 :
1002 0 : err_destroy_maps:
1003 0 : bf_list_foreach (&new_prog->sets, map_node)
1004 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1005 : return r;
1006 : }
1007 :
1008 0 : int bf_program_load(struct bf_program *prog)
1009 : {
1010 : _cleanup_free_ char *log_buf = NULL;
1011 : int r;
1012 :
1013 0 : bf_assert(prog && prog->img);
1014 :
1015 0 : r = _bf_program_load_sets_maps(prog);
1016 0 : if (r)
1017 : return r;
1018 :
1019 0 : r = _bf_program_load_counters_map(prog);
1020 0 : if (r)
1021 : return r;
1022 :
1023 0 : r = _bf_program_load_printer_map(prog);
1024 0 : if (r)
1025 : return r;
1026 :
1027 0 : r = _bf_program_load_log_map(prog);
1028 0 : if (r)
1029 : return r;
1030 :
1031 0 : if (bf_opts_is_verbose(BF_VERBOSE_DEBUG)) {
1032 0 : log_buf = malloc(_BF_LOG_BUF_SIZE);
1033 0 : if (!log_buf) {
1034 0 : return bf_err_r(-ENOMEM,
1035 : "failed to allocate BPF_PROG_LOAD logs buffer");
1036 : }
1037 : }
1038 :
1039 0 : if (bf_opts_is_verbose(BF_VERBOSE_BYTECODE))
1040 0 : bf_program_dump_bytecode(prog);
1041 :
1042 0 : r = bf_bpf_prog_load(
1043 0 : prog->prog_name, bf_hook_to_bpf_prog_type(prog->runtime.chain->hook),
1044 0 : prog->img, prog->img_size,
1045 0 : bf_hook_to_bpf_attach_type(prog->runtime.chain->hook), log_buf,
1046 : log_buf ? _BF_LOG_BUF_SIZE : 0, bf_ctx_token(), &prog->runtime.prog_fd);
1047 0 : if (r) {
1048 0 : return bf_err_r(r, "failed to load bf_program (%lu bytes):\n%s\nerrno:",
1049 : prog->img_size, log_buf ? log_buf : "<NO LOG BUFFER>");
1050 : }
1051 :
1052 : return r;
1053 : }
1054 :
1055 0 : int bf_program_attach(struct bf_program *prog, struct bf_hookopts **hookopts)
1056 : {
1057 : int r;
1058 :
1059 0 : bf_assert(prog && hookopts);
1060 :
1061 0 : r = bf_link_attach(prog->link, prog->runtime.chain->hook, hookopts,
1062 : prog->runtime.prog_fd);
1063 0 : if (r) {
1064 0 : return bf_err_r(r, "failed to attach bf_link for %s program",
1065 : bf_flavor_to_str(prog->flavor));
1066 : }
1067 :
1068 : return r;
1069 : }
1070 :
1071 0 : void bf_program_detach(struct bf_program *prog)
1072 : {
1073 0 : bf_assert(prog);
1074 :
1075 0 : bf_link_detach(prog->link);
1076 0 : }
1077 :
1078 0 : void bf_program_unload(struct bf_program *prog)
1079 : {
1080 0 : bf_assert(prog);
1081 :
1082 0 : closep(&prog->runtime.prog_fd);
1083 0 : bf_link_detach(prog->link);
1084 0 : bf_map_destroy(prog->cmap);
1085 0 : bf_map_destroy(prog->pmap);
1086 0 : bf_map_destroy(prog->lmap);
1087 0 : bf_list_foreach (&prog->sets, map_node)
1088 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1089 0 : }
1090 :
1091 0 : int bf_program_get_counter(const struct bf_program *program,
1092 : uint32_t counter_idx, struct bf_counter *counter)
1093 : {
1094 0 : bf_assert(program);
1095 0 : bf_assert(counter);
1096 :
1097 : int r;
1098 :
1099 0 : r = bf_bpf_map_lookup_elem(program->cmap->fd, &counter_idx, counter);
1100 0 : if (r < 0)
1101 0 : return bf_err_r(errno, "failed to lookup counters map");
1102 :
1103 : return 0;
1104 : }
1105 :
1106 0 : int bf_cgen_set_counters(struct bf_program *program,
1107 : const struct bf_counter *counters)
1108 : {
1109 : UNUSED(program);
1110 : UNUSED(counters);
1111 :
1112 0 : return -ENOTSUP;
1113 : }
1114 :
1115 0 : int bf_program_pin(struct bf_program *prog, int dir_fd)
1116 : {
1117 : const char *name;
1118 : int r;
1119 :
1120 0 : bf_assert(prog);
1121 :
1122 0 : name = prog->runtime.chain->name;
1123 :
1124 0 : r = bf_bpf_obj_pin(prog->prog_name, prog->runtime.prog_fd, dir_fd);
1125 0 : if (r) {
1126 0 : bf_err_r(r, "failed to pin BPF program for '%s'", name);
1127 0 : goto err_unpin_all;
1128 : }
1129 :
1130 0 : r = bf_map_pin(prog->cmap, dir_fd);
1131 0 : if (r) {
1132 0 : bf_err_r(r, "failed to pin BPF counters map for '%s'", name);
1133 0 : goto err_unpin_all;
1134 : }
1135 :
1136 0 : r = bf_map_pin(prog->pmap, dir_fd);
1137 0 : if (r) {
1138 0 : bf_err_r(r, "failed to pin BPF printer map for '%s'", name);
1139 0 : goto err_unpin_all;
1140 : }
1141 :
1142 0 : r = bf_map_pin(prog->lmap, dir_fd);
1143 0 : if (r) {
1144 0 : bf_err_r(r, "failed to pin BPF log map for '%s'", name);
1145 0 : goto err_unpin_all;
1146 : }
1147 :
1148 0 : bf_list_foreach (&prog->sets, set_node) {
1149 0 : r = bf_map_pin(bf_list_node_get_data(set_node), dir_fd);
1150 0 : if (r) {
1151 0 : bf_err_r(r, "failed to pin BPF set map for '%s'", name);
1152 0 : goto err_unpin_all;
1153 : }
1154 : }
1155 :
1156 : // If a link exists, pin it too.
1157 0 : if (prog->link->hookopts) {
1158 0 : r = bf_link_pin(prog->link, dir_fd);
1159 0 : if (r) {
1160 0 : bf_err_r(r, "failed to pin BPF link for '%s'", name);
1161 0 : goto err_unpin_all;
1162 : }
1163 : }
1164 :
1165 : return 0;
1166 :
1167 0 : err_unpin_all:
1168 0 : bf_program_unpin(prog, dir_fd);
1169 0 : return r;
1170 : }
1171 :
1172 0 : void bf_program_unpin(struct bf_program *prog, int dir_fd)
1173 : {
1174 0 : bf_assert(prog);
1175 :
1176 0 : bf_map_unpin(prog->cmap, dir_fd);
1177 0 : bf_map_unpin(prog->pmap, dir_fd);
1178 0 : bf_map_unpin(prog->lmap, dir_fd);
1179 :
1180 0 : bf_list_foreach (&prog->sets, set_node)
1181 0 : bf_map_unpin(bf_list_node_get_data(set_node), dir_fd);
1182 :
1183 0 : bf_link_unpin(prog->link, dir_fd);
1184 :
1185 0 : unlinkat(dir_fd, prog->prog_name, 0);
1186 0 : }
1187 :
1188 0 : size_t bf_program_chain_counter_idx(const struct bf_program *program)
1189 : {
1190 0 : return bf_list_size(&program->runtime.chain->rules);
1191 : }
1192 :
1193 0 : size_t bf_program_error_counter_idx(const struct bf_program *program)
1194 : {
1195 0 : return bf_list_size(&program->runtime.chain->rules) + 1;
1196 : }
|
