Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0-only */
2 : /*
3 : * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
4 : */
5 :
6 : #include "bpfilter/cgen/program.h"
7 :
8 : #include <linux/bpf.h>
9 : #include <linux/bpf_common.h>
10 : #include <linux/limits.h>
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <stddef.h>
16 : #include <stdint.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <string.h>
20 : #include <unistd.h>
21 :
22 : #include "bpfilter/cgen/cgroup.h"
23 : #include "bpfilter/cgen/dump.h"
24 : #include "bpfilter/cgen/fixup.h"
25 : #include "bpfilter/cgen/jmp.h"
26 : #include "bpfilter/cgen/matcher/ip4.h"
27 : #include "bpfilter/cgen/matcher/ip6.h"
28 : #include "bpfilter/cgen/matcher/meta.h"
29 : #include "bpfilter/cgen/matcher/set.h"
30 : #include "bpfilter/cgen/matcher/tcp.h"
31 : #include "bpfilter/cgen/matcher/udp.h"
32 : #include "bpfilter/cgen/nf.h"
33 : #include "bpfilter/cgen/printer.h"
34 : #include "bpfilter/cgen/prog/link.h"
35 : #include "bpfilter/cgen/prog/map.h"
36 : #include "bpfilter/cgen/stub.h"
37 : #include "bpfilter/cgen/tc.h"
38 : #include "bpfilter/cgen/xdp.h"
39 : #include "bpfilter/ctx.h"
40 : #include "core/bpf.h"
41 : #include "core/btf.h"
42 : #include "core/chain.h"
43 : #include "core/counter.h"
44 : #include "core/dump.h"
45 : #include "core/flavor.h"
46 : #include "core/front.h"
47 : #include "core/helper.h"
48 : #include "core/hook.h"
49 : #include "core/io.h"
50 : #include "core/list.h"
51 : #include "core/logger.h"
52 : #include "core/marsh.h"
53 : #include "core/matcher.h"
54 : #include "core/opts.h"
55 : #include "core/rule.h"
56 : #include "core/set.h"
57 : #include "core/verdict.h"
58 :
59 : #include "external/filter.h"
60 : #include "external/murmur3.h"
61 :
62 : #define _BF_PROGRAM_DEFAULT_IMG_SIZE (1 << 6)
63 :
64 16 : static const struct bf_flavor_ops *bf_flavor_ops_get(enum bf_hook hook)
65 : {
66 : static const struct bf_flavor_ops *flavor_ops[] = {
67 : [BF_HOOK_XDP] = &bf_flavor_ops_xdp,
68 : [BF_HOOK_TC_INGRESS] = &bf_flavor_ops_tc,
69 : [BF_HOOK_NF_PRE_ROUTING] = &bf_flavor_ops_nf,
70 : [BF_HOOK_NF_LOCAL_IN] = &bf_flavor_ops_nf,
71 : [BF_HOOK_NF_FORWARD] = &bf_flavor_ops_nf,
72 : [BF_HOOK_CGROUP_INGRESS] = &bf_flavor_ops_cgroup,
73 : [BF_HOOK_CGROUP_EGRESS] = &bf_flavor_ops_cgroup,
74 : [BF_HOOK_NF_LOCAL_OUT] = &bf_flavor_ops_nf,
75 : [BF_HOOK_NF_POST_ROUTING] = &bf_flavor_ops_nf,
76 : [BF_HOOK_TC_EGRESS] = &bf_flavor_ops_tc,
77 : };
78 :
79 16 : bf_assert(0 <= hook && hook < _BF_HOOK_MAX);
80 : static_assert(ARRAY_SIZE(flavor_ops) == _BF_HOOK_MAX,
81 : "missing entries in flavors array");
82 :
83 14 : return flavor_ops[hook];
84 : }
85 :
86 : /**
87 : * Generate the program's identifier.
88 : *
89 : * A program's ID is a way to idenfity it: there shouldn't be two programs
90 : * with the same ID. This ID will be used when we need to avoid name clashes,
91 : * such as when the BPF objects are pinned to the system.
92 : *
93 : * The ID is composed of 3 parts:
94 : * - The @c bf prefix, as the ID will be used as a prefix for the BPF objects
95 : * name.
96 : * - A hook idenfitier
97 : * - A hash of the unique property of the hook: multiple programs can be
98 : * generated for a given hook, so we hash the data that make this specific
99 : * program unique amongst other programs attached to the same hook.
100 : *
101 : * @param program Program to generate the ID for. Can't be NULL.
102 : * @return 0 on success, or a negative errno value on error.
103 : */
104 4 : static int _bf_program_genid(struct bf_program *program)
105 : {
106 : static const char *flavor_keys[] = {
107 : [BF_HOOK_XDP] = "xdp",
108 : [BF_HOOK_TC_INGRESS] = "tci",
109 : [BF_HOOK_NF_PRE_ROUTING] = "nfp",
110 : [BF_HOOK_NF_LOCAL_IN] = "nfi",
111 : [BF_HOOK_CGROUP_INGRESS] = "cgi",
112 : [BF_HOOK_CGROUP_EGRESS] = "cge",
113 : [BF_HOOK_NF_FORWARD] = "nff",
114 : [BF_HOOK_NF_LOCAL_OUT] = "nfo",
115 : [BF_HOOK_NF_POST_ROUTING] = "nfr",
116 : [BF_HOOK_TC_EGRESS] = "tce",
117 : };
118 : static_assert(ARRAY_SIZE(flavor_keys) == _BF_HOOK_MAX,
119 : "missing entries in flavor_keys array");
120 :
121 : char buf[PATH_MAX];
122 : uint32_t hash;
123 :
124 4 : bf_assert(program);
125 :
126 : // If the chain has a name, use it as ID
127 4 : if (program->runtime.chain->hook_opts.used_opts & (1 << BF_HOOK_OPT_NAME)) {
128 0 : (void)snprintf(program->id, BF_PROG_ID_LEN,
129 0 : program->runtime.chain->hook_opts.name);
130 0 : return 0;
131 : }
132 :
133 4 : switch (program->hook) {
134 3 : case BF_HOOK_XDP:
135 : case BF_HOOK_TC_INGRESS:
136 : case BF_HOOK_TC_EGRESS:
137 3 : (void)snprintf(buf, PATH_MAX, "%s_%08x", flavor_keys[program->hook],
138 3 : program->runtime.chain->hook_opts.ifindex);
139 3 : break;
140 1 : case BF_HOOK_NF_PRE_ROUTING:
141 : case BF_HOOK_NF_LOCAL_IN:
142 : case BF_HOOK_NF_FORWARD:
143 : case BF_HOOK_NF_LOCAL_OUT:
144 : case BF_HOOK_NF_POST_ROUTING:
145 1 : (void)snprintf(buf, PATH_MAX, "%s", flavor_keys[program->hook]);
146 1 : break;
147 0 : case BF_HOOK_CGROUP_INGRESS:
148 : case BF_HOOK_CGROUP_EGRESS:
149 0 : (void)snprintf(buf, PATH_MAX, "%s_%s", flavor_keys[program->hook],
150 0 : program->runtime.chain->hook_opts.cgroup);
151 0 : break;
152 0 : default:
153 0 : return bf_err_r(-ENOTSUP, "hook %d is not supported", program->hook);
154 : }
155 :
156 4 : murmur3_x86_32(buf, (int)strlen(buf), 0, &hash);
157 :
158 4 : (void)snprintf(program->id, BF_PROG_ID_LEN, "bf_%3s_%04x",
159 4 : flavor_keys[program->hook], (uint16_t)(hash & 0xff));
160 :
161 4 : return 0;
162 : }
163 :
164 4 : int bf_program_new(struct bf_program **program, enum bf_hook hook,
165 : enum bf_front front, const struct bf_chain *chain)
166 : {
167 4 : _cleanup_bf_program_ struct bf_program *_program = NULL;
168 : char name[BPF_OBJ_NAME_LEN];
169 : uint32_t set_idx = 0;
170 : int r;
171 :
172 4 : bf_assert(chain);
173 :
174 4 : _program = calloc(1, sizeof(*_program));
175 4 : if (!_program)
176 : return -ENOMEM;
177 :
178 4 : _program->hook = hook;
179 4 : _program->front = front;
180 4 : _program->runtime.ops = bf_flavor_ops_get(hook);
181 4 : _program->runtime.chain = chain;
182 :
183 4 : r = _bf_program_genid(_program);
184 4 : if (r) {
185 0 : return bf_err_r(r,
186 : "failed to generate ID for bf_program attached to %s",
187 : bf_hook_to_str(hook));
188 : }
189 :
190 4 : (void)snprintf(_program->prog_name, BPF_OBJ_NAME_LEN, "%s_prg",
191 4 : _program->id);
192 :
193 4 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "%s_cmp", _program->id);
194 4 : r = bf_map_new(&_program->cmap, name, BF_MAP_TYPE_COUNTERS,
195 : BF_MAP_BPF_TYPE_ARRAY, sizeof(uint32_t),
196 : sizeof(struct bf_counter), 1);
197 4 : if (r < 0)
198 0 : return bf_err_r(r, "failed to create the counters bf_map object");
199 :
200 4 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "%s_pmp", _program->id);
201 4 : r = bf_map_new(&_program->pmap, name, BF_MAP_TYPE_PRINTER,
202 : BF_MAP_BPF_TYPE_ARRAY, sizeof(uint32_t),
203 : BF_MAP_VALUE_SIZE_UNKNOWN, 1);
204 4 : if (r < 0)
205 0 : return bf_err_r(r, "failed to create the printer bf_map object");
206 :
207 4 : _program->sets = bf_map_list();
208 8 : bf_list_foreach (&chain->sets, set_node) {
209 0 : struct bf_set *set = bf_list_node_get_data(set_node);
210 0 : _cleanup_bf_map_ struct bf_map *map = NULL;
211 :
212 0 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "%s_s%02x", _program->id,
213 0 : (uint8_t)set_idx++);
214 0 : r = bf_map_new(&map, name, BF_MAP_TYPE_SET, BF_MAP_BPF_TYPE_HASH,
215 0 : set->elem_size, 1, bf_list_size(&set->elems));
216 0 : if (r < 0)
217 : return r;
218 :
219 0 : r = bf_list_add_tail(&_program->sets, map);
220 0 : if (r < 0)
221 : return r;
222 0 : TAKE_PTR(map);
223 : };
224 :
225 4 : _program->links = bf_link_list();
226 :
227 4 : r = bf_printer_new(&_program->printer);
228 4 : if (r)
229 : return r;
230 :
231 4 : bf_list_init(&_program->fixups,
232 4 : (bf_list_ops[]) {{.free = (bf_list_ops_free)bf_fixup_free}});
233 :
234 4 : _program->runtime.prog_fd = -1;
235 :
236 4 : *program = TAKE_PTR(_program);
237 :
238 4 : return 0;
239 : }
240 :
241 14 : void bf_program_free(struct bf_program **program)
242 : {
243 14 : if (!*program)
244 : return;
245 :
246 4 : bf_list_clean(&(*program)->fixups);
247 4 : free((*program)->img);
248 :
249 : /* Close the file descriptors if they are still open. If --transient is
250 : * used, then the file descriptors are already closed (as
251 : * bf_program_unload() has been called). Otherwise, bf_program_unload()
252 : * won't be called, but the programs are pinned, so they can be closed
253 : * safely. */
254 4 : closep(&(*program)->runtime.prog_fd);
255 :
256 4 : bf_map_free(&(*program)->cmap);
257 4 : bf_map_free(&(*program)->pmap);
258 4 : bf_list_clean(&(*program)->sets);
259 4 : bf_list_clean(&(*program)->links);
260 4 : bf_printer_free(&(*program)->printer);
261 :
262 4 : free(*program);
263 4 : *program = NULL;
264 : }
265 :
266 0 : int bf_program_marsh(const struct bf_program *program, struct bf_marsh **marsh)
267 : {
268 0 : _cleanup_bf_marsh_ struct bf_marsh *_marsh = NULL;
269 : int r;
270 :
271 0 : bf_assert(program);
272 0 : bf_assert(marsh);
273 :
274 0 : r = bf_marsh_new(&_marsh, NULL, 0);
275 0 : if (r < 0)
276 : return r;
277 :
278 0 : r |= bf_marsh_add_child_raw(&_marsh, &program->hook, sizeof(program->hook));
279 0 : r |= bf_marsh_add_child_raw(&_marsh, &program->front,
280 : sizeof(program->front));
281 0 : if (r)
282 : return r;
283 :
284 : {
285 : // Serialize bf_program.counters
286 0 : _cleanup_bf_marsh_ struct bf_marsh *counters_elem = NULL;
287 :
288 0 : r = bf_map_marsh(program->cmap, &counters_elem);
289 0 : if (r < 0)
290 : return r;
291 :
292 0 : r = bf_marsh_add_child_obj(&_marsh, counters_elem);
293 0 : if (r < 0)
294 : return r;
295 : }
296 :
297 : {
298 : // Serialize bf_program.pmap
299 0 : _cleanup_bf_marsh_ struct bf_marsh *pmap_elem = NULL;
300 :
301 0 : r = bf_map_marsh(program->pmap, &pmap_elem);
302 0 : if (r < 0)
303 : return r;
304 :
305 0 : r = bf_marsh_add_child_obj(&_marsh, pmap_elem);
306 0 : if (r < 0)
307 : return r;
308 : }
309 :
310 : {
311 : // Serialize bf_program.sets
312 0 : _cleanup_bf_marsh_ struct bf_marsh *sets_elem = NULL;
313 :
314 0 : r = bf_list_marsh(&program->sets, &sets_elem);
315 0 : if (r < 0)
316 : return r;
317 :
318 0 : r = bf_marsh_add_child_obj(&_marsh, sets_elem);
319 0 : if (r < 0) {
320 0 : return bf_err_r(
321 : r,
322 : "failed to insert serialized sets into bf_program serialized data");
323 : }
324 : }
325 :
326 : {
327 : // Serialize bf_program.links
328 0 : _cleanup_bf_marsh_ struct bf_marsh *links_elem = NULL;
329 :
330 0 : r = bf_list_marsh(&program->links, &links_elem);
331 0 : if (r)
332 : return r;
333 :
334 0 : r = bf_marsh_add_child_obj(&_marsh, links_elem);
335 0 : if (r) {
336 0 : return bf_err_r(
337 : r,
338 : "failed to insert serialized links into bf_program serialized data");
339 : }
340 : }
341 :
342 : {
343 : // Serialise bf_program.printer
344 0 : _cleanup_bf_marsh_ struct bf_marsh *child = NULL;
345 :
346 0 : r = bf_printer_marsh(program->printer, &child);
347 0 : if (r)
348 0 : return bf_err_r(r, "failed to marsh bf_printer object");
349 :
350 0 : r = bf_marsh_add_child_obj(&_marsh, child);
351 0 : if (r)
352 0 : return bf_err_r(r, "failed to append object to marsh");
353 : }
354 :
355 0 : r |= bf_marsh_add_child_raw(&_marsh, &program->num_counters,
356 : sizeof(program->num_counters));
357 0 : r |= bf_marsh_add_child_raw(&_marsh, program->img,
358 0 : program->img_size * sizeof(struct bpf_insn));
359 0 : if (r)
360 0 : return bf_err_r(r, "Failed to serialize program");
361 :
362 0 : *marsh = TAKE_PTR(_marsh);
363 :
364 0 : return 0;
365 : }
366 :
367 0 : int bf_program_unmarsh(const struct bf_marsh *marsh,
368 : struct bf_program **program,
369 : const struct bf_chain *chain)
370 : {
371 : enum bf_hook hook;
372 : enum bf_front front;
373 : char dir[PATH_MAX];
374 : _cleanup_close_ int pindir_fd = -1;
375 0 : _cleanup_bf_program_ struct bf_program *_program = NULL;
376 : struct bf_marsh *child = NULL;
377 : int r;
378 :
379 0 : bf_assert(marsh);
380 0 : bf_assert(program);
381 :
382 0 : if (!(child = bf_marsh_next_child(marsh, NULL)))
383 : return -EINVAL;
384 0 : memcpy(&hook, child->data, sizeof(hook));
385 :
386 0 : if (!(child = bf_marsh_next_child(marsh, child)))
387 : return -EINVAL;
388 0 : memcpy(&front, child->data, sizeof(front));
389 :
390 0 : r = bf_program_new(&_program, hook, front, chain);
391 0 : if (r < 0)
392 : return r;
393 :
394 0 : (void)snprintf(dir, PATH_MAX, "%s/%s", BF_PIN_DIR, _program->id);
395 0 : pindir_fd = open(dir, O_DIRECTORY, 0);
396 0 : if (pindir_fd < 0) {
397 0 : return bf_err_r(errno, "failed to open bf_program pin directory %s",
398 : dir);
399 : }
400 :
401 0 : if (!(child = bf_marsh_next_child(marsh, child)))
402 : return -EINVAL;
403 0 : bf_map_free(&_program->cmap);
404 0 : r = bf_map_new_from_marsh(&_program->cmap, pindir_fd, child);
405 0 : if (r < 0)
406 : return r;
407 :
408 0 : if (!(child = bf_marsh_next_child(marsh, child)))
409 : return -EINVAL;
410 0 : bf_map_free(&_program->pmap);
411 0 : r = bf_map_new_from_marsh(&_program->pmap, pindir_fd, child);
412 0 : if (r < 0)
413 : return r;
414 :
415 : /** @todo Avoid creating and filling the list in @ref bf_program_new before
416 : * trashing it all here. Eventually, this function will be replaced with
417 : * @c bf_program_new_from_marsh and this issue could be solved by **not**
418 : * relying on @ref bf_program_new to allocate an initialize @p _program . */
419 0 : bf_list_clean(&_program->sets);
420 0 : _program->sets = bf_map_list();
421 :
422 0 : if (!(child = bf_marsh_next_child(marsh, child)))
423 : return -EINVAL;
424 : {
425 : // Unmarsh bf_program.sets
426 : struct bf_marsh *set_elem = NULL;
427 :
428 0 : while ((set_elem = bf_marsh_next_child(child, set_elem))) {
429 0 : _cleanup_bf_map_ struct bf_map *map = NULL;
430 :
431 0 : r = bf_map_new_from_marsh(&map, pindir_fd, set_elem);
432 0 : if (r < 0)
433 : return r;
434 :
435 0 : r = bf_list_add_tail(&_program->sets, map);
436 0 : if (r < 0)
437 : return r;
438 :
439 0 : TAKE_PTR(map);
440 : }
441 : }
442 :
443 : // Unmarsh bf_program.links
444 0 : if (!(child = bf_marsh_next_child(marsh, child)))
445 : return -EINVAL;
446 : {
447 : struct bf_marsh *link_elem = NULL;
448 :
449 0 : while ((link_elem = bf_marsh_next_child(child, link_elem))) {
450 0 : _cleanup_bf_link_ struct bf_link *link = NULL;
451 :
452 0 : r = bf_link_new_from_marsh(&link, pindir_fd, link_elem);
453 0 : if (r)
454 : return r;
455 :
456 0 : r = bf_list_add_tail(&_program->links, link);
457 0 : if (r)
458 : return r;
459 :
460 0 : TAKE_PTR(link);
461 : }
462 : }
463 :
464 : // Unmarsh bf_program.printer
465 0 : child = bf_marsh_next_child(marsh, child);
466 0 : if (!child)
467 0 : return bf_err_r(-EINVAL, "failed to find valid child");
468 :
469 0 : bf_printer_free(&_program->printer);
470 0 : r = bf_printer_new_from_marsh(&_program->printer, child);
471 0 : if (r)
472 0 : return bf_err_r(r, "failed to restore bf_printer object");
473 :
474 0 : if (!(child = bf_marsh_next_child(marsh, child)))
475 : return -EINVAL;
476 0 : memcpy(&_program->num_counters, child->data,
477 : sizeof(_program->num_counters));
478 :
479 0 : if (!(child = bf_marsh_next_child(marsh, child)))
480 : return -EINVAL;
481 0 : _program->img = bf_memdup(child->data, child->data_len);
482 0 : _program->img_size = child->data_len / sizeof(struct bpf_insn);
483 0 : _program->img_cap = child->data_len / sizeof(struct bpf_insn);
484 :
485 0 : if (bf_marsh_next_child(marsh, child))
486 0 : bf_warn("codegen marsh has more children than expected");
487 :
488 0 : r = bf_bpf_obj_get(_program->prog_name, pindir_fd,
489 0 : &_program->runtime.prog_fd);
490 0 : if (r < 0)
491 0 : return bf_err_r(r, "failed to get prog fd");
492 :
493 0 : *program = TAKE_PTR(_program);
494 :
495 0 : return 0;
496 : }
497 :
498 0 : void bf_program_dump(const struct bf_program *program, prefix_t *prefix)
499 : {
500 0 : bf_assert(program);
501 0 : bf_assert(prefix);
502 :
503 0 : DUMP(prefix, "struct bf_program at %p", program);
504 :
505 0 : bf_dump_prefix_push(prefix);
506 :
507 0 : DUMP(prefix, "id: %s", program->id);
508 0 : DUMP(prefix, "hook: %s", bf_hook_to_str(program->hook));
509 0 : DUMP(prefix, "front: %s", bf_front_to_str(program->front));
510 0 : DUMP(prefix, "num_counters: %lu", program->num_counters);
511 0 : DUMP(prefix, "prog_name: %s", program->prog_name);
512 :
513 0 : DUMP(prefix, "cmap: struct bf_map *");
514 0 : bf_dump_prefix_push(prefix);
515 0 : bf_map_dump(program->cmap, bf_dump_prefix_last(prefix));
516 0 : bf_dump_prefix_pop(prefix);
517 :
518 0 : DUMP(prefix, "pmap: struct bf_map *");
519 0 : bf_dump_prefix_push(prefix);
520 0 : bf_map_dump(program->pmap, bf_dump_prefix_last(prefix));
521 0 : bf_dump_prefix_pop(prefix);
522 :
523 0 : DUMP(prefix, "sets: bf_list<bf_map>[%lu]", bf_list_size(&program->sets));
524 0 : bf_dump_prefix_push(prefix);
525 0 : bf_list_foreach (&program->sets, map_node) {
526 0 : struct bf_map *map = bf_list_node_get_data(map_node);
527 :
528 0 : if (bf_list_is_tail(&program->sets, map_node))
529 0 : bf_dump_prefix_last(prefix);
530 :
531 0 : bf_map_dump(map, prefix);
532 : }
533 0 : bf_dump_prefix_pop(prefix);
534 :
535 0 : DUMP(prefix, "links: bf_list<bf_link>[%lu]", bf_list_size(&program->links));
536 0 : bf_dump_prefix_push(prefix);
537 0 : bf_list_foreach (&program->links, link_node) {
538 0 : struct bf_link *link = bf_list_node_get_data(link_node);
539 :
540 0 : if (bf_list_is_tail(&program->links, link_node))
541 0 : bf_dump_prefix_last(prefix);
542 :
543 0 : bf_link_dump(link, prefix);
544 : }
545 0 : bf_dump_prefix_pop(prefix);
546 :
547 0 : DUMP(prefix, "printer: struct bf_printer *");
548 0 : bf_dump_prefix_push(prefix);
549 0 : bf_printer_dump(program->printer, prefix);
550 0 : bf_dump_prefix_pop(prefix);
551 :
552 0 : DUMP(prefix, "img: %p", program->img);
553 0 : DUMP(prefix, "img_size: %lu", program->img_size);
554 0 : DUMP(prefix, "img_cap: %lu", program->img_cap);
555 :
556 0 : DUMP(prefix, "fixups: bf_list<struct bf_fixup>[%lu]",
557 : bf_list_size(&program->fixups));
558 0 : bf_dump_prefix_push(prefix);
559 0 : bf_list_foreach (&program->fixups, fixup_node) {
560 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
561 :
562 0 : if (bf_list_is_tail(&program->fixups, fixup_node))
563 0 : bf_dump_prefix_last(prefix);
564 :
565 0 : bf_fixup_dump(fixup, prefix);
566 : }
567 0 : bf_dump_prefix_pop(prefix);
568 :
569 0 : DUMP(bf_dump_prefix_last(prefix), "runtime: <anonymous>");
570 0 : bf_dump_prefix_push(prefix);
571 0 : DUMP(prefix, "prog_fd: %d", program->runtime.prog_fd);
572 0 : DUMP(bf_dump_prefix_last(prefix), "ops: %p", program->runtime.ops);
573 0 : bf_dump_prefix_pop(prefix);
574 :
575 0 : bf_dump_prefix_pop(prefix);
576 0 : }
577 :
578 0 : static inline size_t _bf_round_next_power_of_2(size_t value)
579 : {
580 0 : value--;
581 0 : value |= value >> 1;
582 0 : value |= value >> 2;
583 0 : value |= value >> 4;
584 0 : value |= value >> 8;
585 0 : value |= value >> 16;
586 :
587 0 : return ++value;
588 : }
589 :
590 4 : int bf_program_grow_img(struct bf_program *program)
591 : {
592 : size_t new_cap = _BF_PROGRAM_DEFAULT_IMG_SIZE;
593 : int r;
594 :
595 4 : bf_assert(program);
596 :
597 4 : if (program->img)
598 0 : new_cap = _bf_round_next_power_of_2(program->img_cap << 1);
599 :
600 4 : r = bf_realloc((void **)&program->img, new_cap * sizeof(struct bpf_insn));
601 4 : if (r < 0) {
602 0 : return bf_err_r(r, "failed to grow program img from %lu to %lu insn",
603 : program->img_cap, new_cap);
604 : }
605 :
606 4 : program->img_cap = new_cap;
607 :
608 4 : return 0;
609 : }
610 :
611 0 : static void _bf_program_fixup_insn(struct bpf_insn *insn,
612 : enum bf_fixup_insn type, int32_t value)
613 : {
614 0 : switch (type) {
615 0 : case BF_FIXUP_INSN_OFF:
616 0 : bf_assert(!insn->off);
617 0 : bf_assert(value < SHRT_MAX);
618 0 : insn->off = (int16_t)value;
619 0 : break;
620 0 : case BF_FIXUP_INSN_IMM:
621 0 : bf_assert(!insn->imm);
622 0 : insn->imm = value;
623 0 : break;
624 0 : default:
625 0 : bf_abort(
626 : "unsupported fixup instruction type, this should not happen: %d",
627 : type);
628 : break;
629 : }
630 0 : }
631 :
632 0 : static int _bf_program_fixup(struct bf_program *program,
633 : enum bf_fixup_type type)
634 : {
635 0 : bf_assert(program);
636 0 : bf_assert(type >= 0 && type < _BF_FIXUP_TYPE_MAX);
637 :
638 0 : bf_list_foreach (&program->fixups, fixup_node) {
639 : enum bf_fixup_insn insn_type = _BF_FIXUP_INSN_MAX;
640 : int32_t value;
641 : size_t offset;
642 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
643 0 : struct bpf_insn *insn = &program->img[fixup->insn];
644 : struct bf_map *map;
645 :
646 0 : if (type != fixup->type)
647 0 : continue;
648 :
649 0 : switch (type) {
650 0 : case BF_FIXUP_TYPE_JMP_NEXT_RULE:
651 : insn_type = BF_FIXUP_INSN_OFF;
652 0 : value = (int)(program->img_size - fixup->insn - 1U);
653 0 : break;
654 0 : case BF_FIXUP_TYPE_COUNTERS_MAP_FD:
655 : insn_type = BF_FIXUP_INSN_IMM;
656 0 : value = program->cmap->fd;
657 0 : break;
658 0 : case BF_FIXUP_TYPE_PRINTER_MAP_FD:
659 : insn_type = BF_FIXUP_INSN_IMM;
660 0 : value = program->pmap->fd;
661 0 : break;
662 0 : case BF_FIXUP_TYPE_SET_MAP_FD:
663 0 : map = bf_list_get_at(&program->sets, insn->imm);
664 0 : if (!map) {
665 0 : return bf_err_r(-ENOENT, "can't find set map at index %d",
666 : insn->imm);
667 : }
668 : insn_type = BF_FIXUP_INSN_IMM;
669 0 : value = map->fd;
670 0 : break;
671 0 : case BF_FIXUP_TYPE_FUNC_CALL:
672 : insn_type = BF_FIXUP_INSN_IMM;
673 0 : offset = program->functions_location[fixup->attr.function] -
674 : fixup->insn - 1;
675 0 : bf_assert(offset < INT_MAX);
676 0 : value = (int32_t)offset;
677 0 : break;
678 0 : default:
679 0 : bf_abort("unsupported fixup type, this should not happen: %d",
680 : type);
681 : break;
682 : }
683 :
684 0 : _bf_program_fixup_insn(insn, insn_type, value);
685 0 : bf_list_delete(&program->fixups, fixup_node);
686 : }
687 :
688 : return 0;
689 : }
690 :
691 0 : static int _bf_program_generate_rule(struct bf_program *program,
692 : struct bf_rule *rule)
693 : {
694 : int r;
695 :
696 0 : bf_assert(program);
697 0 : bf_assert(rule);
698 :
699 0 : bf_list_foreach (&rule->matchers, matcher_node) {
700 0 : struct bf_matcher *matcher = bf_list_node_get_data(matcher_node);
701 :
702 0 : switch (matcher->type) {
703 0 : case BF_MATCHER_META_IFINDEX:
704 : case BF_MATCHER_META_L3_PROTO:
705 : case BF_MATCHER_META_L4_PROTO:
706 : case BF_MATCHER_META_SPORT:
707 : case BF_MATCHER_META_DPORT:
708 0 : r = bf_matcher_generate_meta(program, matcher);
709 0 : if (r)
710 : return r;
711 : break;
712 0 : case BF_MATCHER_IP4_SRC_ADDR:
713 : case BF_MATCHER_IP4_DST_ADDR:
714 : case BF_MATCHER_IP4_PROTO:
715 0 : r = bf_matcher_generate_ip4(program, matcher);
716 0 : if (r)
717 : return r;
718 : break;
719 0 : case BF_MATCHER_IP6_SADDR:
720 : case BF_MATCHER_IP6_DADDR:
721 0 : r = bf_matcher_generate_ip6(program, matcher);
722 0 : if (r)
723 : return r;
724 : break;
725 0 : case BF_MATCHER_TCP_SPORT:
726 : case BF_MATCHER_TCP_DPORT:
727 : case BF_MATCHER_TCP_FLAGS:
728 0 : r = bf_matcher_generate_tcp(program, matcher);
729 0 : if (r)
730 : return r;
731 : break;
732 0 : case BF_MATCHER_UDP_SPORT:
733 : case BF_MATCHER_UDP_DPORT:
734 0 : r = bf_matcher_generate_udp(program, matcher);
735 0 : if (r)
736 : return r;
737 : break;
738 0 : case BF_MATCHER_SET_SRCIP6PORT:
739 : case BF_MATCHER_SET_SRCIP6:
740 0 : r = bf_matcher_generate_set(program, matcher);
741 0 : if (r)
742 : return r;
743 : break;
744 0 : default:
745 0 : return bf_err_r(-EINVAL, "unknown matcher type %d", matcher->type);
746 : };
747 : }
748 :
749 0 : if (rule->counters) {
750 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_1, rule->index));
751 0 : EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10,
752 : BF_PROG_CTX_OFF(pkt_size)));
753 0 : EMIT_FIXUP_CALL(program, BF_FIXUP_FUNC_UPDATE_COUNTERS);
754 : }
755 :
756 0 : switch (rule->verdict) {
757 0 : case BF_VERDICT_ACCEPT:
758 : case BF_VERDICT_DROP:
759 0 : EMIT(program,
760 : BPF_MOV64_IMM(BPF_REG_0,
761 : program->runtime.ops->get_verdict(rule->verdict)));
762 0 : EMIT(program, BPF_EXIT_INSN());
763 0 : break;
764 : case BF_VERDICT_CONTINUE:
765 : // Fall through to next rule or default chain policy.
766 : break;
767 0 : default:
768 0 : bf_abort("unsupported verdict, this should not happen: %d",
769 : rule->verdict);
770 : break;
771 : }
772 :
773 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_JMP_NEXT_RULE);
774 0 : if (r)
775 0 : return bf_err_r(r, "failed to generate next rule fixups");
776 :
777 : return 0;
778 : }
779 :
780 : /**
781 : * Generate the BPF function to update a rule's counters.
782 : *
783 : * This function defines a new function **in** the generated BPF program to
784 : * be called during packet processing.
785 : *
786 : * Parameters:
787 : * - @c r1 : index of the rule to update the counters for.
788 : * - @c r2 : size of the packet.
789 : * Returns:
790 : * 0 on success, non-zero on error.
791 : *
792 : * @param program Program to emit the function into. Can not be NULL.
793 : * @return 0 on success, or negative errno value on error.
794 : */
795 0 : static int _bf_program_generate_update_counters(struct bf_program *program)
796 : {
797 : // Move the counters key in scratch[0..4] and the packet size in scratch[8..15]
798 0 : EMIT(program,
799 : BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_1, BF_PROG_SCR_OFF(0)));
800 0 : EMIT(program,
801 : BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, BF_PROG_SCR_OFF(8)));
802 :
803 : // Call bpf_map_lookup_elem()
804 0 : EMIT_LOAD_COUNTERS_FD_FIXUP(program, BPF_REG_1);
805 0 : EMIT(program, BPF_MOV64_REG(BPF_REG_2, BPF_REG_10));
806 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, BF_PROG_SCR_OFF(0)));
807 0 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem));
808 :
809 : // If the counters doesn't exist, return from the function
810 : {
811 0 : _cleanup_bf_jmpctx_ struct bf_jmpctx _ =
812 0 : bf_jmpctx_get(program, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 0));
813 :
814 0 : if (bf_opts_is_verbose(BF_VERBOSE_BPF))
815 0 : EMIT_PRINT(program, "failed to fetch the rule's counters");
816 :
817 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_0, 1));
818 0 : EMIT(program, BPF_EXIT_INSN());
819 : }
820 :
821 : // Increment the packets count by 1.
822 0 : EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
823 : offsetof(struct bf_counter, packets)));
824 0 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 1));
825 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1,
826 : offsetof(struct bf_counter, packets)));
827 :
828 : // Increase the total byte by the size of the packet.
829 0 : EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
830 : offsetof(struct bf_counter, bytes)));
831 0 : EMIT(program,
832 : BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, BF_PROG_SCR_OFF(8)));
833 0 : EMIT(program, BPF_ALU64_REG(BPF_ADD, BPF_REG_1, BPF_REG_2));
834 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1,
835 : offsetof(struct bf_counter, bytes)));
836 :
837 : // On success, return 0
838 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_0, 0));
839 0 : EMIT(program, BPF_EXIT_INSN());
840 :
841 0 : return 0;
842 : }
843 :
844 0 : static int _bf_program_generate_functions(struct bf_program *program)
845 : {
846 : int r;
847 :
848 0 : bf_assert(program);
849 :
850 0 : bf_list_foreach (&program->fixups, fixup_node) {
851 0 : struct bf_fixup *fixup = bf_list_node_get_data(fixup_node);
852 0 : size_t off = program->img_size;
853 :
854 0 : if (fixup->type != BF_FIXUP_TYPE_FUNC_CALL)
855 0 : continue;
856 :
857 0 : bf_assert(fixup->attr.function >= 0 &&
858 : fixup->attr.function < _BF_FIXUP_FUNC_MAX);
859 :
860 : // Only generate each function once
861 0 : if (program->functions_location[fixup->attr.function])
862 0 : continue;
863 :
864 0 : switch (fixup->attr.function) {
865 0 : case BF_FIXUP_FUNC_UPDATE_COUNTERS:
866 0 : r = _bf_program_generate_update_counters(program);
867 0 : if (r)
868 : return r;
869 : break;
870 0 : default:
871 0 : bf_abort("unsupported fixup function, this should not happen: %d",
872 : fixup->attr.function);
873 : break;
874 : }
875 :
876 0 : program->functions_location[fixup->attr.function] = off;
877 : }
878 :
879 : return 0;
880 : }
881 :
882 41 : int bf_program_emit(struct bf_program *program, struct bpf_insn insn)
883 : {
884 : int r;
885 :
886 41 : bf_assert(program);
887 :
888 41 : if (program->img_size == program->img_cap) {
889 3 : r = bf_program_grow_img(program);
890 3 : if (r)
891 : return r;
892 : }
893 :
894 41 : program->img[program->img_size++] = insn;
895 :
896 41 : return 0;
897 : }
898 :
899 0 : int bf_program_emit_kfunc_call(struct bf_program *program, const char *name)
900 : {
901 : int r;
902 :
903 0 : bf_assert(program);
904 0 : bf_assert(name);
905 :
906 0 : r = bf_btf_get_id(name);
907 0 : if (r < 0)
908 : return r;
909 :
910 0 : EMIT(program, ((struct bpf_insn) {.code = BPF_JMP | BPF_CALL,
911 : .dst_reg = 0,
912 : .src_reg = BPF_PSEUDO_KFUNC_CALL,
913 : .off = 0,
914 : .imm = r}));
915 :
916 0 : return 0;
917 : }
918 :
919 0 : int bf_program_emit_fixup(struct bf_program *program, enum bf_fixup_type type,
920 : struct bpf_insn insn, const union bf_fixup_attr *attr)
921 : {
922 0 : _cleanup_bf_fixup_ struct bf_fixup *fixup = NULL;
923 : int r;
924 :
925 0 : bf_assert(program);
926 :
927 0 : if (program->img_size == program->img_cap) {
928 0 : r = bf_program_grow_img(program);
929 0 : if (r)
930 : return r;
931 : }
932 :
933 0 : r = bf_fixup_new(&fixup, type, program->img_size, attr);
934 0 : if (r)
935 : return r;
936 :
937 0 : r = bf_list_add_tail(&program->fixups, fixup);
938 0 : if (r)
939 : return r;
940 :
941 0 : TAKE_PTR(fixup);
942 :
943 : /* This call could fail and return an error, in which case it is not
944 : * properly handled. However, this shouldn't be an issue as we previously
945 : * test whether enough room is available in cgen.img, which is currently
946 : * the only reason for EMIT() to fail. */
947 0 : EMIT(program, insn);
948 :
949 : return 0;
950 : }
951 :
952 2 : int bf_program_emit_fixup_call(struct bf_program *program,
953 : enum bf_fixup_func function)
954 : {
955 1 : _cleanup_bf_fixup_ struct bf_fixup *fixup = NULL;
956 : int r;
957 :
958 2 : bf_assert(program);
959 :
960 1 : if (program->img_size == program->img_cap) {
961 1 : r = bf_program_grow_img(program);
962 1 : if (r)
963 : return r;
964 : }
965 :
966 1 : r = bf_fixup_new(&fixup, BF_FIXUP_TYPE_FUNC_CALL, program->img_size, NULL);
967 1 : if (r)
968 : return r;
969 :
970 1 : fixup->attr.function = function;
971 :
972 1 : r = bf_list_add_tail(&program->fixups, fixup);
973 1 : if (r)
974 : return r;
975 :
976 1 : TAKE_PTR(fixup);
977 :
978 : /* This call could fail and return an error, in which case it is not
979 : * properly handled. However, this shouldn't be an issue as we previously
980 : * test whether enough room is available in cgen.img, which is currently
981 : * the only reason for EMIT() to fail. */
982 1 : EMIT(program, BPF_CALL_REL(0));
983 :
984 1 : return 0;
985 : }
986 :
987 0 : int bf_program_generate(struct bf_program *program)
988 : {
989 0 : const struct bf_chain *chain = program->runtime.chain;
990 : int r;
991 :
992 0 : bf_info("generating program for %s::%s", bf_front_to_str(program->front),
993 : bf_hook_to_str(program->hook));
994 :
995 : /* Add 1 to the number of counters for the policy counter, and 1
996 : * for the first reserved error slot. This must be done ahead of
997 : * generation, as we will index into the error counters. */
998 0 : program->num_counters = bf_list_size(&chain->rules) + 2;
999 :
1000 : // Save the program's argument into the context.
1001 0 : EMIT(program,
1002 : BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
1003 :
1004 : // Reset the protocol ID registers
1005 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, 0));
1006 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_8, 0));
1007 :
1008 0 : r = program->runtime.ops->gen_inline_prologue(program);
1009 0 : if (r)
1010 : return r;
1011 :
1012 0 : bf_list_foreach (&chain->rules, rule_node) {
1013 0 : r = _bf_program_generate_rule(program,
1014 0 : bf_list_node_get_data(rule_node));
1015 0 : if (r)
1016 : return r;
1017 : }
1018 :
1019 0 : r = program->runtime.ops->gen_inline_epilogue(program);
1020 0 : if (r)
1021 : return r;
1022 :
1023 : // Call the update counters function
1024 0 : EMIT(program, BPF_MOV32_IMM(BPF_REG_1, bf_list_size(&chain->rules)));
1025 0 : EMIT(program,
1026 : BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, BF_PROG_CTX_OFF(pkt_size)));
1027 0 : EMIT_FIXUP_CALL(program, BF_FIXUP_FUNC_UPDATE_COUNTERS);
1028 :
1029 0 : EMIT(program, BPF_MOV64_IMM(BPF_REG_0, program->runtime.ops->get_verdict(
1030 : chain->policy)));
1031 0 : EMIT(program, BPF_EXIT_INSN());
1032 :
1033 0 : r = _bf_program_generate_functions(program);
1034 0 : if (r)
1035 : return r;
1036 :
1037 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_FUNC_CALL);
1038 0 : if (r)
1039 0 : return bf_err_r(r, "failed to generate function call fixups");
1040 :
1041 : return 0;
1042 : }
1043 :
1044 : static void _bf_program_unpin(const struct bf_program *program,
1045 : const char *dir);
1046 :
1047 : /**
1048 : * Pin the BPF objects that should survive the daemon's lifetime.
1049 : *
1050 : * If any of the BPF objects can't be pinned, unpin all of them to ensure
1051 : * there will be no leftovers.
1052 : *
1053 : * @param program Program containing the objects to pin. Can't be NULL.
1054 : * @return 0 on success, or negative erron value on failure.
1055 : */
1056 0 : static int _bf_program_pin(const struct bf_program *program)
1057 : {
1058 : _cleanup_close_ int pindir_fd = -1;
1059 : char dir[PATH_MAX];
1060 : int r;
1061 :
1062 0 : bf_assert(program);
1063 :
1064 0 : r = bf_ensure_dir(BF_PIN_DIR);
1065 0 : if (r)
1066 0 : return bf_err_r(r, "failed to ensure BPF objects pin directory exists");
1067 :
1068 0 : (void)snprintf(dir, PATH_MAX, "%s/%s", BF_PIN_DIR, program->id);
1069 :
1070 0 : r = bf_ensure_dir(dir);
1071 0 : if (r)
1072 0 : return bf_err_r(r, "failed to validate pin directory %s", dir);
1073 :
1074 0 : pindir_fd = open(dir, O_DIRECTORY, 0);
1075 0 : if (pindir_fd < 0) {
1076 0 : return bf_err_r(errno, "failed to open bf_program pin directory %s",
1077 : dir);
1078 : }
1079 :
1080 0 : r = bf_bpf_obj_pin(program->prog_name, program->runtime.prog_fd, pindir_fd);
1081 0 : if (r < 0) {
1082 0 : bf_err_r(r, "failed to pin program '%s' in %s", program->prog_name,
1083 : BF_PIN_DIR);
1084 0 : goto err_prog_pin;
1085 : }
1086 :
1087 0 : r = bf_map_pin(program->cmap, pindir_fd);
1088 0 : if (r < 0)
1089 0 : goto err_cmap_pin;
1090 :
1091 0 : r = bf_map_pin(program->pmap, pindir_fd);
1092 0 : if (r < 0)
1093 0 : goto err_pmap_pin;
1094 :
1095 0 : bf_list_foreach (&program->sets, set_node) {
1096 0 : r = bf_map_pin(bf_list_node_get_data(set_node), pindir_fd);
1097 0 : if (r < 0)
1098 0 : goto err_set_pin;
1099 : }
1100 :
1101 0 : bf_list_foreach (&program->links, link_node) {
1102 0 : r = bf_link_pin(bf_list_node_get_data(link_node), pindir_fd);
1103 0 : if (r < 0)
1104 0 : goto err_link_pin;
1105 : }
1106 :
1107 : return 0;
1108 :
1109 : err_link_pin:
1110 0 : bf_list_foreach (&program->links, link_node)
1111 0 : bf_link_unpin(bf_list_node_get_data(link_node), pindir_fd);
1112 0 : err_set_pin:
1113 0 : bf_list_foreach (&program->sets, set_node)
1114 0 : bf_map_unpin(bf_list_node_get_data(set_node), pindir_fd);
1115 0 : bf_map_unpin(program->pmap, pindir_fd);
1116 0 : err_pmap_pin:
1117 0 : bf_map_unpin(program->cmap, pindir_fd);
1118 0 : err_cmap_pin:
1119 0 : unlinkat(pindir_fd, program->prog_name, 0);
1120 0 : err_prog_pin:
1121 : closep(&pindir_fd);
1122 : return r;
1123 : }
1124 :
1125 : /**
1126 : * Unpin the BPF objects owned by a program.
1127 : *
1128 : * If the @p program object is deleted, the BPF object will disappear from
1129 : * the system.
1130 : *
1131 : * @param program Program containing the objects to unpin. Can't be NULL.
1132 : * @param dir If non NULL, override the directory to unpin the BPF objects
1133 : * from.
1134 : */
1135 0 : static void _bf_program_unpin(const struct bf_program *program, const char *dir)
1136 : {
1137 : char _dir[PATH_MAX];
1138 : int pindir_fd;
1139 : int r;
1140 :
1141 0 : bf_assert(program);
1142 :
1143 0 : if (dir)
1144 0 : (void)snprintf(_dir, PATH_MAX, "%s", dir);
1145 : else
1146 0 : (void)snprintf(_dir, PATH_MAX, "%s/%s", BF_PIN_DIR, program->id);
1147 :
1148 0 : pindir_fd = open(_dir, O_DIRECTORY, 0);
1149 0 : if (pindir_fd < 0) {
1150 0 : bf_warn_r(
1151 : errno,
1152 : "failed to open bf_program pin directory %s, assuming BPF objects are unpinned",
1153 : dir);
1154 0 : return;
1155 : }
1156 :
1157 0 : unlinkat(pindir_fd, program->prog_name, 0);
1158 0 : bf_map_unpin(program->pmap, pindir_fd);
1159 0 : bf_map_unpin(program->cmap, pindir_fd);
1160 0 : bf_list_foreach (&program->sets, set_node)
1161 0 : bf_map_unpin(bf_list_node_get_data(set_node), pindir_fd);
1162 0 : bf_list_foreach (&program->links, link_node)
1163 0 : bf_link_unpin(bf_list_node_get_data(link_node), pindir_fd);
1164 :
1165 : closep(&pindir_fd);
1166 0 : r = rmdir(_dir);
1167 0 : if (r) {
1168 0 : bf_warn_r(r, "failed to remove bf_program pin directory %s, ignoring",
1169 : _dir);
1170 : }
1171 : }
1172 :
1173 0 : static int _bf_program_load_printer_map(struct bf_program *program)
1174 : {
1175 0 : _cleanup_free_ void *pstr = NULL;
1176 : size_t pstr_len;
1177 0 : uint32_t key = 0;
1178 : int r;
1179 :
1180 0 : bf_assert(program);
1181 :
1182 0 : r = bf_printer_assemble(program->printer, &pstr, &pstr_len);
1183 0 : if (r)
1184 0 : return bf_err_r(r, "failed to assemble printer map string");
1185 :
1186 0 : r = bf_map_set_value_size(program->pmap, pstr_len);
1187 0 : if (r < 0)
1188 : return r;
1189 :
1190 0 : r = bf_map_create(program->pmap, 0);
1191 0 : if (r < 0)
1192 : return r;
1193 :
1194 0 : r = bf_map_set_elem(program->pmap, &key, pstr);
1195 0 : if (r)
1196 : return r;
1197 :
1198 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_PRINTER_MAP_FD);
1199 0 : if (r) {
1200 0 : bf_map_destroy(program->pmap);
1201 0 : return bf_err_r(r, "failed to fixup printer map FD");
1202 : }
1203 :
1204 : return 0;
1205 : }
1206 :
1207 0 : static int _bf_program_load_counters_map(struct bf_program *program)
1208 : {
1209 : _cleanup_close_ int _fd = -1;
1210 : int r;
1211 :
1212 0 : bf_assert(program);
1213 :
1214 0 : r = bf_map_set_n_elems(program->cmap, program->num_counters);
1215 0 : if (r < 0)
1216 : return r;
1217 :
1218 0 : r = bf_map_create(program->cmap, 0);
1219 0 : if (r < 0)
1220 : return r;
1221 :
1222 0 : r = _bf_program_fixup(program, BF_FIXUP_TYPE_COUNTERS_MAP_FD);
1223 0 : if (r < 0) {
1224 0 : bf_map_destroy(program->cmap);
1225 0 : return bf_err_r(r, "failed to fixup counters map FD");
1226 : }
1227 :
1228 : return 0;
1229 : }
1230 :
1231 0 : static int _bf_program_load_sets_maps(struct bf_program *new_prog)
1232 : {
1233 : const bf_list_node *set_node;
1234 : const bf_list_node *map_node;
1235 : int r;
1236 :
1237 0 : bf_assert(new_prog);
1238 :
1239 0 : set_node = bf_list_get_head(&new_prog->runtime.chain->sets);
1240 0 : map_node = bf_list_get_head(&new_prog->sets);
1241 :
1242 : // Fill the bf_map with the sets content
1243 0 : while (set_node && map_node) {
1244 : _cleanup_free_ uint8_t *values = NULL;
1245 : _cleanup_free_ uint8_t *keys = NULL;
1246 0 : struct bf_set *set = bf_list_node_get_data(set_node);
1247 0 : struct bf_map *map = bf_list_node_get_data(map_node);
1248 0 : size_t nelems = bf_list_size(&set->elems);
1249 0 : union bpf_attr attr = {};
1250 : size_t idx = 0;
1251 :
1252 0 : r = bf_map_create(map, 0);
1253 0 : if (r < 0) {
1254 0 : r = bf_err_r(r, "failed to create BPF map for set");
1255 0 : goto err_destroy_maps;
1256 : }
1257 :
1258 0 : values = malloc(nelems);
1259 0 : if (!values) {
1260 0 : r = bf_err_r(errno, "failed to allocate map values");
1261 0 : goto err_destroy_maps;
1262 : }
1263 :
1264 0 : keys = malloc(set->elem_size * nelems);
1265 0 : if (!keys) {
1266 0 : r = bf_err_r(errno, "failed to allocate map keys");
1267 0 : goto err_destroy_maps;
1268 : }
1269 :
1270 0 : bf_list_foreach (&set->elems, elem_node) {
1271 0 : void *elem = bf_list_node_get_data(elem_node);
1272 :
1273 0 : memcpy(keys + (idx * set->elem_size), elem, set->elem_size);
1274 0 : values[idx] = 1;
1275 0 : ++idx;
1276 : }
1277 :
1278 0 : attr.batch.map_fd = map->fd;
1279 0 : attr.batch.keys = (unsigned long long)keys;
1280 0 : attr.batch.values = (unsigned long long)values;
1281 0 : attr.batch.count = nelems;
1282 0 : attr.batch.flags = BPF_ANY;
1283 :
1284 0 : r = bf_bpf(BPF_MAP_UPDATE_BATCH, &attr);
1285 0 : if (r < 0) {
1286 0 : bf_err_r(r, "failed to add set elements to the map");
1287 0 : goto err_destroy_maps;
1288 : }
1289 :
1290 0 : set_node = bf_list_node_next(set_node);
1291 0 : map_node = bf_list_node_next(map_node);
1292 : }
1293 :
1294 0 : r = _bf_program_fixup(new_prog, BF_FIXUP_TYPE_SET_MAP_FD);
1295 0 : if (r < 0)
1296 0 : goto err_destroy_maps;
1297 :
1298 : return 0;
1299 :
1300 0 : err_destroy_maps:
1301 0 : bf_list_foreach (&new_prog->sets, map_node)
1302 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1303 : return r;
1304 : }
1305 :
1306 : /**
1307 : * Callback to create a new @ref bf_link when attaching the program to the hook.
1308 : *
1309 : * See @ref bf_flavor_ops for more details.
1310 : *
1311 : * @param prog Program to create the @ref bf_link for. Can't be NULL.
1312 : * @param old_link Existing BPF link from the old program (program to replace).
1313 : * If not NULL, the new link's FD is a duplicate of the old link's FD.
1314 : * This is useful for flavors using @c BPF_LINK_UPDATE : the link remain
1315 : * the same but the program changes, the old link can then be unpinned
1316 : * and closed safely.
1317 : * @param new_link New link to create. Can't be NULL. On success, @c *link
1318 : * points to a valid @ref bf_link . On failure, @c *link remain unchanged.
1319 : * @return 0 on success, or a negative errno value on success.
1320 : */
1321 0 : static int _bf_program_get_new_link(struct bf_program *prog,
1322 : struct bf_link *old_link,
1323 : struct bf_link **new_link)
1324 : {
1325 0 : _cleanup_bf_link_ struct bf_link *_link = NULL;
1326 : char name[BPF_OBJ_NAME_LEN];
1327 : int r;
1328 :
1329 0 : bf_assert(prog && new_link);
1330 :
1331 0 : (void)snprintf(name, BPF_OBJ_NAME_LEN, "%s_lk%1ld", prog->id,
1332 0 : bf_list_size(&prog->links));
1333 :
1334 0 : r = bf_link_new(&_link, name, prog->hook);
1335 0 : if (r)
1336 0 : return bf_err_r(r, "failed to create bf_link from callback");
1337 :
1338 0 : if (old_link) {
1339 0 : r = dup(old_link->fd);
1340 0 : if (r < 0)
1341 0 : return bf_err_r(r, "failed to duplicate old link FD");
1342 :
1343 0 : _link->fd = r;
1344 : }
1345 :
1346 0 : r = bf_list_add_tail(&prog->links, _link);
1347 0 : if (r) {
1348 0 : return bf_err_r(
1349 : r, "failed to append new bf_link to program from callback");
1350 : }
1351 :
1352 0 : *new_link = TAKE_PTR(_link);
1353 :
1354 0 : return 0;
1355 : }
1356 :
1357 0 : int bf_program_load(struct bf_program *new_prog, struct bf_program *old_prog)
1358 : {
1359 : char dir[PATH_MAX];
1360 : char tmpdir[PATH_MAX];
1361 : int r;
1362 :
1363 0 : bf_assert(new_prog);
1364 :
1365 0 : r = _bf_program_load_sets_maps(new_prog);
1366 0 : if (r < 0)
1367 : return r;
1368 :
1369 0 : r = _bf_program_load_counters_map(new_prog);
1370 0 : if (r)
1371 : return r;
1372 :
1373 0 : r = _bf_program_load_printer_map(new_prog);
1374 0 : if (r)
1375 : return r;
1376 :
1377 0 : if (bf_opts_is_verbose(BF_VERBOSE_BYTECODE))
1378 0 : bf_program_dump_bytecode(new_prog);
1379 :
1380 0 : r = bf_bpf_prog_load(
1381 0 : new_prog->prog_name, bf_hook_to_bpf_prog_type(new_prog->hook),
1382 0 : new_prog->img, new_prog->img_size,
1383 : bf_hook_to_attach_type(new_prog->hook), &new_prog->runtime.prog_fd);
1384 0 : if (r)
1385 0 : return bf_err_r(r, "failed to load new bf_program");
1386 :
1387 0 : if (old_prog) {
1388 0 : (void)snprintf(dir, PATH_MAX, "%s/%s", BF_PIN_DIR, new_prog->id);
1389 0 : (void)snprintf(tmpdir, PATH_MAX, "%s/%s_tmp", BF_PIN_DIR, new_prog->id);
1390 : }
1391 :
1392 0 : if (old_prog && !bf_opts_transient() && (r = rename(dir, tmpdir)))
1393 0 : return bf_err_r(r, "failed to rename old bf_program pin directory");
1394 :
1395 0 : if (new_prog->runtime.chain->hook_opts.attach) {
1396 0 : r = new_prog->runtime.ops->attach_prog(new_prog, old_prog,
1397 : _bf_program_get_new_link);
1398 0 : if (r) {
1399 0 : bf_err_r(r, "failed to attach new program");
1400 0 : goto err_restore_old;
1401 : }
1402 : }
1403 :
1404 0 : if (!bf_opts_transient()) {
1405 0 : r = _bf_program_pin(new_prog);
1406 0 : if (r)
1407 0 : return bf_err_r(r, "failed to pin new program!");
1408 :
1409 0 : if (old_prog)
1410 0 : _bf_program_unpin(old_prog, tmpdir);
1411 : }
1412 :
1413 : return r;
1414 :
1415 : err_restore_old:
1416 : int _r = r;
1417 0 : if ((r = rename(tmpdir, dir))) {
1418 0 : bf_err_r(r,
1419 : "failed to restore old program, the daemon is in bad state!");
1420 : } else {
1421 0 : bf_info("previous program restored");
1422 : }
1423 : return _r;
1424 : }
1425 :
1426 0 : int bf_program_unload(struct bf_program *program)
1427 : {
1428 : int r;
1429 :
1430 0 : bf_assert(program);
1431 :
1432 0 : if (program->runtime.chain->hook_opts.attach) {
1433 0 : r = program->runtime.ops->detach_prog(program);
1434 0 : if (r)
1435 : return r;
1436 : }
1437 :
1438 0 : if (!bf_opts_transient())
1439 0 : _bf_program_unpin(program, NULL);
1440 :
1441 : closep(&program->runtime.prog_fd);
1442 :
1443 0 : bf_map_destroy(program->cmap);
1444 0 : bf_map_destroy(program->pmap);
1445 :
1446 0 : bf_list_foreach (&program->sets, map_node)
1447 0 : bf_map_destroy(bf_list_node_get_data(map_node));
1448 :
1449 0 : bf_list_foreach (&program->links, link_node)
1450 0 : bf_link_detach(bf_list_node_get_data(link_node));
1451 :
1452 0 : bf_dbg("unloaded %s program from %s", bf_front_to_str(program->front),
1453 : bf_hook_to_str(program->hook));
1454 :
1455 : return 0;
1456 : }
1457 :
1458 0 : int bf_program_get_counter(const struct bf_program *program,
1459 : uint32_t counter_idx, struct bf_counter *counter)
1460 : {
1461 0 : bf_assert(program);
1462 0 : bf_assert(counter);
1463 :
1464 : int r;
1465 :
1466 0 : r = bf_bpf_map_lookup_elem(program->cmap->fd, &counter_idx, counter);
1467 0 : if (r < 0)
1468 0 : return bf_err_r(errno, "failed to lookup counters map");
1469 :
1470 : return 0;
1471 : }
1472 :
1473 0 : int bf_cgen_set_counters(struct bf_program *program,
1474 : const struct bf_counter *counters)
1475 : {
1476 : UNUSED(program);
1477 : UNUSED(counters);
1478 :
1479 0 : return -ENOTSUP;
1480 : }
|
