Branch data Line data Source code
1 : : /* SPDX-License-Identifier: GPL-2.0-only */
2 : : /*
3 : : * Copyright (c) Meta Platforms, Inc. and affiliates.
4 : : */
5 : :
6 : : #include "cgen/cgroup_sock_addr.h"
7 : :
8 : : #include <linux/bpf.h>
9 : : #include <linux/bpf_common.h>
10 : : #include <linux/if_ether.h>
11 : :
12 : : #include <assert.h>
13 : : #include <bpf/libbpf.h>
14 : : #include <stddef.h>
15 : : #include <stdint.h>
16 : :
17 : : #include <bpfilter/btf.h>
18 : : #include <bpfilter/chain.h>
19 : : #include <bpfilter/elfstub.h>
20 : : #include <bpfilter/flavor.h>
21 : : #include <bpfilter/hook.h>
22 : : #include <bpfilter/logger.h>
23 : : #include <bpfilter/matcher.h>
24 : : #include <bpfilter/rule.h>
25 : : #include <bpfilter/runtime.h>
26 : : #include <bpfilter/set.h>
27 : : #include <bpfilter/verdict.h>
28 : :
29 : : #include "cgen/jmp.h"
30 : : #include "cgen/matcher/cmp.h"
31 : : #include "cgen/matcher/meta.h"
32 : : #include "cgen/matcher/set.h"
33 : : #include "cgen/program.h"
34 : : #include "cgen/runtime.h"
35 : : #include "cgen/stub.h"
36 : : #include "filter.h"
37 : :
38 : : // Forward definition to avoid header conflicts.
39 : : uint16_t htons(uint16_t hostshort);
40 : :
41 : 84 : static int _bf_cgroup_sock_addr_gen_inline_prologue(struct bf_program *program)
42 : : {
43 : : assert(program);
44 : :
45 : : /* `R6` = `bpf_sock_addr` context pointer. Unlike packet-based flavors where
46 : : * `R6` changes per header, the socket context is fixed so we set it once. */
47 [ + - ]: 84 : EMIT(program, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
48 : :
49 : : // The counters stub reads `pkt_size` unconditionally; zero it out.
50 [ + - ]: 84 : EMIT(program, BPF_ST_MEM(BPF_DW, BPF_REG_10, BF_PROG_CTX_OFF(pkt_size), 0));
51 : :
52 : : /* Take the L3 protocol from the hook, not from `bpf_sock_addr.family`: the
53 : : * kernel also runs the IPv4 hooks for dual-stack `AF_INET6` UDP sockets. */
54 [ + + - ]: 84 : switch (program->runtime.chain->hook) {
55 : 48 : case BF_HOOK_CGROUP_SOCK_ADDR_CONNECT4:
56 : : case BF_HOOK_CGROUP_SOCK_ADDR_SENDMSG4:
57 [ + - ]: 48 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, htons(ETH_P_IP)));
58 : : break;
59 : 36 : case BF_HOOK_CGROUP_SOCK_ADDR_CONNECT6:
60 : : case BF_HOOK_CGROUP_SOCK_ADDR_SENDMSG6:
61 [ + - ]: 36 : EMIT(program, BPF_MOV64_IMM(BPF_REG_7, htons(ETH_P_IPV6)));
62 : : break;
63 : 0 : default:
64 [ # # ]: 0 : return bf_err_r(-ENOTSUP, "unexpected hook: %s",
65 : : bf_hook_to_str(program->runtime.chain->hook));
66 : : }
67 : :
68 : 84 : EMIT(program, BPF_LDX_MEM(BPF_W, BPF_REG_8, BPF_REG_6,
69 : : offsetof(struct bpf_sock_addr, protocol)));
70 : :
71 : : return 0;
72 : : }
73 : :
74 : 84 : static int _bf_cgroup_sock_addr_gen_inline_epilogue(struct bf_program *program)
75 : : {
76 : : (void)program;
77 : :
78 : 84 : return 0;
79 : : }
80 : :
81 : : /**
82 : : * @brief Load a field from the `bpf_sock_addr` context into a register.
83 : : *
84 : : * `R6` must already point to the context. For 16-byte fields, the low
85 : : * 8 bytes go into `reg` and the high 8 bytes into `reg + 1`.
86 : : *
87 : : * When the field offset is not 8-byte aligned, 8- and 16-byte loads fall
88 : : * back to 4-byte reads packed via shift/or. This clobbers `reg + 1` for
89 : : * 8-byte loads and `reg + 2` for 16-byte loads.
90 : : *
91 : : * @param program Program to emit into. Can't be NULL.
92 : : * @param offset Byte offset into `struct bpf_sock_addr`.
93 : : * @param size Field size in bytes: 1, 2, 4, 8, or 16.
94 : : * @param reg BPF register to load the value into.
95 : : * @return 0 on success, negative errno on error.
96 : : */
97 : 94 : static int _bf_cgroup_sock_addr_load_field(struct bf_program *program,
98 : : size_t offset, size_t size, int reg)
99 : : {
100 : : assert(program);
101 : :
102 [ - - + - : 94 : switch (size) {
+ - ]
103 : 0 : case 1:
104 : 0 : EMIT(program, BPF_LDX_MEM(BPF_B, reg, BPF_REG_6, offset));
105 : : break;
106 : 0 : case 2:
107 : 0 : EMIT(program, BPF_LDX_MEM(BPF_H, reg, BPF_REG_6, offset));
108 : : break;
109 : 77 : case 4:
110 : 77 : EMIT(program, BPF_LDX_MEM(BPF_W, reg, BPF_REG_6, offset));
111 : : break;
112 : 0 : case 8:
113 [ # # ]: 0 : if (offset % 8 == 0) {
114 : 0 : EMIT(program, BPF_LDX_MEM(BPF_DW, reg, BPF_REG_6, offset));
115 : : } else {
116 [ # # ]: 0 : EMIT(program, BPF_LDX_MEM(BPF_W, reg, BPF_REG_6, offset));
117 [ # # ]: 0 : EMIT(program, BPF_LDX_MEM(BPF_W, reg + 1, BPF_REG_6, offset + 4));
118 [ # # ]: 0 : EMIT(program, BPF_ALU64_IMM(BPF_LSH, reg + 1, 32));
119 : 0 : EMIT(program, BPF_ALU64_REG(BPF_OR, reg, reg + 1));
120 : : }
121 : : break;
122 : 17 : case 16:
123 [ + + ]: 17 : if (offset % 8 == 0) {
124 [ + - ]: 14 : EMIT(program, BPF_LDX_MEM(BPF_DW, reg, BPF_REG_6, offset));
125 : 14 : EMIT(program, BPF_LDX_MEM(BPF_DW, reg + 1, BPF_REG_6, offset + 8));
126 : : } else {
127 [ + - ]: 3 : EMIT(program, BPF_LDX_MEM(BPF_W, reg, BPF_REG_6, offset));
128 [ + - ]: 3 : EMIT(program, BPF_LDX_MEM(BPF_W, reg + 2, BPF_REG_6, offset + 4));
129 [ + - ]: 3 : EMIT(program, BPF_ALU64_IMM(BPF_LSH, reg + 2, 32));
130 [ + - ]: 3 : EMIT(program, BPF_ALU64_REG(BPF_OR, reg, reg + 2));
131 [ + - ]: 3 : EMIT(program, BPF_LDX_MEM(BPF_W, reg + 1, BPF_REG_6, offset + 8));
132 [ + - ]: 3 : EMIT(program, BPF_LDX_MEM(BPF_W, reg + 2, BPF_REG_6, offset + 12));
133 [ + - ]: 3 : EMIT(program, BPF_ALU64_IMM(BPF_LSH, reg + 2, 32));
134 : 3 : EMIT(program, BPF_ALU64_REG(BPF_OR, reg + 1, reg + 2));
135 : : }
136 : : break;
137 : : default:
138 : : return -EINVAL;
139 : : }
140 : :
141 : : return 0;
142 : : }
143 : :
144 : : /**
145 : : * @brief Store a register value at an offset from `BPF_REG_10`.
146 : : *
147 : : * Counterpart to `_bf_cgroup_sock_addr_load_field`. For 16-byte stores,
148 : : * `reg` holds the low 8 bytes and `reg + 1` the high 8 bytes, matching
149 : : * the layout produced by `_bf_cgroup_sock_addr_load_field`.
150 : : *
151 : : * @param program Program to emit into. Can't be NULL.
152 : : * @param offset Byte offset from `BPF_REG_10`.
153 : : * @param size Field size in bytes: 1, 2, 4, 8, or 16.
154 : : * @param reg BPF register holding the value to store.
155 : : * @return 0 on success, negative errno on error.
156 : : */
157 : 10 : static int _bf_cgroup_sock_addr_store_field(struct bf_program *program,
158 : : int offset, size_t size, int reg)
159 : : {
160 : : assert(program);
161 : :
162 [ - + + - : 10 : switch (size) {
+ - ]
163 : 0 : case 1:
164 : 0 : EMIT(program, BPF_STX_MEM(BPF_B, BPF_REG_10, reg, offset));
165 : : break;
166 : 4 : case 2:
167 : 4 : EMIT(program, BPF_STX_MEM(BPF_H, BPF_REG_10, reg, offset));
168 : : break;
169 : 3 : case 4:
170 : 3 : EMIT(program, BPF_STX_MEM(BPF_W, BPF_REG_10, reg, offset));
171 : : break;
172 : 0 : case 8:
173 : 0 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, reg, offset));
174 : : break;
175 : 3 : case 16:
176 [ + - ]: 3 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, reg, offset));
177 : 3 : EMIT(program, BPF_STX_MEM(BPF_DW, BPF_REG_10, reg + 1, offset + 8));
178 : : break;
179 : : default:
180 : : return -EINVAL;
181 : : }
182 : :
183 : : return 0;
184 : : }
185 : :
186 : 15 : static int _bf_cgroup_sock_addr_load_and_cmp(struct bf_program *program,
187 : : const struct bf_matcher *matcher,
188 : : size_t offset, size_t size)
189 : : {
190 : : int r;
191 : :
192 : : assert(program);
193 : : assert(matcher);
194 : :
195 : 15 : r = _bf_cgroup_sock_addr_load_field(program, offset, size, BPF_REG_1);
196 [ + - ]: 15 : if (r)
197 : : return r;
198 : :
199 : 15 : return bf_cmp_value(program, matcher, bf_matcher_payload(matcher), size,
200 : : BPF_REG_1);
201 : : }
202 : :
203 : 13 : static int _bf_cgroup_sock_addr_generate_net(struct bf_program *program,
204 : : const struct bf_matcher *matcher,
205 : : size_t offset, size_t size)
206 : : {
207 : : uint32_t prefixlen;
208 : : const void *data;
209 : : int r;
210 : :
211 : : assert(program);
212 : : assert(matcher);
213 : :
214 : 13 : prefixlen = *(const uint32_t *)bf_matcher_payload(matcher);
215 : 13 : data = (const uint8_t *)bf_matcher_payload(matcher) + sizeof(uint32_t);
216 : :
217 : 13 : r = _bf_cgroup_sock_addr_load_field(program, offset, size, BPF_REG_1);
218 [ + - ]: 13 : if (r)
219 : : return r;
220 : :
221 : 13 : return bf_cmp_masked_value(program, matcher, data, prefixlen, size,
222 : : BPF_REG_1);
223 : : }
224 : :
225 : : /* `user_port` is a __u32 in network byte order with the upper 16 bits
226 : : * guaranteed zero by the kernel ABI. Loaded as `BPF_W` so EQ/NE compare
227 : : * the full 32-bit register (safe because upper bits are zero). For range
228 : : * comparisons, `BSWAP` converts to host order (and zeroes the upper bits). */
229 : 56 : static int _bf_cgroup_sock_addr_generate_port(struct bf_program *program,
230 : : const struct bf_matcher *matcher)
231 : : {
232 : : int r;
233 : :
234 : : assert(program);
235 : : assert(matcher);
236 : :
237 : 56 : r = _bf_cgroup_sock_addr_load_field(
238 : : program, offsetof(struct bpf_sock_addr, user_port), 4, BPF_REG_1);
239 [ + - ]: 56 : if (r)
240 : : return r;
241 : :
242 [ + + ]: 56 : if (bf_matcher_get_op(matcher) == BF_MATCHER_RANGE) {
243 : 16 : uint16_t *ports = (uint16_t *)bf_matcher_payload(matcher);
244 [ + - ]: 16 : EMIT(program, BPF_BSWAP(BPF_REG_1, 16));
245 : 16 : return bf_cmp_range(program, matcher, ports[0], ports[1], BPF_REG_1);
246 : : }
247 : :
248 : 40 : return bf_cmp_value(program, matcher, bf_matcher_payload(matcher), 2,
249 : : BPF_REG_1);
250 : : }
251 : :
252 : : static ssize_t _bf_cgroup_sock_addr_ctx_offset(enum bf_matcher_type type)
253 : : {
254 : : switch (type) {
255 : : case BF_MATCHER_IP4_SADDR:
256 : : case BF_MATCHER_IP4_SNET:
257 : : return offsetof(struct bpf_sock_addr, msg_src_ip4);
258 : : case BF_MATCHER_IP4_DADDR:
259 : : case BF_MATCHER_IP4_DNET:
260 : : return offsetof(struct bpf_sock_addr, user_ip4);
261 : : case BF_MATCHER_IP6_SADDR:
262 : : case BF_MATCHER_IP6_SNET:
263 : : return offsetof(struct bpf_sock_addr, msg_src_ip6);
264 : : case BF_MATCHER_IP6_DADDR:
265 : : case BF_MATCHER_IP6_DNET:
266 : : return offsetof(struct bpf_sock_addr, user_ip6);
267 : : case BF_MATCHER_IP4_PROTO:
268 : : return offsetof(struct bpf_sock_addr, protocol);
269 : : case BF_MATCHER_META_DPORT:
270 : : case BF_MATCHER_TCP_DPORT:
271 : : case BF_MATCHER_UDP_DPORT:
272 : : return offsetof(struct bpf_sock_addr, user_port);
273 : : default:
274 : : return -ENOTSUP;
275 : : }
276 : : }
277 : :
278 : 24 : static int _bf_cgroup_sock_addr_generate_set(struct bf_program *program,
279 : : const struct bf_matcher *matcher)
280 : : {
281 : : const struct bf_set *set;
282 : : size_t offset = 0;
283 : : int r;
284 : :
285 : : assert(program);
286 : : assert(matcher);
287 : :
288 : 24 : set = bf_chain_get_set_for_matcher(program->runtime.chain, matcher);
289 [ - + ]: 24 : if (!set) {
290 [ # # ]: 0 : return bf_err_r(-ENOENT, "matcher set not found in %s",
291 : : program->runtime.chain->name);
292 : : }
293 : :
294 [ + + ]: 24 : if (set->use_trie) {
295 : 4 : const struct bf_matcher_meta *meta = bf_matcher_get_meta(set->key[0]);
296 [ + - ]: 4 : ssize_t ctx_off = _bf_cgroup_sock_addr_ctx_offset(set->key[0]);
297 : :
298 [ - + ]: 4 : if (!meta) {
299 [ # # ]: 0 : return bf_err_r(-EINVAL, "missing meta for set component '%s'",
300 : : bf_matcher_type_to_str(set->key[0]));
301 : : }
302 : :
303 [ - + ]: 4 : if (ctx_off < 0) {
304 [ # # ]: 0 : return bf_err_r(
305 : : (int)ctx_off,
306 : : "set component '%s' not supported for cgroup_sock_addr",
307 : : bf_matcher_type_to_str(set->key[0]));
308 : : }
309 : :
310 : 4 : return bf_set_generate_trie_lookup(program, matcher, (size_t)ctx_off,
311 : 4 : meta->hdr_payload_size);
312 : : }
313 : :
314 [ + + ]: 54 : for (size_t i = 0; i < set->n_comps; ++i) {
315 : 34 : enum bf_matcher_type type = set->key[i];
316 : 34 : const struct bf_matcher_meta *meta = bf_matcher_get_meta(type);
317 : : ssize_t ctx_off = _bf_cgroup_sock_addr_ctx_offset(type);
318 : :
319 [ - + ]: 34 : if (!meta) {
320 [ # # ]: 0 : return bf_err_r(-EINVAL, "missing meta for set component '%s'",
321 : : bf_matcher_type_to_str(type));
322 : : }
323 : :
324 : : /* The PID is not part of the context, a helper returns it. The key
325 : : * components are packed, so the store offset might be misaligned:
326 : : * `bf_stub_store` splits the store accordingly. */
327 [ + + ]: 34 : if (type == BF_MATCHER_META_PID) {
328 [ + - ]: 6 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_current_pid_tgid));
329 [ + - ]: 6 : EMIT(program, BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, 32));
330 : :
331 : 6 : r = bf_stub_store(program, BPF_REG_0, meta->hdr_payload_size,
332 : 6 : BF_PROG_SCR_OFF(offset));
333 [ + - ]: 6 : if (r)
334 : : return r;
335 : :
336 : 6 : offset += meta->hdr_payload_size;
337 : 6 : continue;
338 : : }
339 : :
340 [ - + ]: 28 : if (ctx_off < 0) {
341 [ # # ]: 0 : return bf_err_r(
342 : : (int)ctx_off,
343 : : "set component '%s' not supported for cgroup_sock_addr",
344 : : bf_matcher_type_to_str(type));
345 : : }
346 : :
347 : : /* The BPF verifier enforces specific ctx access widths on
348 : : * `bpf_sock_addr` fields. `bf_stub_load()` reads
349 : : * `meta->hdr_payload_size` bytes from ctx:
350 : : * - Ports (`hdr_payload_size == 2`): `user_port` is a 4-byte
351 : : * `__u32`, but the 2-byte narrow read is accepted and rewritten
352 : : * to the NBO port value.
353 : : * - Protocol (`hdr_payload_size == 1`): only 4-byte reads are
354 : : * allowed, so a 1-byte `bf_stub_load()` would be rejected. Reuse
355 : : * `BPF_REG_8`, which the prologue loaded with a 4-byte read. */
356 [ + + ]: 28 : if (type == BF_MATCHER_IP4_PROTO) {
357 [ + - ]: 2 : EMIT(program, BPF_STX_MEM(BPF_B, BPF_REG_10, BPF_REG_8,
358 : : BF_PROG_SCR_OFF(offset)));
359 : : } else {
360 : 26 : r = bf_stub_load(program, (size_t)ctx_off, meta->hdr_payload_size,
361 : 26 : BF_PROG_SCR_OFF(offset));
362 [ + - ]: 26 : if (r)
363 : : return r;
364 : : }
365 : :
366 : 28 : offset += meta->hdr_payload_size;
367 : : }
368 : :
369 : 20 : return bf_set_generate_map_lookup(program, matcher, BF_PROG_SCR_OFF(0));
370 : : }
371 : :
372 : : static int
373 : 148 : _bf_cgroup_sock_addr_gen_inline_matcher(struct bf_program *program,
374 : : const struct bf_matcher *matcher)
375 : : {
376 : : assert(program);
377 : : assert(matcher);
378 : :
379 [ + + + + : 148 : switch (bf_matcher_get_type(matcher)) {
+ + + + +
+ + + +
- ]
380 : 27 : case BF_MATCHER_META_L3_PROTO:
381 : : case BF_MATCHER_META_L4_PROTO:
382 : : case BF_MATCHER_META_PROBABILITY:
383 : 27 : return bf_matcher_generate_meta(program, matcher);
384 : 8 : case BF_MATCHER_META_PID:
385 [ + - ]: 8 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_current_pid_tgid));
386 [ + - ]: 8 : EMIT(program, BPF_ALU64_IMM(BPF_RSH, BPF_REG_0, 32));
387 : 8 : return bf_cmp_value(program, matcher, bf_matcher_payload(matcher), 4,
388 : : BPF_REG_0);
389 : 1 : case BF_MATCHER_IP4_SADDR:
390 : 1 : return _bf_cgroup_sock_addr_load_and_cmp(
391 : : program, matcher, offsetof(struct bpf_sock_addr, msg_src_ip4), 4);
392 : 1 : case BF_MATCHER_IP4_SNET:
393 : 1 : return _bf_cgroup_sock_addr_generate_net(
394 : : program, matcher, offsetof(struct bpf_sock_addr, msg_src_ip4), 4);
395 : 7 : case BF_MATCHER_IP4_DADDR:
396 : 7 : return _bf_cgroup_sock_addr_load_and_cmp(
397 : : program, matcher, offsetof(struct bpf_sock_addr, user_ip4), 4);
398 : 5 : case BF_MATCHER_IP4_DNET:
399 : 5 : return _bf_cgroup_sock_addr_generate_net(
400 : : program, matcher, offsetof(struct bpf_sock_addr, user_ip4), 4);
401 : 5 : case BF_MATCHER_IP4_PROTO:
402 [ + - ]: 5 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_8));
403 : 5 : return bf_cmp_value(program, matcher, bf_matcher_payload(matcher), 1,
404 : : BPF_REG_1);
405 : 1 : case BF_MATCHER_IP6_SADDR:
406 : 1 : return _bf_cgroup_sock_addr_load_and_cmp(
407 : : program, matcher, offsetof(struct bpf_sock_addr, msg_src_ip6), 16);
408 : 1 : case BF_MATCHER_IP6_SNET:
409 : 1 : return _bf_cgroup_sock_addr_generate_net(
410 : : program, matcher, offsetof(struct bpf_sock_addr, msg_src_ip6), 16);
411 : 6 : case BF_MATCHER_IP6_DADDR:
412 : 6 : return _bf_cgroup_sock_addr_load_and_cmp(
413 : : program, matcher, offsetof(struct bpf_sock_addr, user_ip6), 16);
414 : 6 : case BF_MATCHER_IP6_DNET:
415 : 6 : return _bf_cgroup_sock_addr_generate_net(
416 : : program, matcher, offsetof(struct bpf_sock_addr, user_ip6), 16);
417 : 56 : case BF_MATCHER_META_DPORT:
418 : : case BF_MATCHER_TCP_DPORT:
419 : : case BF_MATCHER_UDP_DPORT:
420 : 56 : return _bf_cgroup_sock_addr_generate_port(program, matcher);
421 : 24 : case BF_MATCHER_SET:
422 : 24 : return _bf_cgroup_sock_addr_generate_set(program, matcher);
423 : 0 : default:
424 [ # # ]: 0 : return bf_err_r(-ENOTSUP,
425 : : "matcher '%s' not supported for cgroup_sock_addr",
426 : : bf_matcher_type_to_str(bf_matcher_get_type(matcher)));
427 : : }
428 : : }
429 : :
430 : : /**
431 : : * @brief Convert a standard verdict into a return value.
432 : : *
433 : : * @param verdict Verdict to convert. Must be valid.
434 : : * @param ret_code Cgroup return code. Can't be NULL.
435 : : * @return 0 on success, or a negative errno value on failure.
436 : : */
437 : 183 : static int _bf_cgroup_sock_addr_get_verdict(enum bf_verdict verdict,
438 : : int *ret_code)
439 : : {
440 : : assert(ret_code);
441 : :
442 [ + + - ]: 183 : switch (verdict) {
443 : 101 : case BF_VERDICT_ACCEPT:
444 : : case BF_VERDICT_NEXT:
445 : 101 : *ret_code = 1;
446 : 101 : return 0;
447 : 82 : case BF_VERDICT_DROP:
448 : 82 : *ret_code = 0;
449 : 82 : return 0;
450 : : default:
451 : : return -ENOTSUP;
452 : : }
453 : : }
454 : :
455 : : #define _BF_SOCK_ADDR_NS_PID_OFF \
456 : : BF_PROG_SCR_OFF(offsetof(struct bf_runtime_sock_addr, ns_pid))
457 : :
458 : : /* The kernel pointers dereferenced to reach the namespace-local PID are staged
459 : : * in the scratch area, right after `bf_runtime_sock_addr`. */
460 : : #define _BF_SOCK_ADDR_NS_PID_WALK_OFF \
461 : : BF_PROG_SCR_OFF(sizeof(struct bf_runtime_sock_addr))
462 : :
463 : : static_assert(sizeof(struct bf_runtime_sock_addr) + sizeof(__u64) <= 64,
464 : : "no room left in the scratch area for the PID namespace walk");
465 : :
466 : : /**
467 : : * @brief Store the process' namespace-local PID into the staging area.
468 : : *
469 : : * `bpf_get_current_pid_tgid()` reports the PID as seen from the initial PID
470 : : * namespace. The namespace-local one is only reachable through the kernel
471 : : * structures: `current->group_leader->thread_pid->numbers[level].nr`, with
472 : : * `level` the depth of the process' own PID namespace. `level` is 0 in the
473 : : * initial namespace, where this yields the same value as
474 : : * `bpf_get_current_pid_tgid()`.
475 : : *
476 : : * `numbers` is indexed with a runtime value, and the verifier rejects variable
477 : : * offsets on BTF pointers, so the walk uses `bpf_probe_read_kernel()` and
478 : : * computes the addresses as scalars.
479 : : *
480 : : * `ns_pid` is left to 0 if `level` is 0: the process' only PID is the one
481 : : * reported by `bpf_get_current_pid_tgid()`. The PID allocator never hands out
482 : : * 0, so 0 unambiguously means "no namespace-local PID to report", including
483 : : * when the walk failed: `bpf_probe_read_kernel()` zeroes its destination on
484 : : * failure, so an unreadable address leaves the slot zeroed instead of stale.
485 : : *
486 : : * @todo `pid.level` and `upid.nr` are assumed to be 4 bytes: `level` is loaded
487 : : * back from the slot with `BPF_W`, and `ns_pid` is a `__u32`. Validate their
488 : : * size against the BTF data, so a kernel changing either of them fails the
489 : : * program generation instead of logging a wrong PID.
490 : : *
491 : : * @param program Program to emit the instructions into. Can't be NULL.
492 : : * @return 0 on success, or a negative errno value on failure.
493 : : */
494 : 4 : static int _bf_cgroup_sock_addr_store_ns_pid(struct bf_program *program)
495 : : {
496 : : int group_leader_off;
497 : : int thread_pid_off;
498 : : int level_off;
499 : : int r;
500 : :
501 : : assert(program);
502 : :
503 : : // Keep the pre-zeroed ns_pid when lockdown blocks kernel reads.
504 : 4 : r = libbpf_probe_bpf_helper(BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
505 : : BPF_FUNC_probe_read_kernel, NULL);
506 [ + - ]: 4 : if (r <= 0)
507 : : return r;
508 : :
509 : : /* The pointers are only needed to reach the next one, they are read into
510 : : * the walk slot. `level` and the PID itself are read into the `ns_pid`
511 : : * slot, which then holds the 0 to report if the walk stops at the initial
512 : : * namespace. */
513 : :
514 : : // Walk: current->group_leader
515 : 4 : group_leader_off = bf_btf_get_field_off("task_struct", "group_leader");
516 [ + - ]: 4 : if (group_leader_off < 0)
517 : : return group_leader_off;
518 : :
519 [ + - ]: 4 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_get_current_task));
520 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
521 [ + - ]: 4 : EMIT(program,
522 : : BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, _BF_SOCK_ADDR_NS_PID_WALK_OFF));
523 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, 8));
524 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_3, BPF_REG_0));
525 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, group_leader_off));
526 [ + - ]: 4 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
527 : :
528 : : // Walk: current->group_leader->thread_pid
529 : 4 : thread_pid_off = bf_btf_get_field_off("task_struct", "thread_pid");
530 [ + - ]: 4 : if (thread_pid_off < 0)
531 : : return thread_pid_off;
532 : :
533 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
534 [ + - ]: 4 : EMIT(program,
535 : : BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, _BF_SOCK_ADDR_NS_PID_WALK_OFF));
536 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, 8));
537 [ + - ]: 4 : EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_10,
538 : : _BF_SOCK_ADDR_NS_PID_WALK_OFF));
539 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, thread_pid_off));
540 [ + - ]: 4 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
541 : :
542 : : // Walk: current->group_leader->thread_pid->level
543 : 4 : level_off = bf_btf_get_field_off("pid", "level");
544 [ + - ]: 4 : if (level_off < 0)
545 : : return level_off;
546 : :
547 : : /* R9 is callee-saved and free at this point: the rate limiter in
548 : : * `_bf_program_generate_log()` is done with it. It keeps `thread_pid`
549 : : * across the `level` read, to index `numbers` with it. */
550 [ + - ]: 4 : EMIT(program, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_10,
551 : : _BF_SOCK_ADDR_NS_PID_WALK_OFF));
552 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
553 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, _BF_SOCK_ADDR_NS_PID_OFF));
554 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, 4));
555 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_3, BPF_REG_9));
556 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, level_off));
557 [ + - ]: 4 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
558 : :
559 : : /* Walk: current->group_leader->thread_pid->numbers[level].nr, skipped
560 : : * for the initial namespace: the slot then keeps the level, which is the
561 : : * 0 to report. */
562 [ + - ]: 4 : EMIT(program,
563 : : BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_10, _BF_SOCK_ADDR_NS_PID_OFF));
564 : :
565 : : {
566 : : int numbers_off;
567 : : int nr_off;
568 : : int upid_size;
569 : 4 : _clean_bf_jmpctx_ struct bf_jmpctx _ =
570 [ + - ]: 4 : bf_jmpctx_get(program, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 0));
571 : :
572 : 4 : numbers_off = bf_btf_get_field_off("pid", "numbers");
573 [ + - ]: 4 : if (numbers_off < 0)
574 : : return numbers_off;
575 : :
576 : 4 : nr_off = bf_btf_get_field_off("upid", "nr");
577 [ + - ]: 4 : if (nr_off < 0)
578 : : return nr_off;
579 : :
580 : 4 : upid_size = bf_btf_get_type_size("upid");
581 [ + - ]: 4 : if (upid_size < 0)
582 : : return upid_size;
583 : :
584 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
585 [ + - ]: 4 : EMIT(program,
586 : : BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, _BF_SOCK_ADDR_NS_PID_OFF));
587 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, 4));
588 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, upid_size));
589 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, numbers_off + nr_off));
590 [ + - ]: 4 : EMIT(program, BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_9));
591 [ + - ]: 4 : EMIT(program, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
592 : : }
593 : :
594 : 4 : return 0;
595 : : }
596 : :
597 : 4 : static int _bf_cgroup_sock_addr_gen_inline_log(struct bf_program *program,
598 : : const struct bf_rule *rule)
599 : : {
600 : : uint8_t captured_fields = 0;
601 : : bool has_saddr = false;
602 : : size_t addr_size = 0;
603 : : size_t saddr_off = 0;
604 : : size_t daddr_off = 0;
605 : : int r;
606 : :
607 : : assert(program);
608 : : assert(rule);
609 : :
610 : : // Zero the staging area: connect hooks have no source address,
611 : : // and IPv4 hooks only write 4 of the 16 address bytes.
612 [ + + ]: 28 : for (int i = 0; i < (int)sizeof(struct bf_runtime_sock_addr); i += 8)
613 [ + - ]: 24 : EMIT(program, BPF_ST_MEM(BPF_DW, BPF_REG_10, BF_PROG_SCR_OFF(i), 0));
614 : :
615 [ + + - + : 4 : switch (program->runtime.chain->hook) {
+ ]
616 : : case BF_HOOK_CGROUP_SOCK_ADDR_SENDMSG4:
617 : : has_saddr = true;
618 : : saddr_off = offsetof(struct bpf_sock_addr, msg_src_ip4);
619 : : __attribute__((fallthrough));
620 : : case BF_HOOK_CGROUP_SOCK_ADDR_CONNECT4:
621 : : addr_size = 4;
622 : : daddr_off = offsetof(struct bpf_sock_addr, user_ip4);
623 : : break;
624 : : case BF_HOOK_CGROUP_SOCK_ADDR_SENDMSG6:
625 : : has_saddr = true;
626 : : saddr_off = offsetof(struct bpf_sock_addr, msg_src_ip6);
627 : : __attribute__((fallthrough));
628 : : case BF_HOOK_CGROUP_SOCK_ADDR_CONNECT6:
629 : : addr_size = 16;
630 : : daddr_off = offsetof(struct bpf_sock_addr, user_ip6);
631 : : break;
632 : 0 : default:
633 [ # # ]: 0 : return bf_err_r(-ENOTSUP, "unexpected hook: %s",
634 : : bf_hook_to_str(program->runtime.chain->hook));
635 : : }
636 : :
637 : : if (has_saddr) {
638 : : captured_fields |= BF_LOG_SOCK_ADDR_SADDR;
639 : 2 : r = _bf_cgroup_sock_addr_load_field(program, saddr_off, addr_size,
640 : : BPF_REG_1);
641 [ + - ]: 2 : if (r)
642 : : return r;
643 : 2 : r = _bf_cgroup_sock_addr_store_field(
644 : : program,
645 : : BF_PROG_SCR_OFF(offsetof(struct bf_runtime_sock_addr, saddr)),
646 : : addr_size, BPF_REG_1);
647 [ + - ]: 2 : if (r)
648 : : return r;
649 : : }
650 : :
651 : 4 : r = _bf_cgroup_sock_addr_load_field(program, daddr_off, addr_size,
652 : : BPF_REG_1);
653 [ + - ]: 4 : if (r)
654 : : return r;
655 : 4 : r = _bf_cgroup_sock_addr_store_field(
656 : : program, BF_PROG_SCR_OFF(offsetof(struct bf_runtime_sock_addr, daddr)),
657 : : addr_size, BPF_REG_1);
658 [ + - ]: 4 : if (r)
659 : : return r;
660 : :
661 : : /* Destination port: valid for all cgroup_sock_addr hooks.
662 : : * user_port is __be32; BSWAP 16 converts to host order. */
663 : 4 : r = _bf_cgroup_sock_addr_load_field(
664 : : program, offsetof(struct bpf_sock_addr, user_port), 4, BPF_REG_1);
665 [ + - ]: 4 : if (r)
666 : : return r;
667 [ + - ]: 4 : EMIT(program, BPF_BSWAP(BPF_REG_1, 16));
668 : 4 : r = _bf_cgroup_sock_addr_store_field(
669 : : program, BF_PROG_SCR_OFF(offsetof(struct bf_runtime_sock_addr, dport)),
670 : : 2, BPF_REG_1);
671 [ + - ]: 4 : if (r)
672 : : return r;
673 : :
674 : 4 : r = _bf_cgroup_sock_addr_store_ns_pid(program);
675 [ + - ]: 4 : if (r)
676 : : return r;
677 : :
678 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
679 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, BF_PROG_CTX_OFF(arg)));
680 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_2, rule->index));
681 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_3, rule->verdict));
682 [ + - ]: 4 : EMIT(program, BPF_MOV64_REG(BPF_REG_4, BPF_REG_7));
683 [ + - ]: 4 : EMIT(program, BPF_ALU64_IMM(BPF_LSH, BPF_REG_4, 16));
684 [ + - ]: 4 : EMIT(program, BPF_ALU64_REG(BPF_OR, BPF_REG_4, BPF_REG_8));
685 [ + - ]: 4 : EMIT(program, BPF_MOV64_IMM(BPF_REG_5, captured_fields));
686 : 4 : EMIT_FIXUP_ELFSTUB(program, BF_ELFSTUB_SOCK_ADDR_LOG);
687 : :
688 : : return 0;
689 : : }
690 : :
691 : : const struct bf_flavor_ops bf_flavor_ops_cgroup_sock_addr = {
692 : : .gen_inline_prologue = _bf_cgroup_sock_addr_gen_inline_prologue,
693 : : .gen_inline_epilogue = _bf_cgroup_sock_addr_gen_inline_epilogue,
694 : : .get_verdict = _bf_cgroup_sock_addr_get_verdict,
695 : : .gen_inline_matcher = _bf_cgroup_sock_addr_gen_inline_matcher,
696 : : .gen_inline_log = _bf_cgroup_sock_addr_gen_inline_log,
697 : : };
|
