Bpftool sync 2024-10-14

BPF-CHECKPOINT-COMMIT

@@ -1 +1 @@
-b24d7f0da6ef5a23456a301eaf51b170f961d4ae
+b836cbdf3b81a4a22b3452186efa2e5105a77e10

CHECKPOINT-COMMIT

@@ -1 +1 @@
-b24d7f0da6ef5a23456a301eaf51b170f961d4ae
+989a29cfed9b5092c3e18be14e9032c51bb1c9f6

include/uapi/linux/bpf.h

@@ -1970,6 +1970,8 @@ union bpf_attr {
  * 		program.
  * 	Return
  * 		The SMP id of the processor running the program.
+ * 	Attributes
+ * 		__bpf_fastcall
  *
  * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
  * 	Description
@@ -3101,10 +3103,6 @@ union bpf_attr {
  * 		with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration
  * 		option, and in this case it only works on functions tagged with
  * 		**ALLOW_ERROR_INJECTION** in the kernel code.
- *
- * 		Also, the helper is only available for the architectures having
- * 		the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing,
- * 		x86 architecture is the only one to support this feature.
  * 	Return
  * 		0
  *
@@ -5369,7 +5367,7 @@ union bpf_attr {
  *		Currently, the **flags** must be 0. Currently, nr_loops is
  *		limited to 1 << 23 (~8 million) loops.
  *
- *		long (\*callback_fn)(u32 index, void \*ctx);
+ *		long (\*callback_fn)(u64 index, void \*ctx);
  *
  *		where **index** is the current index in the loop. The index
  *		is zero-indexed.

src/btf.c

@@ -1,11 +1,15 @@
 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
 /* Copyright (C) 2019 Facebook */
 
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
 #include <errno.h>
 #include <fcntl.h>
 #include <linux/err.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 #include <linux/btf.h>
@@ -21,6 +25,7 @@
 #include "main.h"
 
 #define KFUNC_DECL_TAG		"bpf_kfunc"
+#define FASTCALL_DECL_TAG	"bpf_fastcall"
 
 static const char * const btf_kind_str[NR_BTF_KINDS] = {
 	[BTF_KIND_UNKN]		= "UNKNOWN",
@@ -464,47 +469,113 @@ static int dump_btf_raw(const struct btf *btf,
 	return 0;
 }
 
+struct ptr_array {
+	__u32 cnt;
+	__u32 cap;
+	const void **elems;
+};
+
+static int ptr_array_push(const void *ptr, struct ptr_array *arr)
+{
+	__u32 new_cap;
+	void *tmp;
+
+	if (arr->cnt == arr->cap) {
+		new_cap = (arr->cap ?: 16) * 2;
+		tmp = realloc(arr->elems, sizeof(*arr->elems) * new_cap);
+		if (!tmp)
+			return -ENOMEM;
+		arr->elems = tmp;
+		arr->cap = new_cap;
+	}
+	arr->elems[arr->cnt++] = ptr;
+	return 0;
+}
+
+static void ptr_array_free(struct ptr_array *arr)
+{
+	free(arr->elems);
+}
+
+static int cmp_kfuncs(const void *pa, const void *pb, void *ctx)
+{
+	struct btf *btf = ctx;
+	const struct btf_type *a = *(void **)pa;
+	const struct btf_type *b = *(void **)pb;
+
+	return strcmp(btf__str_by_offset(btf, a->name_off),
+		      btf__str_by_offset(btf, b->name_off));
+}
+
 static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)
 {
 	LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts);
-	int cnt = btf__type_cnt(btf);
-	int i;
+	__u32 cnt = btf__type_cnt(btf), i, j;
+	struct ptr_array fastcalls = {};
+	struct ptr_array kfuncs = {};
+	int err = 0;
 
 	printf("\n/* BPF kfuncs */\n");
 	printf("#ifndef BPF_NO_KFUNC_PROTOTYPES\n");
 
 	for (i = 1; i < cnt; i++) {
 		const struct btf_type *t = btf__type_by_id(btf, i);
+		const struct btf_type *ft;
 		const char *name;
-		int err;
 
 		if (!btf_is_decl_tag(t))
 			continue;
 
 		if (btf_decl_tag(t)->component_idx != -1)
 			continue;
 
-		name = btf__name_by_offset(btf, t->name_off);
-		if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)))
+		ft = btf__type_by_id(btf, t->type);
+		if (!btf_is_func(ft))
 			continue;
 
-		t = btf__type_by_id(btf, t->type);
-		if (!btf_is_func(t))
-			continue;
+		name = btf__name_by_offset(btf, t->name_off);
+		if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)) == 0) {
+			err = ptr_array_push(ft, &kfuncs);
+			if (err)
+				goto out;
+		}
+
+		if (strncmp(name, FASTCALL_DECL_TAG, sizeof(FASTCALL_DECL_TAG)) == 0) {
+			err = ptr_array_push(ft, &fastcalls);
+			if (err)
+				goto out;
+		}
+	}
+
+	/* Sort kfuncs by name for improved vmlinux.h stability  */
+	qsort_r(kfuncs.elems, kfuncs.cnt, sizeof(*kfuncs.elems), cmp_kfuncs, (void *)btf);
+	for (i = 0; i < kfuncs.cnt; i++) {
+		const struct btf_type *t = kfuncs.elems[i];
 
 		printf("extern ");
 
+		/* Assume small amount of fastcall kfuncs */
+		for (j = 0; j < fastcalls.cnt; j++) {
+			if (fastcalls.elems[j] == t) {
+				printf("__bpf_fastcall ");
+				break;
+			}
+		}
+
 		opts.field_name = btf__name_by_offset(btf, t->name_off);
 		err = btf_dump__emit_type_decl(d, t->type, &opts);
 		if (err)
-			return err;
+			goto out;
 
 		printf(" __weak __ksym;\n");
 	}
 
 	printf("#endif\n\n");
 
-	return 0;
+out:
+	ptr_array_free(&fastcalls);
+	ptr_array_free(&kfuncs);
+	return err;
 }
 
 static void __printf(2, 0) btf_dump_printf(void *ctx,
@@ -718,6 +789,13 @@ static int dump_btf_c(const struct btf *btf,
 	printf("#ifndef __weak\n");
 	printf("#define __weak __attribute__((weak))\n");
 	printf("#endif\n\n");
+	printf("#ifndef __bpf_fastcall\n");
+	printf("#if __has_attribute(bpf_fastcall)\n");
+	printf("#define __bpf_fastcall __attribute__((bpf_fastcall))\n");
+	printf("#else\n");
+	printf("#define __bpf_fastcall\n");
+	printf("#endif\n");
+	printf("#endif\n\n");
 
 	if (root_type_cnt) {
 		for (i = 0; i < root_type_cnt; i++) {