From: Björn Töpel bjorn@rivosinc.com
The sched_ext selftests is missing proper cross-compilation support, a proper target entry, and out-of-tree build support.
When building the kselftest suite, e.g.:
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- \ SKIP_TARGETS="" O=/output/foo -C tools/testing/selftests install
The expectation is that the sched_ext is included, cross-built, and placed into /output/foo.
Add CROSS_COMPILE, OUTPUT, and TARGETS support to the sched_ext selftest. Also, remove some variables that were unused by the Makefile.
Signed-off-by: Björn Töpel bjorn@rivosinc.com --- v2: * Removed the duplicated LLVM prefix parsing (David) * Made sure make clean didn't do a complete mess (David) * Added sched_ext to default skip (Shuah) --- tools/testing/selftests/Makefile | 9 +-- tools/testing/selftests/sched_ext/Makefile | 80 +++++++++++----------- 2 files changed, 45 insertions(+), 44 deletions(-)
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index b38199965f99..363d031a16f7 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -88,6 +88,7 @@ TARGETS += rlimits TARGETS += rseq TARGETS += rtc TARGETS += rust +TARGETS += sched_ext TARGETS += seccomp TARGETS += sgx TARGETS += sigaltstack @@ -129,10 +130,10 @@ ifeq ($(filter net/lib,$(TARGETS)),) endif endif
-# User can optionally provide a TARGETS skiplist. By default we skip -# BPF since it has cutting edge build time dependencies which require -# more effort to install. -SKIP_TARGETS ?= bpf +# User can optionally provide a TARGETS skiplist. By default we skip +# targets using BPF since it has cutting edge build time dependencies +# which require more effort to install. +SKIP_TARGETS ?= bpf sched_ext ifneq ($(SKIP_TARGETS),) TMP := $(filter-out $(SKIP_TARGETS), $(TARGETS)) override TARGETS := $(TMP) diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 0754a2c110a1..acab9732b23e 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -3,24 +3,13 @@ include ../../../build/Build.include include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include + +TEST_GEN_PROGS := runner + +# override lib.mk's default rules +OVERRIDE_TARGETS := 1 include ../lib.mk
-ifneq ($(LLVM),) -ifneq ($(filter %/,$(LLVM)),) -LLVM_PREFIX := $(LLVM) -else ifneq ($(filter -%,$(LLVM)),) -LLVM_SUFFIX := $(LLVM) -endif - -CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as -else -CC := gcc -endif # LLVM - -ifneq ($(CROSS_COMPILE),) -$(error CROSS_COMPILE not supported for scx selftests) -endif # CROSS_COMPILE - CURDIR := $(abspath .) REPOROOT := $(abspath ../../../..) TOOLSDIR := $(REPOROOT)/tools @@ -34,18 +23,31 @@ GENHDR := $(GENDIR)/autoconf.h SCXTOOLSDIR := $(TOOLSDIR)/sched_ext SCXTOOLSINCDIR := $(TOOLSDIR)/sched_ext/include
-OUTPUT_DIR := $(CURDIR)/build +OUTPUT_DIR := $(OUTPUT)/build OBJ_DIR := $(OUTPUT_DIR)/obj INCLUDE_DIR := $(OUTPUT_DIR)/include BPFOBJ_DIR := $(OBJ_DIR)/libbpf SCXOBJ_DIR := $(OBJ_DIR)/sched_ext BPFOBJ := $(BPFOBJ_DIR)/libbpf.a LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/libbpf.a -DEFAULT_BPFTOOL := $(OUTPUT_DIR)/sbin/bpftool -HOST_BUILD_DIR := $(OBJ_DIR) -HOST_OUTPUT_DIR := $(OUTPUT_DIR)
-VMLINUX_BTF_PATHS ?= ../../../../vmlinux \ +ifneq ($(CROSS_COMPILE),) +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/host/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/host/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/host/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/host/ +HOST_DESTDIR := $(OUTPUT_DIR)/host/ +else +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/ +HOST_DESTDIR := $(OUTPUT_DIR)/ +endif + +VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ + $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ + ../../../../vmlinux \ /sys/kernel/btf/vmlinux \ /boot/vmlinux-$(shell uname -r) VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) @@ -80,17 +82,23 @@ IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ # Use '-idirafter': Don't interfere with include mechanics except where the # build would have failed anyways. define get_sys_includes -$(shell $(1) -v -E - </dev/null 2>&1 \ +$(shell $(1) $(2) -v -E - </dev/null 2>&1 \ | sed -n '/<...> search starts here:/,/End of search list./{ s| (/.*)|-idirafter \1|p }') \ -$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}') +$(shell $(1) $(2) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}') endef
+ifneq ($(CROSS_COMPILE),) +CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%)) +endif + +CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) + BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \ $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) \ -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat \ -I$(INCLUDE_DIR) -I$(APIDIR) -I$(SCXTOOLSINCDIR) \ -I$(REPOROOT)/include \ - $(call get_sys_includes,$(CLANG)) \ + $(CLANG_SYS_INCLUDES) \ -Wall -Wno-compare-distinct-pointer-types \ -Wno-incompatible-function-pointer-types \ -O2 -mcpu=v3 @@ -98,7 +106,7 @@ BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \ # sort removes libbpf duplicates when not cross-building MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(OBJ_DIR)/libbpf \ $(OBJ_DIR)/bpftool $(OBJ_DIR)/resolve_btfids \ - $(INCLUDE_DIR) $(SCXOBJ_DIR)) + $(HOST_OBJ_DIR) $(INCLUDE_DIR) $(SCXOBJ_DIR))
$(MAKE_DIRS): $(call msg,MKDIR,,$@) @@ -112,14 +120,14 @@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ DESTDIR=$(OUTPUT_DIR) prefix= all install_headers
$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ - $(LIBBPF_OUTPUT) | $(OBJ_DIR)/bpftool + $(LIBBPF_OUTPUT) | $(HOST_OBJ_DIR) $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) \ EXTRA_CFLAGS='-g -O0' \ - OUTPUT=$(OBJ_DIR)/bpftool/ \ - LIBBPF_OUTPUT=$(OBJ_DIR)/libbpf/ \ - LIBBPF_DESTDIR=$(OUTPUT_DIR)/ \ - prefix= DESTDIR=$(OUTPUT_DIR)/ install-bin + OUTPUT=$(HOST_OBJ_DIR)/ \ + LIBBPF_OUTPUT=$(HOST_LIBBPF_OUTPUT) \ + LIBBPF_DESTDIR=$(HOST_LIBBPF_DESTDIR) \ + prefix= DESTDIR=$(HOST_DESTDIR) install-bin
$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) ifeq ($(VMLINUX_H),) @@ -150,9 +158,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
override define CLEAN rm -rf $(OUTPUT_DIR) - rm -f *.o *.bpf.o *.bpf.skel.h *.bpf.subskel.h rm -f $(TEST_GEN_PROGS) - rm -f runner endef
# Every testcase takes all of the BPF progs are dependencies by default. This @@ -196,21 +202,15 @@ $(SCXOBJ_DIR)/runner.o: runner.c | $(SCXOBJ_DIR) # function doesn't support using implicit rules otherwise. $(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bpfprogs) | $(SCXOBJ_DIR) $(eval test=$(patsubst %.o,%.c,$(notdir $@))) - $(CC) $(CFLAGS) -c $< -o $@ $(SCXOBJ_DIR)/runner.o + $(CC) $(CFLAGS) -c $< -o $@
$(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR) $(CC) $(CFLAGS) -c $< -o $@
-runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets) +$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets) @echo "$(testcase-targets)" $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
-TEST_GEN_PROGS := runner - -all: runner - -.PHONY: all clean help - .DEFAULT_GOAL := all
.DELETE_ON_ERROR:
base-commit: 8cf0b93919e13d1e8d4466eb4080a4c4d9d66d7b
On Mon, Oct 07, 2024 at 09:31:32AM +0200, Björn Töpel wrote:
When building the kselftest suite, e.g.:
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- \ SKIP_TARGETS="" O=/output/foo -C tools/testing/selftests install
The expectation is that the sched_ext is included, cross-built, and placed into /output/foo.
When building for arm64 with this applied on top of mainline or -next I'm seeing:
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -C tools/testing/selftests TARGETS=sched_ext SKIP_TARGETS="
give
CLNG-BPF create_dsq.bpf.o In file included from create_dsq.bpf.c:9: /home/broonie/git/linux/tools/sched_ext/include/scx/common.bpf.h:33:17: error: use of undeclared identifier 'SCX_DSQ_FLAG_BUILTIN' 33 | _Static_assert(SCX_DSQ_FLAG_BUILTIN, | ^
and more (my system clang is clang 20). It's also failing with a native x86_64 build in the same way. I've run "make headers_install", it looks like clang is not getting told about the copy of the headers installed in ./usr/include:
clang -g -D__TARGET_ARCH_x86 -mlittle-endian -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/include -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/include/bpf-compat -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/build/include -I/home/broonie/git/linux/tools/include/uapi -I/home/broonie/git/linux/tools/sched_ext/include -I/home/broonie/git/linux/include -idirafter /usr/lib/llvm-20/lib/clang/20/include -idirafter /usr/local/include -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include -Wall -Wno-compare-distinct-pointer-types -Wno-incompatible-function-pointer-types -O2 -mcpu=v3 -target bpf -c create_dsq.bpf.c -o /home/broonie/git/linux/tools/testing/selftests/sched_ext/build/obj/sched_ext/create_dsq.bpf.o
Add CROSS_COMPILE, OUTPUT, and TARGETS support to the sched_ext selftest. Also, remove some variables that were unused by the Makefile.
+ifneq ($(CROSS_COMPILE),) +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/host/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/host/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/host/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/host/ +HOST_DESTDIR := $(OUTPUT_DIR)/host/ +else +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/ +HOST_DESTDIR := $(OUTPUT_DIR)/ +endif
This won't detect a cross compile when building using LLVM as that doesn't need to set CROSS_COMPILE, it can use the same binaries for all targets. There's runes in the Makefile for the mm selftests for identifying the target architecture, though actually I'm wondering if it's worth just always building the host copy and never having to worry if the target build is a cross build or not? It's obvious overhead in the native case though if we actually need the target copy.
Mark Brown broonie@kernel.org writes:
On Mon, Oct 07, 2024 at 09:31:32AM +0200, Björn Töpel wrote:
When building the kselftest suite, e.g.:
make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu- \ SKIP_TARGETS="" O=/output/foo -C tools/testing/selftests install
The expectation is that the sched_ext is included, cross-built, and placed into /output/foo.
When building for arm64 with this applied on top of mainline or -next I'm seeing:
Thanks for taking it for a spin!
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -C tools/testing/selftests TARGETS=sched_ext SKIP_TARGETS="
give
CLNG-BPF create_dsq.bpf.o In file included from create_dsq.bpf.c:9: /home/broonie/git/linux/tools/sched_ext/include/scx/common.bpf.h:33:17: error: use of undeclared identifier 'SCX_DSQ_FLAG_BUILTIN' 33 | _Static_assert(SCX_DSQ_FLAG_BUILTIN, | ^
This is most likely due to incorrect VMLINUX_BTF_PATHS, so that vmlinux.h is incorrectly generated. Try grepping for SCX_DSQ_FLAG_BUILTIN in vmlinux.h.
and more (my system clang is clang 20). It's also failing with a native x86_64 build in the same way. I've run "make headers_install", it looks like clang is not getting told about the copy of the headers installed in ./usr/include:
clang -g -D__TARGET_ARCH_x86 -mlittle-endian -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/include -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/include/bpf-compat -I/home/broonie/git/linux/tools/testing/selftests/sched_ext/build/include -I/home/broonie/git/linux/tools/include/uapi -I/home/broonie/git/linux/tools/sched_ext/include -I/home/broonie/git/linux/include -idirafter /usr/lib/llvm-20/lib/clang/20/include -idirafter /usr/local/include -idirafter /usr/include/x86_64-linux-gnu -idirafter /usr/include -Wall -Wno-compare-distinct-pointer-types -Wno-incompatible-function-pointer-types -O2 -mcpu=v3 -target bpf -c create_dsq.bpf.c -o /home/broonie/git/linux/tools/testing/selftests/sched_ext/build/obj/sched_ext/create_dsq.bpf.o
The sched_ext BPF programs relies on a vmlinux.h, which is generated using bpftool and the vmlinux with BTF information. Have you built a kernel with BTF support?
Add CROSS_COMPILE, OUTPUT, and TARGETS support to the sched_ext selftest. Also, remove some variables that were unused by the Makefile.
+ifneq ($(CROSS_COMPILE),) +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/host/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/host/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/host/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/host/ +HOST_DESTDIR := $(OUTPUT_DIR)/host/ +else +DEFAULT_BPFTOOL := $(OUTPUT_DIR)/sbin/bpftool +HOST_OBJ_DIR := $(OBJ_DIR)/bpftool +HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/ +HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/ +HOST_DESTDIR := $(OUTPUT_DIR)/ +endif
This won't detect a cross compile when building using LLVM as that doesn't need to set CROSS_COMPILE, it can use the same binaries for all targets. There's runes in the Makefile for the mm selftests for identifying the target architecture, though actually I'm wondering if it's worth just always building the host copy and never having to worry if the target build is a cross build or not? It's obvious overhead in the native case though if we actually need the target copy.
Yeah, that would indeed simplify things! I'll spin a v3 with that (and wait for more feedback).
Thanks for testing the patch! Björn
On Mon, Oct 07, 2024 at 06:00:57PM +0200, Björn Töpel wrote:
Mark Brown broonie@kernel.org writes:
On Mon, Oct 07, 2024 at 09:31:32AM +0200, Björn Töpel wrote:
CLNG-BPF create_dsq.bpf.o In file included from create_dsq.bpf.c:9: /home/broonie/git/linux/tools/sched_ext/include/scx/common.bpf.h:33:17: error: use of undeclared identifier 'SCX_DSQ_FLAG_BUILTIN' 33 | _Static_assert(SCX_DSQ_FLAG_BUILTIN, | ^
This is most likely due to incorrect VMLINUX_BTF_PATHS, so that vmlinux.h is incorrectly generated. Try grepping for SCX_DSQ_FLAG_BUILTIN in vmlinux.h.
Yeah, it's not in the generated files:
broonie@finisterre:~/git/linux$ grep SCX_DSQ_FLAG_BUILTIN ./tools/testing/selftests/sched_ext/build/include/vmlinux.h ./tools/testing/selftests/sched_ext/build/obj/bpftool/vmlinux.h broonie@finisterre:~/git/linux$
I didn't actually build a kernel, if the build system needs a kernel it's just silently not detected that it's missing?
Mark Brown broonie@kernel.org writes:
On Mon, Oct 07, 2024 at 06:00:57PM +0200, Björn Töpel wrote:
Mark Brown broonie@kernel.org writes:
On Mon, Oct 07, 2024 at 09:31:32AM +0200, Björn Töpel wrote:
CLNG-BPF create_dsq.bpf.o In file included from create_dsq.bpf.c:9: /home/broonie/git/linux/tools/sched_ext/include/scx/common.bpf.h:33:17: error: use of undeclared identifier 'SCX_DSQ_FLAG_BUILTIN' 33 | _Static_assert(SCX_DSQ_FLAG_BUILTIN, | ^
This is most likely due to incorrect VMLINUX_BTF_PATHS, so that vmlinux.h is incorrectly generated. Try grepping for SCX_DSQ_FLAG_BUILTIN in vmlinux.h.
Yeah, it's not in the generated files:
broonie@finisterre:~/git/linux$ grep SCX_DSQ_FLAG_BUILTIN ./tools/testing/selftests/sched_ext/build/include/vmlinux.h ./tools/testing/selftests/sched_ext/build/obj/bpftool/vmlinux.h broonie@finisterre:~/git/linux$
I didn't actually build a kernel, if the build system needs a kernel it's just silently not detected that it's missing?
It tries to find a kernel with BTF: | VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ | $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ | ../../../../vmlinux \ | /sys/kernel/btf/vmlinux \ | /boot/vmlinux-$(shell uname -r)
Similar to all the other selftests using BPF.
(Oh, and at some point the BPF parts should be in lib.mk...)
Björn
On Mon, Oct 07, 2024 at 06:45:32PM +0200, Björn Töpel wrote:
Mark Brown broonie@kernel.org writes:
I didn't actually build a kernel, if the build system needs a kernel it's just silently not detected that it's missing?
It tries to find a kernel with BTF: | VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ | $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ | ../../../../vmlinux \ | /sys/kernel/btf/vmlinux \ | /boot/vmlinux-$(shell uname -r)
Similar to all the other selftests using BPF.
(Oh, and at some point the BPF parts should be in lib.mk...)
Ah, that'll have found my host kernel. There should probably be some sort of check that it's at least targeting the correct architecture since this *will* go wrong.
On Mon, Oct 07, 2024 at 06:00:57PM +0200, Björn Töpel wrote:
Mark Brown broonie@kernel.org writes:
When building for arm64 with this applied on top of mainline or -next I'm seeing:
Thanks for taking it for a spin!
make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -C tools/testing/selftests TARGETS=sched_ext SKIP_TARGETS="
Oh, and for arm64 as previously noted the sched_ext config fragment odesn't DTRT, merge_config.sh says:
Value requested for CONFIG_SCHED_DEBUG not in final .config Requested value: CONFIG_SCHED_DEBUG=y Actual value:
Value requested for CONFIG_SCHED_CLASS_EXT not in final .config Requested value: CONFIG_SCHED_CLASS_EXT=y Actual value:
Value requested for CONFIG_EXT_GROUP_SCHED not in final .config Requested value: CONFIG_EXT_GROUP_SCHED=y Actual value:
Value requested for CONFIG_DEBUG_INFO not in final .config Requested value: CONFIG_DEBUG_INFO=y Actual value:
Value requested for CONFIG_DEBUG_INFO_BTF not in final .config Requested value: CONFIG_DEBUG_INFO_BTF=y Actual value:
On Mon, Oct 07, 2024 at 06:00:57PM +0200, Björn Töpel wrote:
The sched_ext BPF programs relies on a vmlinux.h, which is generated using bpftool and the vmlinux with BTF information. Have you built a kernel with BTF support?
OK, so having beaten the kernel config into shape and using GCC rather than clang for my build I did get this to build fine. I think the main gotchas are the issues with the arm64 defconfig not supporting BTF and the fact that the Makefile silently picked up the host kernel.
linux-kselftest-mirror@lists.linaro.org
-
Björn Töpel -
Mark Brown