[PATCH v3 5/7] kbuild: unify two modpost invocations

From: Masahiro Yamada
Date: Sat Sep 24 2022 - 14:21:05 EST

Next message: Masahiro Yamada: "[PATCH v3 3/7] kbuild: move .vmlinux.objs rule to Makefile.modpost"
Previous message: Masahiro Yamada: "[PATCH v3 2/7] kbuild: list sub-directories in ./Kbuild"
In reply to: Nicolas Schier: "Re: [PATCH v3 2/7] kbuild: list sub-directories in ./Kbuild"
Next in thread: Nicolas Schier: "Re: [PATCH v3 5/7] kbuild: unify two modpost invocations"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]

Currently, modpost is executed twice; first for vmlinux, second
for modules.

This commit merges them.

Current build flow
==================

1) build obj-y and obj-m objects
2) link vmlinux.o
3) modpost for vmlinux
4) link vmlinux
5) modpost for modules
6) link modules (*.ko)

The build steps 1) through 6) are serialized, that is, modules are
built after vmlinux. You do not get benefits of parallel builds when
scripts/link-vmlinux.sh is being run.

New build flow
==============

1) build obj-y and obj-m objects
2) link vmlinux.o
3) modpost for vmlinux and modules
4a) link vmlinux
4b) link modules (*.ko)

In the new build flow, modpost is invoked just once.

vmlinux and modules are built in parallel. One exception is
CONFIG_DEBUG_INFO_BTF_MODULES=y, where modules depend on vmlinux.

Signed-off-by: Masahiro Yamada <masahiroy@xxxxxxxxxx>
---

(no changes since v1)

Makefile | 30 ++++++++++---
scripts/Makefile.modfinal | 2 +-
scripts/Makefile.modpost | 93 ++++++++++++---------------------------
scripts/link-vmlinux.sh | 3 --
4 files changed, 53 insertions(+), 75 deletions(-)

diff --git a/Makefile b/Makefile
index b5dfb54b1993..cf9d7b1d8c14 100644
--- a/Makefile
+++ b/Makefile
@@ -1152,7 +1152,7 @@ cmd_link-vmlinux = \
$(CONFIG_SHELL) $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)"; \
$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true)

-vmlinux: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE
+vmlinux: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) modpost FORCE
+$(call if_changed_dep,link-vmlinux)

targets := vmlinux
@@ -1428,7 +1428,13 @@ endif
# Build modules
#

-modules: $(if $(KBUILD_BUILTIN),vmlinux) modules_prepare
+# *.ko are usually independent of vmlinux, but CONFIG_DEBUG_INFOBTF_MODULES
+# is an exception.
+ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+modules: vmlinux
+endif
+
+modules: modules_prepare

# Target to prepare building external modules
modules_prepare: prepare
@@ -1741,8 +1747,12 @@ ifdef CONFIG_MODULES
$(MODORDER): $(build-dir)
@:

-modules: modules_check
- $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost
+# KBUILD_MODPOST_NOFINAL can be set to skip the final link of modules.
+# This is solely useful to speed up test compiles.
+modules: modpost
+ifneq ($(KBUILD_MODPOST_NOFINAL),1)
+ $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal
+endif

PHONY += modules_check
modules_check: $(MODORDER)
@@ -1773,6 +1783,11 @@ KBUILD_MODULES :=

endif # CONFIG_MODULES

+PHONY += modpost
+modpost: $(if $(single-build),, $(if $(KBUILD_BUILTIN), vmlinux.o)) \
+ $(if $(KBUILD_MODULES), modules_check)
+ $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost
+
# Single targets
# ---------------------------------------------------------------------------
# To build individual files in subdirectories, you can do like this:
@@ -1792,16 +1807,19 @@ single-ko := $(sort $(filter %.ko, $(MAKECMDGOALS)))
single-no-ko := $(filter-out $(single-ko), $(MAKECMDGOALS)) \
$(foreach x, o mod, $(patsubst %.ko, %.$x, $(single-ko)))

-$(single-ko): single_modpost
+$(single-ko): single_modules
@:
$(single-no-ko): $(build-dir)
@:

# Remove MODORDER when done because it is not the real one.
PHONY += single_modpost
-single_modpost: $(single-no-ko) modules_prepare
+single_modules: $(single-no-ko) modules_prepare
$(Q){ $(foreach m, $(single-ko), echo $(extmod_prefix)$m;) } > $(MODORDER)
$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost
+ifneq ($(KBUILD_MODPOST_NOFINAL),1)
+ $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal
+endif
$(Q)rm -f $(MODORDER)

single-goals := $(addprefix $(build-dir)/, $(single-no-ko))
diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal
index 35100e981f4a..a3cf9e3647c9 100644
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -55,7 +55,7 @@ if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \
printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:)

# Re-generate module BTFs if either module's .ko or vmlinux changed
-$(modules): %.ko: %.o %.mod.o scripts/module.lds $(if $(KBUILD_BUILTIN),vmlinux) FORCE
+$(modules): %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),vmlinux) FORCE
+$(call if_changed_except,ld_ko_o,vmlinux)
ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+$(if $(newer-prereqs),$(call cmd,btf_ko))
diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost
index 04ad00917b2f..2daf760eeb25 100644
--- a/scripts/Makefile.modpost
+++ b/scripts/Makefile.modpost
@@ -32,9 +32,6 @@
# Step 4 is solely used to allow module versioning in external modules,
# where the CRC of each module is retrieved from the Module.symvers file.

-# KBUILD_MODPOST_NOFINAL can be set to skip the final link of modules.
-# This is solely useful to speed up test compiles
-
PHONY := __modpost
__modpost:

@@ -45,24 +42,23 @@ MODPOST = scripts/mod/modpost \
$(if $(CONFIG_MODVERSIONS),-m) \
$(if $(CONFIG_MODULE_SRCVERSION_ALL),-a) \
$(if $(CONFIG_SECTION_MISMATCH_WARN_ONLY),,-E) \
+ $(if $(KBUILD_NSDEPS),-d $(MODULES_NSDEPS)) \
+ $(if $(CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS)$(KBUILD_NSDEPS),-N) \
-o $@

-ifdef MODPOST_VMLINUX
-
-quiet_cmd_modpost = MODPOST $@
- cmd_modpost = $(MODPOST) $<
-
-vmlinux.symvers: vmlinux.o
- $(call cmd,modpost)
+# 'make -i -k' ignores compile errors, and builds as many modules as possible.
+ifneq ($(findstring i,$(filter-out --%,$(MAKEFLAGS))),)
+MODPOST += -n
+endif

-__modpost: vmlinux.symvers
+ifeq ($(KBUILD_EXTMOD),)

# Generate the list of in-tree objects in vmlinux
# ---------------------------------------------------------------------------

# This is used to retrieve symbol versions generated by genksyms.
ifdef CONFIG_MODVERSIONS
-vmlinux.symvers: .vmlinux.objs
+vmlinux.symvers Module.symvers: .vmlinux.objs
endif

# Ignore libgcc.a
@@ -83,24 +79,12 @@ targets += .vmlinux.objs
.vmlinux.objs: $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS) FORCE
$(call if_changed,vmlinux_objs)

-else
-
-ifeq ($(KBUILD_EXTMOD),)
-
-input-symdump := vmlinux.symvers
-output-symdump := modules-only.symvers
-
-quiet_cmd_cat = GEN $@
- cmd_cat = cat $(real-prereqs) > $@
-
-ifneq ($(wildcard vmlinux.symvers),)
-
-__modpost: Module.symvers
-Module.symvers: vmlinux.symvers modules-only.symvers FORCE
- $(call if_changed,cat)
-
-targets += Module.symvers
+vmlinux.o-if-present := $(wildcard vmlinux.o)
+output-symdump := vmlinux.symvers

+ifdef KBUILD_MODULES
+output-symdump := $(if $(vmlinux.o-if-present), Module.symvers, modules-only.symvers)
+missing-input := $(filter-out $(vmlinux.o-if-present),vmlinux.o)
endif

else
@@ -112,56 +96,35 @@ src := $(obj)
# Include the module's Makefile to find KBUILD_EXTRA_SYMBOLS
include $(or $(wildcard $(src)/Kbuild), $(src)/Makefile)

-# modpost option for external modules
-MODPOST += -e
-
-input-symdump := Module.symvers $(KBUILD_EXTRA_SYMBOLS)
+module.symvers-if-present := $(wildcard Module.symvers)
output-symdump := $(KBUILD_EXTMOD)/Module.symvers
+missing-input := $(filter-out $(module.symvers-if-present), Module.symvers)

-endif
-
-existing-input-symdump := $(wildcard $(input-symdump))
-
-# modpost options for modules (both in-kernel and external)
-MODPOST += \
- $(addprefix -i ,$(existing-input-symdump)) \
- $(if $(KBUILD_NSDEPS),-d $(MODULES_NSDEPS)) \
- $(if $(CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS)$(KBUILD_NSDEPS),-N)
-
-# 'make -i -k' ignores compile errors, and builds as many modules as possible.
-ifneq ($(findstring i,$(filter-out --%,$(MAKEFLAGS))),)
-MODPOST += -n
-endif
+MODPOST += -e $(addprefix -i ,$(module.symvers-if-present) $(KBUILD_EXTRA_SYMBOLS))

-# Clear VPATH to not search for *.symvers in $(srctree). Check only $(objtree).
-VPATH :=
-$(input-symdump):
- @echo >&2 'WARNING: Symbol version dump "$@" is missing.'
- @echo >&2 ' Modules may not have dependencies or modversions.'
- @echo >&2 ' You may get many unresolved symbol warnings.'
+endif # ($(KBUILD_EXTMOD),)

-# KBUILD_MODPOST_WARN can be set to avoid error out in case of undefined symbols
-ifneq ($(KBUILD_MODPOST_WARN)$(filter-out $(existing-input-symdump), $(input-symdump)),)
+ifneq ($(KBUILD_MODPOST_WARN)$(missing-input),)
MODPOST += -w
endif

+modorder-if-needed := $(if $(KBUILD_MODULES), $(MODORDER))
+
# Read out modules.order to pass in modpost.
# Otherwise, allmodconfig would fail with "Argument list too long".
quiet_cmd_modpost = MODPOST $@
- cmd_modpost = sed 's/ko$$/o/' $< | $(MODPOST) -T -
-
-$(output-symdump): $(MODORDER) $(input-symdump) FORCE
- $(call if_changed,modpost)
+ cmd_modpost = \
+ $(if $(missing-input), \
+ echo >&2 "WARNING: $(missing-input) is missing."; \
+ echo >&2 " Modules may not have dependencies or modversions."; \
+ echo >&2 " You may get many unresolved symbol warnings.";) \
+ sed 's/ko$$/o/' $(or $(modorder-if-needed), /dev/null) | $(MODPOST) $(vmlinux.o-if-present) -T -

targets += $(output-symdump)
+$(output-symdump): $(modorder-if-needed) $(vmlinux.o-if-present) $(moudle.symvers-if-present) FORCE
+ $(call if_changed,modpost)

__modpost: $(output-symdump)
-ifneq ($(KBUILD_MODPOST_NOFINAL),1)
- $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal
-endif
-
-endif
-
PHONY += FORCE
FORCE:

diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
index 07486f90d5e2..6a197d8a88ac 100755
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -214,9 +214,6 @@ if [ "$1" = "clean" ]; then
exit 0
fi

-# modpost vmlinux.o to check for section mismatches
-${MAKE} -f "${srctree}/scripts/Makefile.modpost" MODPOST_VMLINUX=1
-
info MODINFO modules.builtin.modinfo
${OBJCOPY} -j .modinfo -O binary vmlinux.o modules.builtin.modinfo
info GEN modules.builtin
--
2.34.1

Next message: Masahiro Yamada: "[PATCH v3 3/7] kbuild: move .vmlinux.objs rule to Makefile.modpost"
Previous message: Masahiro Yamada: "[PATCH v3 2/7] kbuild: list sub-directories in ./Kbuild"
In reply to: Nicolas Schier: "Re: [PATCH v3 2/7] kbuild: list sub-directories in ./Kbuild"
Next in thread: Nicolas Schier: "Re: [PATCH v3 5/7] kbuild: unify two modpost invocations"
Messages sorted by: [ date ] [ thread ] [ subject ] [ author ]