[Buildroot] [PATCH v5 1/1] Makefile: Parallelize glibc locale generation
Gleb Mazovetskiy
glex.spb at gmail.com
Sun Jan 3 17:15:25 UTC 2021
Parallelizes locale generation based on `BR2_JLEVEL` setting.
Locale generation always runs during the finalize stage and can consume
a significant amount of time. Parallelizing it greatly reduces that time
on multi-core machines.
To parallelize it, we first invoke `localedef` for every locale in
parallel with the `--no-archive` option. This creates the intermediate
locale data instead of writing to the finally archive directly.
Then, we invoke `localedef` again once to create the archive from the
intermediate compiled locale data files.
We have to do it this way because `localedef` does not do any locking
when writing to the archive file, so calling it without `--no-archive`
concurrently could result in a corrupt archive file or an archive file
that is missing some locales.
Signed-off-by: Gleb Mazovetskiy <glex.spb at gmail.com>
---
Makefile | 29 +++++++---------------
support/misc/gen-glibc-locales.mk | 41 +++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+), 20 deletions(-)
create mode 100644 support/misc/gen-glibc-locales.mk
diff --git a/Makefile b/Makefile
index 4d334adcd6..dbeabf9e61 100644
--- a/Makefile
+++ b/Makefile
@@ -661,32 +661,21 @@ endef
TARGET_FINALIZE_HOOKS += TOOLCHAIN_ECLIPSE_REGISTER
endif
-# Generate locale data. Basically, we call the localedef program
-# (built by the host-localedef package) for each locale. The input
-# data comes preferably from the toolchain, or if the toolchain does
-# not have them (Linaro toolchains), we use the ones available on the
-# host machine.
+# Generate locale data.
ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),y)
GLIBC_GENERATE_LOCALES = $(call qstrip,$(BR2_GENERATE_LOCALE))
ifneq ($(GLIBC_GENERATE_LOCALES),)
PACKAGES += host-localedef
define GENERATE_GLIBC_LOCALES
- $(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/
- $(Q)for locale in $(GLIBC_GENERATE_LOCALES) ; do \
- inputfile=`echo $${locale} | cut -f1 -d'.'` ; \
- charmap=`echo $${locale} | cut -f2 -d'.' -s` ; \
- if test -z "$${charmap}" ; then \
- charmap="UTF-8" ; \
- fi ; \
- echo "Generating locale $${inputfile}.$${charmap}" ; \
- I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
- $(HOST_DIR)/bin/localedef \
- --prefix=$(TARGET_DIR) \
- --$(call LOWERCASE,$(BR2_ENDIAN))-endian \
- -i $${inputfile} -f $${charmap} \
- $${locale} ; \
- done
+ $(TARGET_CONFIGURE_OPTS) \
+ $(MAKE) -j$(PARALLEL_JOBS) -f support/misc/gen-glibc-locales.mk \
+ HOST_DIR=$(HOST_DIR) \
+ TARGET_DIR=$(TARGET_DIR) \
+ STAGING_DIR=$(STAGING_DIR) \
+ ENDIAN=$(call LOWERCASE,$(BR2_ENDIAN)) \
+ LOCALES="$(GLIBC_GENERATE_LOCALES)" \
+ Q=$(Q)
endef
TARGET_FINALIZE_HOOKS += GENERATE_GLIBC_LOCALES
endif
diff --git a/support/misc/gen-glibc-locales.mk b/support/misc/gen-glibc-locales.mk
new file mode 100644
index 0000000000..3db1e0dc11
--- /dev/null
+++ b/support/misc/gen-glibc-locales.mk
@@ -0,0 +1,41 @@
+# Generates glibc locale data for target.
+
+inputfile = $(firstword $(subst ., ,$(1)))
+charmap = $(or $(word 2,$(subst ., ,$(1))),UTF-8)
+
+# Packages all the generated locale data into the final archive.
+#
+# We sort the file names to produce consistent output regardless of
+# the `find` outputs order.
+$(TARGET_DIR)/usr/lib/locale/locale-archive: $(LOCALES)
+ $(Q)rm -f $(@)
+ $(Q)find $(TARGET_DIR)/usr/lib/locale/ -maxdepth 1 -mindepth 1 -type d -print0 \
+ | sort -z \
+ | xargs -0 \
+ $(HOST_DIR)/bin/localedef \
+ --prefix=$(TARGET_DIR) \
+ --$(ENDIAN)-endian \
+ --add-to-archive
+
+# Generates locale data for each locale.
+#
+# The input data comes preferably from the toolchain, or if the toolchain
+# does not have them (Linaro toolchains), we use the ones available on the
+# host machine.
+#
+# Uses `localedef`, which is built by the `host-localedef` package.
+$(LOCALES): | $(TARGET_DIR)/usr/lib/locale/
+ $(Q)echo "Generating locale $(@)"
+ $(Q)I18NPATH=$(STAGING_DIR)/usr/share/i18n:/usr/share/i18n \
+ $(HOST_DIR)/bin/localedef \
+ --prefix=$(TARGET_DIR) \
+ --$(ENDIAN)-endian \
+ --no-archive \
+ -i $(call inputfile,$(@)) \
+ -f $(call charmap,$(@)) \
+ $(@)
+
+.PHONY: $(LOCALES)
+
+$(TARGET_DIR)/usr/lib/locale/:
+ $(Q)mkdir -p $(TARGET_DIR)/usr/lib/locale/
--
2.27.0
More information about the buildroot
mailing list