Merge pull request #20311 from kaspar030/git-cache-rs

ci: support git-cache-rs

pkg/cmsis/Makefile

@@ -3,10 +3,6 @@ PKG_URL=https://github.com/ARM-software/CMSIS_5
 PKG_VERSION=2b7495b8535bdcb306dac29b9ded4cfb679d7e5c  # 5.9.0
 PKG_LICENSE=Apache-2.0
 
-include $(RIOTBASE)/pkg/pkg.mk
-
-CFLAGS += -Wno-cast-align
-
 CMSIS_NN_MODULES =                 \
   cmsis-nn_activationfunctions     \
   cmsis-nn_convolutionfunctions    \
@@ -16,13 +12,6 @@ CMSIS_NN_MODULES =                 \
   cmsis-nn_softmaxfunctions        \
   #
 
-DIR_activationfunctions        := ActivationFunctions
-DIR_convolutionfunctions       := ConvolutionFunctions
-DIR_fullyconnectedfunctions    := FullyConnectedFunctions
-DIR_nnsupportfunctions         := NNSupportFunctions
-DIR_poolingfunctions           := PoolingFunctions
-DIR_softmaxfunctions           := SoftmaxFunctions
-
 CMSIS_DSP_MODULES =              \
   cmsis-dsp_basicmathfunctions   \
   cmsis-dsp_commontables         \
@@ -36,6 +25,32 @@ CMSIS_DSP_MODULES =              \
   cmsis-dsp_transformfunctions   \
   #
 
+CMSIS_DSP_MODULES_USED = $(filter $(CMSIS_DSP_MODULES),$(USEMODULE))
+CMSIS_NN_MODULES_USED = $(filter $(CMSIS_NN_MODULES),$(USEMODULE))
+CMSIS_MODULES_USED = $(CMSIS_DSP_MODULES_USED) $(CMSIS_NN_MODULES_USED)
+
+PKG_SPARSE_PATHS=CMSIS/Core/Include
+
+ifneq (, $(CMSIS_NN_MODULES_USED))
+  PKG_SPARSE_PATHS+=CMSIS/NN
+  CMSIS_DSP_NEEDED=1
+endif
+ifneq (, $(CMSIS_DSP_MODULES_USED)$(CMSIS_DSP_NEEDED))
+  PKG_SPARSE_PATHS+=CMSIS/DSP
+endif
+
+
+include $(RIOTBASE)/pkg/pkg.mk
+
+CFLAGS += -Wno-cast-align
+
+DIR_activationfunctions        := ActivationFunctions
+DIR_convolutionfunctions       := ConvolutionFunctions
+DIR_fullyconnectedfunctions    := FullyConnectedFunctions
+DIR_nnsupportfunctions         := NNSupportFunctions
+DIR_poolingfunctions           := PoolingFunctions
+DIR_softmaxfunctions           := SoftmaxFunctions
+
 DIR_basicmathfunctions      := BasicMathFunctions
 DIR_commontables            := CommonTables
 DIR_complexmathfunctions    := ComplexMathFunctions
@@ -49,8 +64,6 @@ DIR_transformfunctions      := TransformFunctions
 
 .PHONY: cmsis-dsp_% cmsis-nn_%
 
-CMSIS_MODULES_USED = $(filter $(CMSIS_DSP_MODULES) $(CMSIS_NN_MODULES),$(USEMODULE))
-
 all: $(CMSIS_MODULES_USED)
 
 cmsis-dsp_%:

pkg/pkg.mk

@@ -128,7 +128,28 @@ $(PKG_DOWNLOADED): $(MAKEFILE_LIST) | $(PKG_SOURCE_DIR)/.git
 	fi
 	$(Q)echo $(PKG_VERSION) > $@
 
-ifeq ($(GIT_CACHE_DIR),$(wildcard $(GIT_CACHE_DIR)))
+# This snippet ensures that for packages that have dynamic sparse paths (e.g.,
+# pkg/cmsis), the sparse paths of the time of checkout are the same as needed
+# now.
+# E.g., build a) only needs CMSIS/Core. Build b) also needs CMSIS/DSP.
+# If b) is built after a) and the cmsis checkout does not contain CMSIS/DSP,
+# the sources need to be checked out again.
+# (Inside, this is doing an ad-hoc "|$(LAZYSPONGE)", but using the python version turned out
+# to be significantly slower).
+ifneq (, $(PKG_SPARSE_PATHS))
+PKG_SPARSE_TAG = $(PKG_SOURCE_DIR).sparse
+$(PKG_SPARSE_TAG): FORCE
+	$(Q)if test -f $@; then \
+		test "$$(cat $@)" = "$(PKG_SPARSE_PATHS)" && exit 0; \
+	fi ; mkdir -p $$(dirname $@) && echo "$(PKG_SPARSE_PATHS)" > $@
+endif
+
+ifneq (,$(GIT_CACHE_RS))
+$(PKG_SOURCE_DIR)/.git: $(PKG_SPARSE_TAG) | $(PKG_CUSTOM_PREPARED)
+	$(if $(QUIETER),,$(info [INFO] cloning $(PKG_NAME)))
+	$(Q)rm -Rf $(PKG_SOURCE_DIR)
+	$(Q)$(GIT_CACHE_RS) clone --commit $(PKG_VERSION) $(addprefix --sparse-add ,$(PKG_SPARSE_PATHS)) -- $(PKG_URL) $(PKG_SOURCE_DIR)
+else ifeq ($(GIT_CACHE_DIR),$(wildcard $(GIT_CACHE_DIR)))
 $(PKG_SOURCE_DIR)/.git: | $(PKG_CUSTOM_PREPARED)
 	$(if $(QUIETER),,$(info [INFO] cloning $(PKG_NAME)))
 	$(Q)rm -Rf $(PKG_SOURCE_DIR)