From 386961166571d6c5843b55fa51302e9accf8f06a Mon Sep 17 00:00:00 2001 From: tresf Date: Wed, 8 Nov 2017 00:20:57 -0500 Subject: [PATCH 1/4] New attempt at fetching submodules --- CMakeLists.txt | 1 + cmake/modules/CheckSubmodules.cmake | 76 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 cmake/modules/CheckSubmodules.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index ee47d120eef..99619b9e77b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ IF(COMMAND CMAKE_POLICY) ENDIF() ENDIF(COMMAND CMAKE_POLICY) +INCLUDE(CheckSubmodules) INCLUDE(AddFileDependencies) INCLUDE(CheckIncludeFiles) INCLUDE(FindPkgConfig) diff --git a/cmake/modules/CheckSubmodules.cmake b/cmake/modules/CheckSubmodules.cmake new file mode 100644 index 00000000000..9f80d894fbf --- /dev/null +++ b/cmake/modules/CheckSubmodules.cmake @@ -0,0 +1,76 @@ +# Utility for validating and, if needed, cloning all submodules +# +# Looks for a .gitmodules in the root project folder +# Loops over all modules looking well-known configure/build scripts +# +# Usage: +# INCLUDE(CheckSubmodules) +# +# Options: +# SET(SKIP_SUBMODULES "foo;bar") +# +# Or via command line: +# cmake -DSKIP_SUBMODULES=foo;bar +# +# Copyright (c) 2017, Tres Finocchiaro, +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + +# Files which confirm a successful clone +SET(VALID_CRUMBS "CMakeLists.txt;Makefile.in;Makefile.am;configure.ac;configure.py;autogen.sh") + +MESSAGE("\nValidating submodules...") +FILE(READ "${CMAKE_SOURCE_DIR}/.gitmodules" SUBMODULE_DATA) + +# Assume alpha-numeric paths +STRING(REGEX MATCHALL "path = [0-9A-Za-z/]+" SUBMODULE_LIST ${SUBMODULE_DATA}) +FOREACH(_part ${SUBMODULE_LIST}) + STRING(REPLACE "path = " "" SUBMODULE_PATH ${_part}) + + # Remove submodules from validation as specified in -DSKIP_SUBMODULES=foo;bar + SET(SKIP false) + IF(SKIP_SUBMODULES) + FOREACH(_skip ${SKIP_SUBMODULES}) + IF(${SUBMODULE_PATH} MATCHES ${_skip}) + MESSAGE("-- Skipping ${SUBMODULE_PATH} matches \"${_skip}\"") + SET(SKIP true) + ENDIF() + ENDFOREACH() + ENDIF() + LIST(REMOVE_ITEM SUBMODULE_LIST ${_part}) + IF(NOT SKIP) + LIST(APPEND SUBMODULE_LIST ${SUBMODULE_PATH}) + ENDIF() +ENDFOREACH() + +LIST(SORT SUBMODULE_LIST) + +# Attempt to do lazy clone +FOREACH(_submodule ${SUBMODULE_LIST}) + STRING(REPLACE "/" ";" PATH_PARTS ${_submodule}) + LIST(REVERSE PATH_PARTS) + LIST(GET PATH_PARTS 0 SUBMODULE_NAME) + + MESSAGE("-- Checking ${SUBMODULE_NAME}...") + SET(CRUMB_FOUND false) + FOREACH(_crumb ${VALID_CRUMBS}) + IF(EXISTS "${CMAKE_SOURCE_DIR}/${_submodule}/${_crumb}") + SET(CRUMB_FOUND true) + MESSAGE("-- Found ${_submodule}/${_crumb}") + ENDIF() + ENDFOREACH() + IF(NOT CRUMB_FOUND) + FIND_PACKAGE(Git REQUIRED) + MESSAGE("-- Missing ${_submodule}") + EXECUTE_PROCESS( + COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive ${CMAKE_SOURCE_DIR}/${_submodule} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ERROR_VARIABLE GIT_STDERR + ) + IF(GIT_STDERR) + MESSAGE(FATAL_ERROR ${GIT_STDERR}) + ENDIF() + ENDIF() +ENDFOREACH() +MESSAGE("-- Done validating submodules.\n") From 02ff9fd4255947ac29e9d6297472ffeaea187c01 Mon Sep 17 00:00:00 2001 From: Tres Finocchiaro Date: Wed, 8 Nov 2017 03:05:09 -0500 Subject: [PATCH 2/4] Add rety support, fixups --- cmake/modules/CheckSubmodules.cmake | 63 ++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/cmake/modules/CheckSubmodules.cmake b/cmake/modules/CheckSubmodules.cmake index 9f80d894fbf..34e17a1b925 100644 --- a/cmake/modules/CheckSubmodules.cmake +++ b/cmake/modules/CheckSubmodules.cmake @@ -18,13 +18,13 @@ # For details see the accompanying COPYING-CMAKE-SCRIPTS file. # Files which confirm a successful clone -SET(VALID_CRUMBS "CMakeLists.txt;Makefile.in;Makefile.am;configure.ac;configure.py;autogen.sh") +SET(VALID_CRUMBS "CMakeLists.txt;Makefile;Makefile.in;Makefile.am;configure.ac;configure.py;autogen.sh") MESSAGE("\nValidating submodules...") FILE(READ "${CMAKE_SOURCE_DIR}/.gitmodules" SUBMODULE_DATA) # Assume alpha-numeric paths -STRING(REGEX MATCHALL "path = [0-9A-Za-z/]+" SUBMODULE_LIST ${SUBMODULE_DATA}) +STRING(REGEX MATCHALL "path = [-0-9A-Za-z/]+" SUBMODULE_LIST ${SUBMODULE_DATA}) FOREACH(_part ${SUBMODULE_LIST}) STRING(REPLACE "path = " "" SUBMODULE_PATH ${_part}) @@ -46,6 +46,36 @@ ENDFOREACH() LIST(SORT SUBMODULE_LIST) +# Once called, status is stored in GIT_RESULT respectively. +# Note: Git likes to write to stderr. Don't assume stderr is error; Check GIT_RESULT instead. +MACRO(GIT_SUBMODULE SUBMODULE_PATH FORCE_DEINIT) + FIND_PACKAGE(Git REQUIRED) + IF(${FORCE_DEINIT}) + MESSAGE("-- Resetting ${SUBMODULE_PATH}") + EXECUTE_PROCESS( + COMMAND ${GIT_EXECUTABLE} submodule deinit -f ${CMAKE_SOURCE_DIR}/${SUBMODULE_PATH} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_QUIET + ) + # Recurse + GIT_SUBMODULE(${SUBMODULE_PATH} false) + ELSE() + MESSAGE("-- Fetching ${SUBMODULE_PATH}") + EXECUTE_PROCESS( + COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive ${CMAKE_SOURCE_DIR}/${SUBMODULE_PATH} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE GIT_RESULT + OUTPUT_VARIABLE GIT_STDOUT + ERROR_VARIABLE GIT_STDERR + ) + + SET(GIT_MESSAGE "${GIT_STDOUT}${GIT_STDERR}") + MESSAGE(${GIT_MESSAGE}) + ENDIF() +ENDMACRO() + +SET(RETRY_PHRASES "Failed to recurse;unadvertised object;cannot create directory") + # Attempt to do lazy clone FOREACH(_submodule ${SUBMODULE_LIST}) STRING(REPLACE "/" ";" PATH_PARTS ${_submodule}) @@ -61,15 +91,26 @@ FOREACH(_submodule ${SUBMODULE_LIST}) ENDIF() ENDFOREACH() IF(NOT CRUMB_FOUND) - FIND_PACKAGE(Git REQUIRED) - MESSAGE("-- Missing ${_submodule}") - EXECUTE_PROCESS( - COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive ${CMAKE_SOURCE_DIR}/${_submodule} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - ERROR_VARIABLE GIT_STDERR - ) - IF(GIT_STDERR) - MESSAGE(FATAL_ERROR ${GIT_STDERR}) + GIT_SUBMODULE(${_submodule} false) + + SET(MAX_ATTEMPTS 2) + SET(COUNTED 0) + SET(COUNTING "") + # Handle edge-cases where submodule didn't clone properly or re-uses a non-empty directory + WHILE(NOT GIT_RESULT EQUAL 0 AND COUNTED LESS MAX_ATTEMPTS) + LIST(APPEND COUNTING "x") + LIST(LENGTH COUNTING COUNTED) + + FOREACH(_phrase ${RETRY_PHRASES}) + IF("${GIT_MESSAGE}" MATCHES "${_phrase}") + MESSAGE("-- Retrying ${_submodule} using 'deinit' (attempt ${COUNTED} of ${MAX_ATTEMPTS})...") + GIT_SUBMODULE(${_submodule} true) + ENDIF() + ENDFOREACH() + ENDWHILE() + + IF(NOT GIT_RESULT EQUAL 0) + MESSAGE(FATAL_ERROR "${GIT_EXECUTABLE} exited with status of ${GIT_RESULT}") ENDIF() ENDIF() ENDFOREACH() From d6a35e3f8b9e78237173eb0f0dcd24a21d81eb79 Mon Sep 17 00:00:00 2001 From: Tres Finocchiaro Date: Fri, 10 Nov 2017 21:54:38 -0500 Subject: [PATCH 3/4] Fix spamming of CMP0045 warnings when switching branches --- src/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e67c1ec6ed..28d2455b030 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -160,10 +160,12 @@ TARGET_LINK_LIBRARIES(lmms ) FOREACH(LIB ${LMMS_REQUIRED_LIBS}) - GET_TARGET_PROPERTY(INCLUDE_DIRS ${LIB} INTERFACE_INCLUDE_DIRECTORIES) - if (INCLUDE_DIRS) - TARGET_INCLUDE_DIRECTORIES(lmmsobjs PRIVATE ${INCLUDE_DIRS}) - ENDIF() + IF(TARGET ${LIB}) + GET_TARGET_PROPERTY(INCLUDE_DIRS ${LIB} INTERFACE_INCLUDE_DIRECTORIES) + IF(INCLUDE_DIRS) + TARGET_INCLUDE_DIRECTORIES(lmmsobjs PRIVATE ${INCLUDE_DIRS}) + ENDIF() + ENDIF() ENDFOREACH() From 8801deda49fd0dc1d41c42f3c0be8cf31c8b63cd Mon Sep 17 00:00:00 2001 From: tresf Date: Tue, 14 Nov 2017 13:11:26 -0500 Subject: [PATCH 4/4] Use shallow cloning whever possible --- cmake/modules/CheckSubmodules.cmake | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cmake/modules/CheckSubmodules.cmake b/cmake/modules/CheckSubmodules.cmake index 34e17a1b925..d90cd622b87 100644 --- a/cmake/modules/CheckSubmodules.cmake +++ b/cmake/modules/CheckSubmodules.cmake @@ -20,6 +20,12 @@ # Files which confirm a successful clone SET(VALID_CRUMBS "CMakeLists.txt;Makefile;Makefile.in;Makefile.am;configure.ac;configure.py;autogen.sh") +# Try and use the specified shallow clone on submodules, if supported +SET(DEPTH_VALUE 100) + +# Number of times git commands will retry before failing +SET(MAX_ATTEMPTS 2) + MESSAGE("\nValidating submodules...") FILE(READ "${CMAKE_SOURCE_DIR}/.gitmodules" SUBMODULE_DATA) @@ -60,9 +66,16 @@ MACRO(GIT_SUBMODULE SUBMODULE_PATH FORCE_DEINIT) # Recurse GIT_SUBMODULE(${SUBMODULE_PATH} false) ELSE() - MESSAGE("-- Fetching ${SUBMODULE_PATH}") + # Try to use the depth switch + SET(DEPTH_CMD "") + MESSAGE("-- Fetching ${SUBMODULE_PATH}") + IF(DEPTH_VALUE) + SET(DEPTH_CMD "--depth" ) + MESSAGE("-- Fetching ${SUBMODULE_PATH} @ --depth ${DEPTH_VALUE}") + ENDIF() + EXECUTE_PROCESS( - COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive ${CMAKE_SOURCE_DIR}/${SUBMODULE_PATH} + COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive ${DEPTH_CMD} ${DEPTH_VALUE} ${CMAKE_SOURCE_DIR}/${SUBMODULE_PATH} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} RESULT_VARIABLE GIT_RESULT OUTPUT_VARIABLE GIT_STDOUT @@ -93,7 +106,6 @@ FOREACH(_submodule ${SUBMODULE_LIST}) IF(NOT CRUMB_FOUND) GIT_SUBMODULE(${_submodule} false) - SET(MAX_ATTEMPTS 2) SET(COUNTED 0) SET(COUNTING "") # Handle edge-cases where submodule didn't clone properly or re-uses a non-empty directory @@ -104,6 +116,15 @@ FOREACH(_submodule ${SUBMODULE_LIST}) FOREACH(_phrase ${RETRY_PHRASES}) IF("${GIT_MESSAGE}" MATCHES "${_phrase}") MESSAGE("-- Retrying ${_submodule} using 'deinit' (attempt ${COUNTED} of ${MAX_ATTEMPTS})...") + + # Shallow submodules were introduced in 1.8.4 + # Shallow commits can fail to clone from non-default branches, only try once + IF(GIT_VERSION_STRING VERSION_GREATER "1.8.3" AND COUNTED LESS 2) + # Try a shallow submodule clone + ELSE() + UNSET(DEPTH_VALUE) + ENDIF() + GIT_SUBMODULE(${_submodule} true) ENDIF() ENDFOREACH()