✨ Configurations embed and retrieve (MarlinFirmware#21321)

.gitignore

@@ -22,6 +22,9 @@
 # Generated files
 _Version.h
 bdf2u8g
+marlin_config.json
+mczip.h
+*.gen
 
 #
 # OS

Marlin/Configuration_adv.h

@@ -1593,6 +1593,14 @@
     #define SD_FIRMWARE_UPDATE_INACTIVE_VALUE 0xFF
   #endif
 
+  /**
+   * Enable this option if you have more than ~3K of unused flash space.
+   * Marlin will embed all settings in the firmware binary as compressed data.
+   * Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'.
+   * See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'.
+   */
+  //#define CONFIGURATION_EMBEDDING
+
   // Add an optimized binary file transfer mode, initiated with 'M28 B1'
   //#define BINARY_FILE_TRANSFER
 

Marlin/src/gcode/eeprom/M500-M504.cpp

@@ -25,6 +25,11 @@
 #include "../../core/serial.h"
 #include "../../inc/MarlinConfig.h"
 
+#if ENABLED(CONFIGURATION_EMBEDDING)
+  #include "../../sd/SdBaseFile.h"
+  #include "../../mczip.h"
+#endif
+
 /**
  * M500: Store settings in EEPROM
  */
@@ -50,9 +55,22 @@ void GcodeSuite::M502() {
 
   /**
    * M503: print settings currently in memory
+   *
+   * With CONFIGURATION_EMBEDDING:
+   *   C<flag> : Save the full Marlin configuration to SD Card as "mc.zip"
    */
   void GcodeSuite::M503() {
     (void)settings.report(!parser.boolval('S', true));
+
+    #if ENABLED(CONFIGURATION_EMBEDDING)
+      if (parser.seen_test('C')) {
+        SdBaseFile file;
+        const uint16_t size = sizeof(mc_zip);
+        // Need to create the config size on the SD card
+        if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close())
+          SERIAL_ECHO_MSG("Configuration saved as 'mc.zip'");
+      }
+    #endif
   }
 
 #endif // !DISABLE_M503

Marlin/src/gcode/host/M115.cpp

@@ -24,7 +24,6 @@
 #include "../../inc/MarlinConfig.h"
 #include "../queue.h"           // for getting the command port
 
-
 #if ENABLED(M115_GEOMETRY_REPORT)
   #include "../../module/motion.h"
 #endif
@@ -33,13 +32,25 @@
   #include "../../feature/caselight.h"
 #endif
 
+//#define MINIMAL_CAP_LINES // Don't even mention the disabled capabilities
+
 #if ENABLED(EXTENDED_CAPABILITIES_REPORT)
-  static void cap_line(FSTR_P const name, bool ena=false) {
-    SERIAL_ECHOPGM("Cap:");
-    SERIAL_ECHOF(name);
-    SERIAL_CHAR(':', '0' + ena);
-    SERIAL_EOL();
-  }
+  #if ENABLED(MINIMAL_CAP_LINES)
+    #define cap_line(S,C) if (C) _cap_line(S)
+    static void _cap_line(FSTR_P const name) {
+      SERIAL_ECHOPGM("Cap:");
+      SERIAL_ECHOF(name);
+      SERIAL_ECHOLNPGM(":1");
+    }
+  #else
+    #define cap_line(V...) _cap_line(V)
+    static void _cap_line(FSTR_P const name, bool ena=false) {
+      SERIAL_ECHOPGM("Cap:");
+      SERIAL_ECHOF(name);
+      SERIAL_CHAR(':', '0' + ena);
+      SERIAL_EOL();
+    }
+  #endif
 #endif
 
 /**
@@ -167,6 +178,9 @@ void GcodeSuite::M115() {
     // MEATPACK Compression
     cap_line(F("MEATPACK"), SERIAL_IMPL.has_feature(port, SerialFeature::MeatPack));
 
+    // CONFIG_EXPORT
+    cap_line(F("CONFIG_EXPORT"), ENABLED(CONFIG_EMBED_AND_SAVE_TO_SD));
+
     // Machine Geometry
     #if ENABLED(M115_GEOMETRY_REPORT)
       const xyz_pos_t bmin = { 0, 0, 0 },

Marlin/src/inc/Conditionals_adv.h

@@ -1004,3 +1004,9 @@
 #if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2)
   #define HAS_MEATPACK 1
 #endif
+
+// AVR are (usually) too limited in resources to store the configuration into the binary
+#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503))
+  #undef CONFIGURATION_EMBEDDING
+  #define CANNOT_EMBED_CONFIGURATION defined(__AVR__)
+#endif

Marlin/src/inc/Warnings.cpp

@@ -549,3 +549,7 @@
 #elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS)
   #warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)"
 #endif
+
+#if CANNOT_EMBED_CONFIGURATION
+  #warning "Disabled CONFIGURATION_EMBEDDING because the target usually has less flash storage. Define FORCE_CONFIG_EMBED to override."
+#endif

buildroot/bin/restore_configs

@@ -2,4 +2,4 @@
 
 git checkout Marlin/Configuration*.h 2>/dev/null
 git checkout Marlin/src/pins/ramps/pins_RAMPS.h 2>/dev/null
-rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h
+rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h marlin_config.json .pio/build/mc.zip

buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py

@@ -6,7 +6,6 @@
 if pioutil.is_pio_build():
 	from os.path import join, isfile
 	import shutil
-	from pprint import pprint
 
 	Import("env")
 

buildroot/share/PlatformIO/scripts/common-dependencies.py

@@ -192,63 +192,6 @@ def apply_features_config():
 				lib_ignore = env.GetProjectOption('lib_ignore') + [feat['lib_ignore']]
 				set_env_field('lib_ignore', lib_ignore)
 
-	#
-	# Find a compiler, considering the OS
-	#
-	ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV'])
-	GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path")
-	def search_compiler():
-		try:
-			filepath = env.GetProjectOption('custom_gcc')
-			blab("Getting compiler from env")
-			return filepath
-		except:
-			pass
-
-		if os.path.exists(GCC_PATH_CACHE):
-			with open(GCC_PATH_CACHE, 'r') as f:
-				return f.read()
-
-		# Find the current platform compiler by searching the $PATH
-		# which will be in a platformio toolchain bin folder
-		path_regex = re.escape(env['PROJECT_PACKAGES_DIR'])
-
-		# See if the environment provides a default compiler
-		try:
-			gcc = env.GetProjectOption('custom_deps_gcc')
-		except:
-			gcc = "g++"
-
-		if env['PLATFORM'] == 'win32':
-			path_separator = ';'
-			path_regex += r'.*\\bin'
-			gcc += ".exe"
-		else:
-			path_separator = ':'
-			path_regex += r'/.+/bin'
-
-		# Search for the compiler
-		for pathdir in env['ENV']['PATH'].split(path_separator):
-			if not re.search(path_regex, pathdir, re.IGNORECASE):
-				continue
-			for filepath in os.listdir(pathdir):
-				if not filepath.endswith(gcc):
-					continue
-				# Use entire path to not rely on env PATH
-				filepath = os.path.sep.join([pathdir, filepath])
-				# Cache the g++ path to no search always
-				if os.path.exists(ENV_BUILD_PATH):
-					with open(GCC_PATH_CACHE, 'w+') as f:
-						f.write(filepath)
-
-				return filepath
-
-		filepath = env.get('CXX')
-		if filepath == 'CC':
-			filepath = gcc
-		blab("Couldn't find a compiler! Fallback to %s" % filepath)
-		return filepath
-
 	#
 	# Use the compiler to get a list of all enabled features
 	#
@@ -257,25 +200,8 @@ def load_marlin_features():
 			return
 
 		# Process defines
-		build_flags = env.get('BUILD_FLAGS')
-		build_flags = env.ParseFlagsExtended(build_flags)
-
-		cxx = search_compiler()
-		cmd = ['"' + cxx + '"']
-
-		# Build flags from board.json
-		#if 'BOARD' in env:
-		#	cmd += [env.BoardConfig().get("build.extra_flags")]
-		for s in build_flags['CPPDEFINES']:
-			if isinstance(s, tuple):
-				cmd += ['-D' + s[0] + '=' + str(s[1])]
-			else:
-				cmd += ['-D' + s]
-
-		cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++ buildroot/share/PlatformIO/scripts/common-dependencies.h']
-		cmd = ' '.join(cmd)
-		blab(cmd, 4)
-		define_list = subprocess.check_output(cmd, shell=True).splitlines()
+		from preprocessor import run_preprocessor
+		define_list = run_preprocessor(env)
 		marlin_features = {}
 		for define in define_list:
 			feature = define[8:].strip().decode().split(' ')
@@ -310,9 +236,18 @@ def MarlinFeatureIsEnabled(env, feature):
 	except:
 		pass
 
+	#
 	# Add a method for other PIO scripts to query enabled features
+	#
 	env.AddMethod(MarlinFeatureIsEnabled)
 
+	#
 	# Add dependencies for enabled Marlin features
+	#
 	apply_features_config()
 	force_ignore_unused_libs()
+
+	#print(env.Dump())
+
+	from signature import compute_build_signature
+	compute_build_signature(env)

buildroot/share/PlatformIO/scripts/mc-apply.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+#
+# Create a Configuration from marlin_config.json
+#
+import json
+import sys
+import shutil
+import re
+
+opt_output = '--opt' in sys.argv
+output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
+
+try:
+	with open('marlin_config.json', 'r') as infile:
+		conf = json.load(infile)
+		for key in conf:
+			# We don't care about the hash when restoring here
+			if key == '__INITIAL_HASH':
+				continue
+			if key == 'VERSION':
+				for k, v in sorted(conf[key].items()):
+					print(k + ': ' + v)
+				continue
+			# The key is the file name, so let's build it now
+			outfile = open('Marlin/' + key + output_suffix, 'w')
+			for k, v in sorted(conf[key].items()):
+				# Make define line now
+				if opt_output:
+					if v != '':
+						if '"' in v:
+							v = "'%s'" % v
+						elif ' ' in v:
+							v = '"%s"' % v
+						define = 'opt_set ' + k + ' ' + v + '\n'
+					else:
+						define = 'opt_enable ' + k + '\n'
+				else:
+					define = '#define ' + k + ' ' + v + '\n'
+				outfile.write(define)
+			outfile.close()
+
+			# Try to apply changes to the actual configuration file (in order to keep useful comments)
+			if output_suffix != '':
+				# Move the existing configuration so it doesn't interfere
+				shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig')
+				infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n')
+				outfile = open('Marlin/' + key, 'w')
+				for line in infile_lines:
+					sline = line.strip(" \t\n\r")
+					if sline[:7] == "#define":
+						# Extract the key here (we don't care about the value)
+						kv = sline[8:].strip().split(' ')
+						if kv[0] in conf[key]:
+							outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n')
+							# Remove the key from the dict, so we can still write all missing keys at the end of the file
+							del conf[key][kv[0]]
+						else:
+							outfile.write(line + '\n')
+					else:
+						outfile.write(line + '\n')
+				# Process any remaining defines here
+				for k, v in sorted(conf[key].items()):
+					define = '#define ' + k + ' ' + v + '\n'
+					outfile.write(define)
+				outfile.close()
+
+			print('Output configuration written to: ' + 'Marlin/' + key + output_suffix)
+except:
+	print('No marlin_config.json found.')