From fbd1a1ce4b8dd043c1ce7d1c800d6da9a504d741 Mon Sep 17 00:00:00 2001 From: Geod24 Date: Wed, 30 Nov 2022 00:31:40 +0100 Subject: [PATCH] Replace `settings.json` with `settings.yaml` and provide a new syntax This adds parsing for `settings.yaml` (`settings.yml` is not supported), which takes precedence over `settings.json`. As we are introducing a new feature, this also slightly change the format to be more intuitive, consistent, and less repetitive. The main change is on the way one can specify registries: instead of having two fields, only have one, which might lead to a bit more repetition in some cases, but generally should be more intuitive to users. --- source/dub/data/settings.d | 407 ++++++++++++++++++++++++++++++++++--- source/dub/dub.d | 88 +++++--- source/dub/test/base.d | 4 +- 3 files changed, 439 insertions(+), 60 deletions(-) diff --git a/source/dub/data/settings.d b/source/dub/data/settings.d index 20c667247..02d8c10eb 100644 --- a/source/dub/data/settings.d +++ b/source/dub/data/settings.d @@ -20,6 +20,20 @@ public enum SkipPackageSuppliers { default_, /// The value wasn't specified. It is provided in order to know when it is safe to ignore it } +/// Transitional structure used to mix `UserConfiguration` and `OldUserConfiguration` +package(dub) struct InternalSettings { + /// registryUrls behaves differently with the new syntax + public string[] registryUrls; + /// This field is no longer present in the new syntax (absorbed) + public SetInfo!(SkipPackageSuppliers) skipRegistry; + + /// Configuration itself + public Settings config; + + /// + alias config this; +} + /** * User-provided settings (configuration) * @@ -37,6 +51,341 @@ public enum SkipPackageSuppliers { * non-additive are marked as `SetInfo`. */ package(dub) struct Settings { + /// Configuration that affect all of dub + private struct DubConfig + { + /** + * Control where the user directory is placed. + * + * The user directory is the place where Dub downloads packages + * and store build artifacts by default. This location defaults to: + * - `$HOME/.dub/`, also known as `~/.dub/`, on Linux; + * - A mix of `%APPDATA%/dub/` and `%LOCALAPPDATA%/dub/` on Windows; + * + * The `dub.home` settings allows to set Dub's home directory explicitly. + * Not that if this is read from the config file located in the home + * directory (using one of the location listed above), no configuration + * will be read from the explicitly-set value, and only packages + * will be stored there. + * + * This field supports environment variables. + * + * ``` + * # This file is located in /etc/dub/settings.yaml + * # Overwrite the user repository to be in $HOME/dlang alongside `install.sh` + * dub: + * home: /home/$USERNAME/dlang/dub-home/ + * ``` + * + * ``` + * # This file is located in /home/$USERNAME/.dub/settings.yaml + * # Store all packages downloaded in the temp folder. + * dub: + * home: /tmp/dub-packages/ + * ``` + */ + public SetInfo!(string) home; + + /** + * Extra locations where to look for packages + * + * This is usually used to load packages installed through other means + * than `dub fetch`, such as a distribution's package manager. + * + * Those locations are assumed to be read only and to follow the same + * directory structure as dub uses, which is + * `$PACKAGE_FOLDER/$PACKAGE_NAME-$PACKAGE_VERSION/$PACKAGE_NAME/`. + * + * This property is additive: if `dub.extraPackages` is specified + * in a system configuration and a user configuration, + * the user-configured locations will be searched first, + * then the system ones. Hence, is a package that is found both in + * `extraPackages` and `dub fetch`-ed, + * the fetched one will take precedence. + * + * ``` + * dub: + * extraPackages: + * - "/usr/lib/dlang/" + * ``` + */ + public @Optional NativePath[] extraPackages; + + /** + * Adds or overrides environment variables for each command + * + * Various parts of dub are able to use environment variables. + * This variable allows to set global environment variables, + * that will be available to every environment-user. + * + * The following, more fine grained variables are also accessible: + * - `build.environment`: For variable only accessible when building a project; + * - `build.preGenerateEnvironment`: For variable only accessible in `preGenerateCommands`; + * - `build.postGenerateEnvironment`: For variable only accessible in `postGenerateCommands`; + * - `build.preBuildEnvironment`: For variable only accessible in `preGenerateCommands`; + * - `build.postBuildEnvironment`: For variable only accessible in `postGenerateCommands`; + * - `run.environment`: For variable only accessible when running a project; + * + * ``` + * # Overwrite the global environment username and add some CI variable + * dub: + * environment: + * USERNAME: dman + * CI_MAX_RUN_TIME_SECONDS: 180 + * ``` + * + * Fields are additive: if `dub.environment`, `build.environment, + * `build.preBuildEnvironment` are defined, all variables defined in + * it will be available to `preBuildCommands`, + * with variables in `build.preBuildEnvironment` taking precedence + * over variables in `build.environment`, themselves taking precedence + * over variables in `dub.environment`. + * + * However, while parsing configuration files, only the most-specialized + * `environment` is retained. + */ + public SetInfo!(string[string]) environment; + } + + /// Ditto + public DubConfig dub; + + /// Configuration that affects the `fetch` command or dependent + private struct FetchConfig + { + /** + * The list of registries to use + * + * By default, this is set to the list of default registries. + * If you wish to use a different set of registries (e.g. company ones), + * setting this properties overwrites them. + * + * ``` + * # This uses no registry + * fetch: + * registries: [] + * ``` + * + * ``` + * # This uses a corporate registry plus the default ones + * fetch: + * registries: + * - https://dub.myfancy.corp/token/private + * - https://code.dlang.org/ + * - https://codemirror.dlang.org/ + * - https://dub.bytecraft.nl/ + * - https://code-mirror.dlang.io/ + * ``` + */ + public SetInfo!(string[]) registries = SetInfo!(string[])( + [ "https://code.dlang.org/", "https://codemirror.dlang.org/" ], + false); +; + } + + /// Ditto + public FetchConfig fetch; + + /// Configuration that affects the `generate` command or indirect invocations + private struct GenerateConfig + { + /** + * Additional environment variables available in `preGenerateCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) preGenerateEnvironment; + + /** + * Additional environment variables available in `postGenerateCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) postGenerateEnvironment; + } + + /// Ditto + public GenerateConfig generate; + + + /// Configuration that affects the `build` command or indirect invocations + private struct BuildConfig + { + /** + * The default compiler to use when building a project + * + * This is similar to passing `--compiler` to every dub invocation. + * The values can be either a path or a binary in the path. + * + * ``` + * build: + * compiler: /usr/bin/ldc2 + * ``` + * + * ``` + * build: + * compiler: gdc-12 + * ``` + */ + public SetInfo!(string) compiler; + + /** + * The default architecture to target when building a project + * + * This is similar to passing `--arch` to every dub invocation. + * The values are expected to be a target triplet. + * + * ``` + * build: + * compiler: ldc2 + * architecture: aarch64-unknown-linux-android + * ``` + * + * If not specified, the architecture targeted is the host one. + */ + public SetInfo!(string) architecture; + + /** + * Whether `-lowmem` is passed to the compiler on builds + * + * By default, `-lowmem` is not used, which can lead to large memory + * usage while building, or even compiler aborting compilation. + * Setting this value to `true` will enable the GC in the compiler. + */ + public SetInfo!(bool) lowmem; + + + /** + * Additional environment variables available while building a project + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) environment; + + /** + * Additional environment variables available in `preBuildCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) preBuildEnvironment; + + /** + * Additional environment variables available in `postBuildCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) postBuildEnvironment; + } + + /// Ditto + public BuildConfig build; + + /// Configuration that affects the `run` command + private struct RunConfig + { + /** + * Additional environment variables available while running a project + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) environment; + + /** + * Additional environment variables available in `preRunCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) preRunEnvironment; + + /** + * Additional environment variables available in `postRunCommands` + * + * For an extended description of this field and how it interacts + * with other similar variables, see `dub.environment` which includes + * a complete description. + */ + public SetInfo!(string[string]) postRunEnvironment; + } + + /// Ditto + public RunConfig run; + + /// Merge a lower priority config (`this`) with a `higher` priority config + public Settings merge(Settings higher) return @safe pure nothrow + { + return .merge(this, higher); + } + + /// Ditto + public Settings merge( + OldSettings higher, ref string[] registryUrls, + ref SetInfo!(SkipPackageSuppliers) skipRegistry) + return @safe pure nothrow + { + Settings result = this; + // Handle `dub` section + if (higher.dubHome.set) + result.dub.home = higher.dubHome; + if (higher.customCachePaths.length) + result.dub.extraPackages ~= higher.customCachePaths; + if (higher.defaultEnvironments.set) + result.dub.environment = higher.defaultEnvironments; + + // Handle `generate` + if (higher.defaultPreGenerateEnvironments.set) + result.generate.preGenerateEnvironment = higher.defaultPreGenerateEnvironments; + if (higher.defaultPostGenerateEnvironments.set) + result.generate.postGenerateEnvironment = higher.defaultPostGenerateEnvironments; + + // Handle `build` section + if (higher.defaultCompiler.set) + result.build.compiler = higher.defaultCompiler; + if (higher.defaultArchitecture.set) + result.build.architecture = higher.defaultArchitecture; + if (higher.defaultLowMemory.set) + result.build.lowmem = higher.defaultLowMemory; + if (higher.defaultBuildEnvironments.set) + result.build.environment = higher.defaultBuildEnvironments; + if (higher.defaultPreBuildEnvironments.set) + result.build.preBuildEnvironment = higher.defaultPreBuildEnvironments; + if (higher.defaultPostBuildEnvironments.set) + result.build.postBuildEnvironment = higher.defaultPostBuildEnvironments; + + // Handling fetch is quite complex due to the two fields being reduced + // to a single one, so we do it in the caller instead. + if (higher.registryUrls.length) + registryUrls ~= higher.registryUrls; + if (higher.skipRegistry.set) + skipRegistry = SetInfo!SkipPackageSuppliers(higher.skipRegistry.value); + + // Handle `run` section + if (higher.defaultRunEnvironments.set) + result.run.environment = higher.defaultRunEnvironments; + if (higher.defaultPreRunEnvironments.set) + result.run.preRunEnvironment = higher.defaultPreRunEnvironments; + if (higher.defaultPostRunEnvironments.set) + result.run.postRunEnvironment = higher.defaultPostRunEnvironments; + + return result; + } +} + +/// Ditto +package(dub) struct OldSettings { @Optional string[] registryUrls; @Optional NativePath[] customCachePaths; @@ -70,37 +419,39 @@ package(dub) struct Settings { SetInfo!(string[string]) defaultPreRunEnvironments; SetInfo!(string[string]) defaultPostRunEnvironments; SetInfo!(string) dubHome; +} - /// Merge a lower priority config (`this`) with a `higher` priority config - public Settings merge(Settings higher) - return @safe pure nothrow - { - import std.traits : hasUDA; - Settings result; - - static foreach (idx, _; Settings.tupleof) { - static if (hasUDA!(Settings.tupleof[idx], Optional)) - result.tupleof[idx] = higher.tupleof[idx] ~ this.tupleof[idx]; - else static if (IsSetInfo!(typeof(this.tupleof[idx]))) { - if (higher.tupleof[idx].set) - result.tupleof[idx] = higher.tupleof[idx]; - else - result.tupleof[idx] = this.tupleof[idx]; - } else - static assert(false, - "Expect `@Optional` or `SetInfo` on: `" ~ - __traits(identifier, this.tupleof[idx]) ~ - "` of type : `" ~ - typeof(this.tupleof[idx]).stringof ~ "`"); - } +/// Merge a lower priority config (`this_`) with a `higher` priority config +public T merge(T)(T this_, T higher) @safe pure nothrow +{ + import std.traits : hasUDA; + T result; - return result; + static foreach (idx, _; T.tupleof) { + static if (hasUDA!(T.tupleof[idx], Optional)) + result.tupleof[idx] = higher.tupleof[idx] ~ this_.tupleof[idx]; + else static if (IsSetInfo!(typeof(T.init.tupleof[idx]))) { + if (higher.tupleof[idx].set) + result.tupleof[idx] = higher.tupleof[idx]; + else + result.tupleof[idx] = this_.tupleof[idx]; + } else static if (is(T == struct)) { + result.tupleof[idx] = merge(this_.tupleof[idx], higher.tupleof[idx]); + } else { + static assert(false, + "Expect `@Optional` or `SetInfo` on: `" ~ + __traits(identifier, this_.tupleof[idx]) ~ + "` of type : `" ~ + typeof(this_.tupleof[idx]).stringof ~ "`"); + } } - /// Workaround multiple `E` declaration in `static foreach` when inline - private template IsSetInfo(T) { enum bool IsSetInfo = is(T : SetInfo!E, E); } + return result; } +/// Workaround multiple `E` declaration in `static foreach` when inline +private template IsSetInfo(T) { enum bool IsSetInfo = is(T : SetInfo!E, E); } + unittest { import dub.internal.configy.Read; @@ -134,7 +485,7 @@ unittest { } }`; - auto c1 = parseConfigString!Settings(str1, "/dev/null"); + auto c1 = parseConfigString!OldSettings(str1, "/dev/null"); assert(c1.registryUrls == [ "http://foo.bar/optional/escape" ]); assert(c1.customCachePaths == [ NativePath("foo/bar"), NativePath("foo/foo") ]); assert(c1.skipRegistry == SkipPackageSuppliers.all); @@ -146,7 +497,7 @@ unittest { assert(c1.defaultEnvironments["VAR3"] == "settings.VAR3"); assert(c1.defaultEnvironments["VAR4"] == "settings.VAR4"); - auto c2 = parseConfigString!Settings(str2, "/dev/null"); + auto c2 = parseConfigString!OldSettings(str2, "/dev/null"); assert(c2.registryUrls == [ "http://bar.foo" ]); assert(c2.customCachePaths == [ NativePath("bar/foo"), NativePath("bar/bar") ]); assert(c2.skipRegistry == SkipPackageSuppliers.none); @@ -184,7 +535,7 @@ unittest { assert(m2.defaultLowMemory == c2.defaultLowMemory); assert(m2.defaultEnvironments == c2.defaultEnvironments); - auto m3 = Settings.init.merge(c1); + auto m3 = OldSettings.init.merge(c1); assert(m3 == c1); } diff --git a/source/dub/dub.d b/source/dub/dub.d index 98247b832..a75cb8910 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -8,9 +8,10 @@ module dub.dub; import dub.compilers.compiler; -import dub.data.settings : SPS = SkipPackageSuppliers, Settings; +import dub.data.settings : SPS = SkipPackageSuppliers, Settings, InternalSettings, OldSettings; import dub.dependency; import dub.dependencyresolver; +import dub.internal.configy.Attributes : Optional, SetInfo; import dub.internal.utils; import dub.internal.vibecompat.core.file; import dub.internal.vibecompat.data.json; @@ -124,7 +125,7 @@ class Dub { NativePath m_rootPath; string m_mainRecipePath; SpecialDirs m_dirs; - Settings m_config; + InternalSettings m_config; Project m_project; string m_defaultCompiler; } @@ -171,7 +172,7 @@ class Dub { m_packageSuppliers = this.makePackageSuppliers(base, skip, registry_var); m_packageManager = this.makePackageManager(); - auto ccps = m_config.customCachePaths; + auto ccps = m_config.dub.extraPackages; if (ccps.length) m_packageManager.customCachePaths = ccps; @@ -249,24 +250,41 @@ class Dub { * Returns: * A populated `Settings` instance. */ - protected Settings loadConfig(ref SpecialDirs dirs) const + protected InternalSettings loadConfig(ref SpecialDirs dirs) const { import dub.internal.configy.Read; - static void readSettingsFile (NativePath path_, ref Settings current) + static void readSettingsFile (NativePath path_noext_, ref InternalSettings current) { + const path_noext = path_noext_.toNativeString(); + + const yaml_path = path_noext ~ ".yaml"; + if (yaml_path.exists) { + auto newConf = parseConfigFileSimple!Settings(yaml_path); + if (!newConf.isNull()) + current.config = current.merge(newConf.get()); + return; + } + // TODO: Remove `StrictMode.Warn` after v1.40 release // The default is to error, but as the previous parser wasn't // complaining, we should first warn the user. - const path = path_.toNativeString(); - if (path.exists) { - auto newConf = parseConfigFileSimple!Settings(path, StrictMode.Warn); + const json_path = path_noext ~ ".json"; + if (json_path.exists) { + // Disabled until v1.40 (or later) is released so we start supporting + // the new format for a new version before we deprecate the old one. + version (all) { + logWarn("A deprecated JSON settings file was found at: %s", json_path); + logWarn("Support for `settings.json` is deprecated, use `settings.yaml` instead"); + } + auto newConf = parseConfigFileSimple!OldSettings(json_path); if (!newConf.isNull()) - current = current.merge(newConf.get()); + current.config = current.merge( + newConf.get(), current.registryUrls, current.skipRegistry); } } - Settings result; + InternalSettings result; const dubFolderPath = NativePath(thisExePath).parentPath; // override default userSettings + userPackages if a $DPATH or @@ -289,11 +307,11 @@ class Dub { } } - readSettingsFile(dirs.systemSettings ~ "settings.json", result); - readSettingsFile(dubFolderPath ~ "../etc/dub/settings.json", result); + readSettingsFile(dirs.systemSettings ~ "settings", result); + readSettingsFile(dubFolderPath ~ "../etc/dub/settings", result); version (Posix) { if (dubFolderPath.absolute && dubFolderPath.startsWith(NativePath("usr"))) - readSettingsFile(NativePath("/etc/dub/settings.json"), result); + readSettingsFile(NativePath("/etc/dub/settings"), result); } // Override user + local package path from system / binary settings @@ -302,8 +320,8 @@ class Dub { // // Don't use it if either $DPATH or $DUB_HOME are set, as environment // variables usually take precedence over configuration. - if (!overrideDubHomeFromEnv && result.dubHome.set) { - dirs.userSettings = NativePath(result.dubHome.expandEnvironmentVariables); + if (!overrideDubHomeFromEnv && result.dub.home.set) { + dirs.userSettings = NativePath(result.dub.home.expandEnvironmentVariables); } // load user config: @@ -315,11 +333,21 @@ class Dub { // same as userSettings above, but taking into account the // config loaded from user settings and per-package config as well. - if (!overrideDubHomeFromEnv && result.dubHome.set) { - dirs.userPackages = NativePath(result.dubHome.expandEnvironmentVariables); + if (!overrideDubHomeFromEnv && result.dub.home.set) { + dirs.userPackages = NativePath(result.dub.home.expandEnvironmentVariables); dirs.cache = dirs.userPackages ~ "cache"; } + if (this.m_config.skipRegistry.set && this.m_config.fetch.registries.set) { + logError("Cannot mix settings.json `skipRegistry` with settings.yaml `fetch.registries`"); + logError("`skipRegistry` value be ignored - the `standard` behavior will apply"); + } + + if (this.m_config.registryUrls.length && this.m_config.fetch.registries.set) { + logError("Cannot mix settings.json `registryUrls` with settings.yaml `fetch.registries`"); + logError("`registryUrls` will be ignored"); + } + return result; } @@ -485,24 +513,24 @@ class Dub { If set, the "defaultArchitecture" field of the DUB user or system configuration file will be used. Otherwise null will be returned. */ - @property string defaultArchitecture() const { return this.m_config.defaultArchitecture; } + @property string defaultArchitecture() const { return this.m_config.build.architecture; } /** Returns the default low memory option to use for building D code. If set, the "defaultLowMemory" field of the DUB user or system configuration file will be used. Otherwise false will be returned. */ - @property bool defaultLowMemory() const { return this.m_config.defaultLowMemory; } - - @property const(string[string]) defaultEnvironments() const { return this.m_config.defaultEnvironments; } - @property const(string[string]) defaultBuildEnvironments() const { return this.m_config.defaultBuildEnvironments; } - @property const(string[string]) defaultRunEnvironments() const { return this.m_config.defaultRunEnvironments; } - @property const(string[string]) defaultPreGenerateEnvironments() const { return this.m_config.defaultPreGenerateEnvironments; } - @property const(string[string]) defaultPostGenerateEnvironments() const { return this.m_config.defaultPostGenerateEnvironments; } - @property const(string[string]) defaultPreBuildEnvironments() const { return this.m_config.defaultPreBuildEnvironments; } - @property const(string[string]) defaultPostBuildEnvironments() const { return this.m_config.defaultPostBuildEnvironments; } - @property const(string[string]) defaultPreRunEnvironments() const { return this.m_config.defaultPreRunEnvironments; } - @property const(string[string]) defaultPostRunEnvironments() const { return this.m_config.defaultPostRunEnvironments; } + @property bool defaultLowMemory() const { return this.m_config.build.lowmem; } + + @property const(string[string]) defaultEnvironments() const { return this.m_config.dub.environment; } + @property const(string[string]) defaultBuildEnvironments() const { return this.m_config.build.environment; } + @property const(string[string]) defaultRunEnvironments() const { return this.m_config.run.environment; } + @property const(string[string]) defaultPreGenerateEnvironments() const { return this.m_config.generate.preGenerateEnvironment; } + @property const(string[string]) defaultPostGenerateEnvironments() const { return this.m_config.generate.postGenerateEnvironment; } + @property const(string[string]) defaultPreBuildEnvironments() const { return this.m_config.build.preBuildEnvironment; } + @property const(string[string]) defaultPostBuildEnvironments() const { return this.m_config.build.postBuildEnvironment; } + @property const(string[string]) defaultPreRunEnvironments() const { return this.m_config.run.preRunEnvironment; } + @property const(string[string]) defaultPostRunEnvironments() const { return this.m_config.run.postRunEnvironment; } /** Loads the package that resides within the configured `rootPath`. */ @@ -1603,7 +1631,7 @@ class Dub { if (auto envCompiler = environment.get("DC")) result = envCompiler; else - result = this.m_config.defaultCompiler.expandTilde; + result = this.m_config.build.compiler.expandTilde; if (result.length && result.isAbsolute) return result; diff --git a/source/dub/test/base.d b/source/dub/test/base.d index 81e96fa8e..eaa2adf4d 100644 --- a/source/dub/test/base.d +++ b/source/dub/test/base.d @@ -285,10 +285,10 @@ public class TestDub : Dub } /// Avoid loading user configuration - protected override Settings loadConfig(ref SpecialDirs dirs) const + protected override InternalSettings loadConfig(ref SpecialDirs dirs) const { dirs = Paths; - return Settings.init; + return typeof(return).init; } ///