From 82c3a173685ebac71e0dbf6fcb88369b9c319235 Mon Sep 17 00:00:00 2001 From: danbst Date: Fri, 18 Jan 2019 12:19:50 +0200 Subject: [PATCH 1/3] overlays: (partially) recursive merge One of the problems newcomers face when working with overlays is non-expected behavior of overriding packages in package sets. Let's take, for example, Python. Here's an overlay: ``` self: super: { pythonPackages.my-new-package = super.callPackages ./xxx { }; } ``` Big supprise is that `pythonPackages` set is now empty! ``` $ nix-build -E '(with import ./. { overlays = [ (self: super: { pythonPackages.my-new-package = self.pythonPackages.pip; }) ]; }; pythonPackages.ipython)' error: attribute 'ipython' missing, at (string):5:7 ``` It is not empty, but contains only one package. That's because attribute sets are not merged recursively (like NixOS modules do), but are replaced on each `extends`. Recursive merge is tricky, because it has to force evaluation in lot of places and breaks the knot. But we can do poor-man recursive merge "whitelist". Say, when overlay contains attribute `_merge_pythonPackages = true;`, then we can merge it recursively: ``` $ nix-build -E '(with import ./. { overlays = [ (self: super: { _merge_pythonPackages = true; pythonPackages.my-new-package = self.pythonPackages.pip; }) ]; }; pythonPackages.ipython)' /nix/store/k4wf0244qq8xcml85n45sdwalizy0i6k-python2.7-ipython-5.8.0 ``` We can also support this attribute in left-hand side overlay (for example, `all-packages.nix`): ``` ... _merge_pythonPackages = true; pythonPackages = python.pkgs; _merge_python2Packages = true; python2Packages = python2.pkgs; _merge_python3Packages = true; python3Packages = python3.pkgs; ... ``` So we get merge behavior by default for `pythonPackages`. We can also disable merging in our overlay if desired: ``` $ nix-build -E '(with import ./. { overlays = [ (self: super: { _merge_pythonPackages = false; pythonPackages = self.python3Packages; }) ]; }; pythonPackages.ipython)' /nix/store/0yx0q6v6nisx4fq1vj8x2vc51n7aj5pz-python3.7-ipython-7.1.1 ``` --- lib/fixed-points.nix | 14 +++++++++++++- pkgs/top-level/all-packages.nix | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix index 2f818c88de5db..e28dbb909eb44 100644 --- a/lib/fixed-points.nix +++ b/lib/fixed-points.nix @@ -63,7 +63,19 @@ rec { # = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // { foo = "foo" + " + "; } # = self: { foo = "foo + "; bar = "bar"; foobar = self.foo + self.bar; } # - extends = f: rattrs: self: let super = rattrs self; in super // f self super; + mergeUpdate = lhs: rhs: + let + merge = name: value: + if builtins.hasAttr name lhs + && (rhs."_merge_${name}" or lhs."_merge_${name}" or false) + then mergeUpdate lhs.${name} value + else value; + in lhs // (builtins.mapAttrs merge rhs); + + extends = f: rattrs: self: + let super = rattrs self; + in mergeUpdate super (f self super); + #in super // (f self super); # <-- NON RECURSIVE # Compose two extending functions of the type expected by 'extends' # into one where changes made in the first are available in the diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 90ea01bd6c8b1..18238b422c3d9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7996,8 +7996,11 @@ in python37Full = python37.override{x11Support=true;}; # pythonPackages further below, but assigned here because they need to be in sync + _merge_pythonPackages = true; pythonPackages = python.pkgs; + _merge_python2Packages = true; python2Packages = python2.pkgs; + _merge_python3Packages = true; python3Packages = python3.pkgs; pythonInterpreters = callPackage ./../development/interpreters/python {}; From 47517c05d0bc567faabfa768dfab05cf80b4ae36 Mon Sep 17 00:00:00 2001 From: danbst Date: Fri, 18 Jan 2019 16:05:44 +0200 Subject: [PATCH 2/3] add overriden python packages to fixpoint --- pkgs/top-level/python-packages.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 657e1cb433085..e66abb0607847 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -20,7 +20,7 @@ let let inherit (python.passthru) isPy27 isPy33 isPy34 isPy35 isPy36 isPy37 isPy3k isPyPy pythonAtLeast pythonOlder; - callPackage = pkgs.newScope self; + callPackage = pkgs.newScope (self // python.pkgs); namePrefix = python.libPrefix + "-"; From b09b8c2b64e40a3d91b6fce9dd779e0e5b186b62 Mon Sep 17 00:00:00 2001 From: danbst Date: Fri, 18 Jan 2019 17:44:52 +0200 Subject: [PATCH 3/3] python: use self in one more place --- pkgs/development/interpreters/python/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index f1461d784be06..27203823fbb7b 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -33,7 +33,7 @@ with pkgs; isPyPy = interpreter == "pypy"; buildEnv = callPackage ./wrapper.nix { python = self; inherit (pythonPackages) requiredPythonModules; }; - withPackages = import ./with-packages.nix { inherit buildEnv pythonPackages;}; + withPackages = import ./with-packages.nix { inherit buildEnv; pythonPackages = self.pkgs; }; pkgs = pythonPackages; interpreter = "${self}/bin/${executable}"; inherit executable implementation libPrefix pythonVersion sitePackages;