pythonPackages: ensure all derivations provide python modules

This adds a test to ensure no new uses of `buildPythonApplication` can
be added to `python-packages.nix`.

Python packages can be grouped into two groups: 1) applications and 2)
packages providing importable modules. In `python-packages.nix` we only
want to have 2). 1) should be in the top-level package set.

To achieve this, all setup hooks need to be marked as being a setup hook.
For the setup hooks in the Python packages set this is done by creating
a new builder, `makePythonHook`.

Because there were issues with splicing, the file importing all the hooks
is converted to an extension. All non-packages were moved out of `python-packages.nix`
into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing
was cleaned up as well; there is no need to keep this one manually in sync
reducing the risk of breaking cross-compilation.

pkgs/development/interpreters/python/default.nix

@@ -31,13 +31,19 @@
     , pythonAttr ? null
     , self # is pythonOnHostForTarget
     }: let
-      pythonPackages = callPackage
+      pythonPackages = let
+        ensurePythonModules = items: let
+          providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
+          notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
+          func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
+        in lib.mapAttrs func items;
+      in ensurePythonModules (callPackage
         # Function that when called
         # - imports python-packages.nix
         # - adds spliced package sets to the package set
         # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
         ({ pkgs, stdenv, python, overrides }: let
-          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+          pythonPackagesFun = import ./python-packages-base.nix {
             inherit stdenv pkgs lib;
             python = self;
           };
@@ -48,59 +54,31 @@
             selfHostHost = pythonOnHostForHost.pkgs;
             selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
           };
-          keep = self: {
-            # TODO maybe only define these here so nothing is needed to be kept in sync.
-            inherit (self)
-              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
-              python bootstrapped-pip buildPythonPackage buildPythonApplication
-              fetchPypi
-              hasPythonModule requiredPythonModules makePythonPath disabledIf
-              toPythonModule toPythonApplication
-              buildSetupcfg
-
-              condaInstallHook
-              condaUnpackHook
-              eggUnpackHook
-              eggBuildHook
-              eggInstallHook
-              flitBuildHook
-              pipBuildHook
-              pipInstallHook
-              pytestCheckHook
-              pythonCatchConflictsHook
-              pythonImportsCheckHook
-              pythonNamespacesHook
-              pythonRecompileBytecodeHook
-              pythonRemoveBinBytecodeHook
-              pythonRemoveTestsDirHook
-              setuptoolsBuildHook
-              setuptoolsCheckHook
-              venvShellHook
-              wheelUnpackHook
-
-              wrapPython
-
-              pythonPackages
-
-              recursivePthLoader
-            ;
-          };
+          hooks = import ./hooks/default.nix;
+          keep = lib.extends hooks pythonPackagesFun;
           extra = _: {};
           optionalExtensions = cond: as: if cond then as else [];
+          pythonExtension = import ../../../top-level/python-packages.nix;
           python2Extension = import ../../../top-level/python2-packages.nix;
-          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
+          extensions = lib.composeManyExtensions ([
+            pythonExtension
+          ] ++ (optionalExtensions (!self.isPy3k) [
+            python2Extension
+          ]) ++ pythonPackagesExtensions ++ [
+            overrides
+          ]);
           aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
         in lib.makeScopeWithSplicing
           splicePackages
           newScope
           otherSplices
           keep
           extra
-          (lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
+          (lib.extends (lib.composeExtensions aliases extensions) keep))
         {
           overrides = packageOverrides;
           python = self;
-        };
+        });
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";

pkgs/development/interpreters/python/hooks/default.nix

@@ -1,123 +1,117 @@
-# Hooks for building Python packages.
-{ python
-, lib
-, makeSetupHook
-, disabledIf
-, isPy3k
-}:
+self: super: with self;
 
 let
-  callPackage = python.pythonForBuild.pkgs.callPackage;
-  pythonInterpreter = python.pythonForBuild.interpreter;
-  pythonSitePackages = python.sitePackages;
-  pythonCheckInterpreter = python.interpreter;
+  pythonInterpreter = super.python.pythonForBuild.interpreter;
+  pythonSitePackages = super.python.sitePackages;
+  pythonCheckInterpreter = super.python.interpreter;
   setuppy = ../run_setup.py;
-in rec {
+in {
+  makePythonHook = args: pkgs.makeSetupHook ({passthru.provides.setupHook = true; } // args);
 
-  condaInstallHook = callPackage ({ gnutar, lbzip2 }:
-    makeSetupHook {
+  condaInstallHook = callPackage ({ makePythonHook, gnutar, lbzip2 }:
+    makePythonHook {
       name = "conda-install-hook";
       deps = [ gnutar lbzip2 ];
       substitutions = {
         inherit pythonSitePackages;
       };
     } ./conda-install-hook.sh) {};
 
-  condaUnpackHook = callPackage ({}:
-    makeSetupHook {
+  condaUnpackHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "conda-unpack-hook";
       deps = [];
     } ./conda-unpack-hook.sh) {};
 
-  eggBuildHook = callPackage ({ }:
-    makeSetupHook {
+  eggBuildHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "egg-build-hook.sh";
       deps = [ ];
     } ./egg-build-hook.sh) {};
 
-  eggInstallHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  eggInstallHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "egg-install-hook.sh";
       deps = [ setuptools ];
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
       };
     } ./egg-install-hook.sh) {};
 
-  eggUnpackHook = callPackage ({ }:
-    makeSetupHook {
+  eggUnpackHook = callPackage ({ makePythonHook, }:
+    makePythonHook {
       name = "egg-unpack-hook.sh";
       deps = [ ];
     } ./egg-unpack-hook.sh) {};
 
-  flitBuildHook = callPackage ({ flit }:
-    makeSetupHook {
+  flitBuildHook = callPackage ({ makePythonHook, flit }:
+    makePythonHook {
       name = "flit-build-hook";
       deps = [ flit ];
       substitutions = {
         inherit pythonInterpreter;
       };
     } ./flit-build-hook.sh) {};
 
-  pipBuildHook = callPackage ({ pip, wheel }:
-    makeSetupHook {
+  pipBuildHook = callPackage ({ makePythonHook, pip, wheel }:
+    makePythonHook {
       name = "pip-build-hook.sh";
       deps = [ pip wheel ];
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
       };
     } ./pip-build-hook.sh) {};
 
-  pipInstallHook = callPackage ({ pip }:
-    makeSetupHook {
+  pipInstallHook = callPackage ({ makePythonHook, pip }:
+    makePythonHook {
       name = "pip-install-hook";
       deps = [ pip ];
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
       };
     } ./pip-install-hook.sh) {};
 
-  pytestCheckHook = callPackage ({ pytest }:
-    makeSetupHook {
+  pytestCheckHook = callPackage ({ makePythonHook, pytest }:
+    makePythonHook {
       name = "pytest-check-hook";
       deps = [ pytest ];
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./pytest-check-hook.sh) {};
 
-  pythonCatchConflictsHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  pythonCatchConflictsHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "python-catch-conflicts-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages setuptools;
         catchConflicts=../catch_conflicts/catch_conflicts.py;
       };
     } ./python-catch-conflicts-hook.sh) {};
 
-  pythonImportsCheckHook = callPackage ({}:
-    makeSetupHook {
+  pythonImportsCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-imports-check-hook.sh";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./python-imports-check-hook.sh) {};
 
-  pythonNamespacesHook = callPackage ({ findutils }:
-    makeSetupHook {
+  pythonNamespacesHook = callPackage ({ makePythonHook, findutils }:
+    makePythonHook {
       name = "python-namespaces-hook.sh";
       substitutions = {
         inherit pythonSitePackages findutils;
       };
     } ./python-namespaces-hook.sh) {};
 
-  pythonOutputDistHook = callPackage ({ }:
-    makeSetupHook {
+  pythonOutputDistHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-output-dist-hook";
   } ./python-output-dist-hook.sh ) {};
 
-  pythonRecompileBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRecompileBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-recompile-bytecode-hook";
       substitutions = {
         inherit pythonInterpreter pythonSitePackages;
@@ -126,71 +120,75 @@ in rec {
       };
     } ./python-recompile-bytecode-hook.sh ) {};
 
-  pythonRelaxDepsHook = callPackage ({ wheel }:
-    makeSetupHook {
+  pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "python-relax-deps-hook";
       deps = [ wheel ];
       substitutions = {
         inherit pythonInterpreter;
       };
     } ./python-relax-deps-hook.sh) {};
 
-  pythonRemoveBinBytecodeHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveBinBytecodeHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-bin-bytecode-hook";
     } ./python-remove-bin-bytecode-hook.sh) {};
 
-  pythonRemoveTestsDirHook = callPackage ({ }:
-    makeSetupHook {
+  pythonRemoveTestsDirHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "python-remove-tests-dir-hook";
       substitutions = {
         inherit pythonSitePackages;
       };
     } ./python-remove-tests-dir-hook.sh) {};
 
-  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
-    makeSetupHook {
+  setuptoolsBuildHook = callPackage ({ makePythonHook, setuptools, wheel }:
+    makePythonHook {
       name = "setuptools-setup-hook";
       deps = [ setuptools wheel ];
       substitutions = {
         inherit pythonInterpreter pythonSitePackages setuppy;
       };
     } ./setuptools-build-hook.sh) {};
 
-  setuptoolsCheckHook = callPackage ({ setuptools }:
-    makeSetupHook {
+  setuptoolsCheckHook = callPackage ({ makePythonHook, setuptools }:
+    makePythonHook {
       name = "setuptools-check-hook";
       deps = [ setuptools ];
       substitutions = {
         inherit pythonCheckInterpreter setuppy;
       };
     } ./setuptools-check-hook.sh) {};
 
-  unittestCheckHook = callPackage ({ }:
-    makeSetupHook {
+  unittestCheckHook = callPackage ({ makePythonHook }:
+    makePythonHook {
       name = "unittest-check-hook";
       substitutions = {
         inherit pythonCheckInterpreter;
       };
     } ./unittest-check-hook.sh) {};
 
-  venvShellHook = disabledIf (!isPy3k) (callPackage ({ ensureNewerSourcesForZipFilesHook }:
-    makeSetupHook {
+  venvShellHook = disabledIf (!isPy3k) (callPackage ({ makePythonHook, ensureNewerSourcesForZipFilesHook }:
+    makePythonHook {
       name = "venv-shell-hook";
       deps = [ ensureNewerSourcesForZipFilesHook ];
       substitutions = {
         inherit pythonInterpreter;
       };
     } ./venv-shell-hook.sh) {});
 
-  wheelUnpackHook = callPackage ({ wheel }:
-    makeSetupHook {
+  wheelUnpackHook = callPackage ({ makePythonHook, wheel }:
+    makePythonHook {
       name = "wheel-unpack-hook.sh";
       deps = [ wheel ];
     } ./wheel-unpack-hook.sh) {};
 
-  sphinxHook = callPackage ({ sphinx, installShellFiles }:
-    makeSetupHook {
+  wrapPython = callPackage ../wrap-python.nix {
+    inherit (pkgs.buildPackages) makeWrapper;
+  };
+
+  sphinxHook = callPackage ({ makePythonHook, sphinx, installShellFiles }:
+    makePythonHook {
       name = "python${python.pythonVersion}-sphinx-hook";
       deps = [ sphinx installShellFiles ];
     } ./sphinx-hook.sh) {};