resholve: 0.5.1 -> 0.6.0, refactor, +binlore

pkgs/development/misc/resholve/README.md

@@ -84,6 +84,9 @@ that the `resholve` CLI expects. Here's an overview:
 | fake          | attrset | [directives](#controlling-resolution-with-directives) |
 | fix           | attrset | [directives](#controlling-resolution-with-directives) |
 | keep          | attrset | [directives](#controlling-resolution-with-directives) |
+| lore          | string  | [lore directory](#controlling-nested-resolution-with-lore)  |
+| execers       | list    | [execer lore directive](#controlling-nested-resolution-with-lore) |
+| wrappers      | list    | [wrapper lore directive](#controlling-nested-resolution-with-lore) |
 
 ## Controlling resolution with directives
 
@@ -156,3 +159,52 @@ keep = {
   "~/.bashrc" = true;
 };
 ```
+
+## Controlling nested resolution with lore
+
+Initially, resolution of commands in the arguments to command-executing
+commands was limited to one level for a hard-coded list of builtins and
+external commands. resholve can now resolve these recursively.
+
+This feature combines information (_lore_) that the resholve Nix API
+obtains via binlore ([nixpkgs](../../tools/analysis/binlore), [repo](https://github.com/abathur/resholve)),
+with some rules (internal to resholve) for locating sub-executions in
+some of the more common commands.
+
+- "execer" lore identifies whether an executable can, cannot,
+  or might execute its arguments. Every "can" or "might" verdict requires
+  either built-in rules for finding the executable, or human triage.
+- "wrapper" lore maps shell exec wrappers to the programs they exec so
+  that resholve can substitute an executable's verdict for its wrapper's.
+
+There will be more mechanisms for controlling this process in the future
+(and your reports/experiences will play a role in shaping them...) For now,
+the main lever is the ability to substitute your own lore. This is how you'd
+do it piecemeal:
+
+```
+# --execer 'cannot:${openssl.bin}/bin/openssl can:${openssl.bin}/bin/c_rehash'
+execer = [
+  /*
+    This is the same verdict binlore will
+    come up with. It's a no-op just to demo
+    how to fiddle lore via the Nix API.
+  */
+  "cannot:${openssl.bin}/bin/openssl"
+  # different verdict, but not used
+  "can:${openssl.bin}/bin/c_rehash"
+];
+
+# --wrapper '${gnugrep}/bin/egrep:${gnugrep}/bin/grep'
+execer = [
+  /*
+    This is the same verdict binlore will
+    come up with. It's a no-op just to demo
+    how to fiddle lore via the Nix API.
+  */
+  "${gnugrep}/bin/egrep:${gnugrep}/bin/grep"
+];
+```
+
+The format is fairly simple to generate--you can script your own generator if
+you need to modify the lore.

pkgs/development/misc/resholve/default.nix

@@ -1,9 +1,18 @@
 { callPackage
-, doCheck ? true
+, ...
 }:
 
+let
+  source = callPackage ./source.nix { };
+  deps = callPackage ./deps.nix { };
+in
 rec {
-  resholve = callPackage ./resholve.nix { inherit doCheck; };
-  resholvePackage =
-    callPackage ./resholve-package.nix { inherit resholve; };
+  resholve = callPackage ./resholve.nix {
+    inherit (source) rSrc;
+    inherit (source) version;
+    inherit (deps.oil) oildev;
+  };
+  resholvePackage = callPackage ./resholve-package.nix {
+    inherit resholve;
+  };
 }

pkgs/development/misc/resholve/oildev.nix

@@ -0,0 +1,121 @@
+{ lib
+, stdenv
+, python27Packages
+, callPackage
+, fetchFromGitHub
+, makeWrapper
+, # re2c deps
+  autoreconfHook
+, # py-yajl deps
+  git
+, # oil deps
+  readline
+, cmark
+, file
+, glibcLocales
+}:
+
+rec {
+  re2c = stdenv.mkDerivation rec {
+    pname = "re2c";
+    version = "1.0.3";
+    sourceRoot = "${src.name}/re2c";
+    src = fetchFromGitHub {
+      owner = "skvadrik";
+      repo = "re2c";
+      rev = version;
+      sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi";
+    };
+    nativeBuildInputs = [ autoreconfHook ];
+    preCheck = ''
+      patchShebangs run_tests.sh
+    '';
+  };
+
+  py-yajl = python27Packages.buildPythonPackage rec {
+    pname = "oil-pyyajl-unstable";
+    version = "2019-12-05";
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "py-yajl";
+      rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df";
+      sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh";
+      fetchSubmodules = true;
+    };
+    # just for submodule IIRC
+    nativeBuildInputs = [ git ];
+  };
+
+  oildev = python27Packages.buildPythonPackage rec {
+    pname = "oildev-unstable";
+    version = "2021-07-14";
+
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "oil";
+      # rev == present HEAD of release/0.8.12
+      rev = "799c0703d1da86cb80d1f5b163edf9369ad77cf1";
+      hash = "sha256-QNSISr719ycZ1Z0quxHWzCb3IvHGj9TpogaYz20hDM4=";
+
+      /*
+        It's not critical to drop most of these; the primary target is
+        the vendored fork of Python-2.7.13, which is ~ 55M and over 3200
+        files, dozens of which get interpreter script patches in fixup.
+      */
+      extraPostFetch = ''
+        rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp
+      '';
+    };
+
+    # TODO: not sure why I'm having to set this for nix-build...
+    #       can anyone tell if I'm doing something wrong?
+    SOURCE_DATE_EPOCH = 315532800;
+
+    # patch to support a python package, pass tests on macOS, etc.
+    patchSrc = fetchFromGitHub {
+      owner = "abathur";
+      repo = "nix-py-dev-oil";
+      rev = "v0.8.12";
+      hash = "sha256-/EvwxL201lGsioL0lIhzM8VTghe6FuVbc3PBJgY8c8E=";
+    };
+    patches = [
+      "${patchSrc}/0001-add_setup_py.patch"
+      "${patchSrc}/0002-add_MANIFEST_in.patch"
+      "${patchSrc}/0004-disable-internal-py-yajl-for-nix-built.patch"
+      "${patchSrc}/0006-disable_failing_libc_tests.patch"
+      "${patchSrc}/0007-namespace_via_init.patch"
+    ];
+
+    buildInputs = [ readline cmark py-yajl ];
+
+    nativeBuildInputs = [ re2c file makeWrapper ];
+
+    propagatedBuildInputs = with python27Packages; [ six typing ];
+
+    doCheck = true;
+
+    preBuild = ''
+      build/dev.sh all
+    '';
+
+    postPatch = ''
+      patchShebangs asdl build core doctools frontend native oil_lang
+    '';
+
+    # TODO: this may be obsolete?
+    _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
+
+    # See earlier note on glibcLocales TODO: verify needed?
+    LOCALE_ARCHIVE = lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive";
+
+    # not exhaustive; just a spot-check for now
+    pythonImportsCheck = [ "oil" "oil._devbuild" ];
+
+    meta = {
+      license = with lib.licenses; [
+        psfl # Includes a portion of the python interpreter and standard library
+        asl20 # Licence for Oil itself
+      ];
+    };
+  };
+}

pkgs/development/misc/resholve/resholve-package.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, resholve }:
+{ stdenv, lib, resholve, binlore }:
 
 { pname
 , src
@@ -10,11 +10,11 @@
 let
   inherit stdenv;
   /* These functions break up the work of partially validating the
-   * 'solutions' attrset and massaging it into env/cli args.
-   *
-   * Note: some of the left-most args do not *have* to be passed as
-   * deep as they are, but I've done so to provide more error context
-   */
+    'solutions' attrset and massaging it into env/cli args.
+
+    Note: some of the left-most args do not *have* to be passed as
+    deep as they are, but I've done so to provide more error context
+  */
 
   # for brevity / line length
   spaces = l: builtins.concatStringsSep " " l;
@@ -72,7 +72,8 @@ let
   /* Build a single resholve invocation */
   makeInvocation = solution: value:
     if validateSolution value then
-      "${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
+    # we pass resholve a directory
+      "RESHOLVE_LORE=${binlore.collect { drvs = value.inputs; } } ${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
     else throw "invalid solution"; # shouldn't trigger for now
 
   /* Build resholve invocation for each solution. */
@@ -87,10 +88,19 @@ let
     # enable below for verbose debug info if needed
     # supports default python.logging levels
     # LOGLEVEL="INFO";
+    /*
+      subshell/PS4/set -x and : command to output resholve envs
+      and invocation. Extra context makes it clearer what the
+      Nix API is doing, makes nix-shell debugging easier, etc.
+    */
     preFixup = ''
-      pushd "$out"
-      ${builtins.concatStringsSep "\n" (makeCommands solutions)}
-      popd
+      (
+        cd "$out"
+        PS4=$'\x1f'"\033[33m[resholve context]\033[0m "
+        set -x
+        : changing directory to $PWD
+        ${builtins.concatStringsSep "\n" (makeCommands solutions)}
+      )
     '';
   }));
 in