Package crypto libs for GHAs (#533)

This packages libsodium{-vrf}, libsecp256k1, libblst with nix for the GHA runners.

Co-authored-by: Andrea Bedini <andrea@andreabedini.com>

---------

Co-authored-by: Andrea Bedini <andrea@andreabedini.com>

.github/workflows/github-action.yml

@@ -0,0 +1,17 @@
+name: Validate IOG Core GH Action
+on: push
+jobs:
+  integration:
+    strategy:
+        fail-fast: false
+        matrix:
+          os: [ubuntu-latest, windows-latest, macos-latest]
+    name: GitHub Action integration test
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: input-output-hk/actions/base@latest
+    - name: Check pkg-config
+      run: |
+        pkg-config --libs libblst
+        pkg-config --libs libsodium
+        pkg-config --libs libsecp256k1

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install nix
+        uses: cachix/install-nix-action@v20
+        with:
+          nix_path: nixpkgs=https://github.com/nixos/nixpkgs/archive/release-22.11.tar.gz
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
+            substituters = https://cache.nixos.org/ https://cache.iog.io/ https://cache.zw3rk.com/
+      - name: Build
+        uses: workflow/nix-shell-action@v3.2.1
+        with:
+          packages: zip
+          script: |
+            mkdir __pkgs
+            echo ${{ github.sha }} > __pkgs/COMMIT_SHA
+            for sys in msys2 macos debian; do
+              for lib in libsodium libsodium-vrf libsecp256k1 libblst; do
+                out=$(nix build .#dist.$sys.$lib --no-link --print-out-paths -L)
+                for ext in pkg.tar.zstd pkg deb; do
+                  if [ -f "$out"/*."$ext" ]; then
+                    cp "$out"/*."$ext" __pkgs/"$sys.$lib.$ext"
+                    base=$(basename "$out"/*."$ext")
+                    echo -e "$sys.$lib.$ext\t$base" >> __pkgs/INFO
+                  fi
+                done
+              done
+              (cd __pkgs && zip -9 $sys.zip $sys.*)
+            done
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: __pkgs/*

flake.nix

@@ -1,6 +1,8 @@
 {
   description = "IOHK nix lib, packages and overlays";
 
+  inputs.nixpkgs.url = "github:nixos/nixpkgs?ref=release-22.11";
+
   outputs = { self, nixpkgs }: rec {
 
     lib = import ./lib nixpkgs.lib;
@@ -28,6 +30,239 @@
     };
 
     pkgs = import nixpkgs { system = "x86_64-linux"; overlays = builtins.attrValues overlays; };
+    darwin-pkgs = import nixpkgs { system = "x86_64-darwin"; overlays = builtins.attrValues overlays; };
+
+
+    # we can use this, to get a coherent picture of the sources for
+    # the various libraries.  The following command will produce a
+    # JSON output, that contains each of our libs, with their respective
+    # versions.
+    #
+    #    nix eval --json .#lib-srcs
+    #
+    lib-srcs = {
+      secp256k1 = pkgs.secp256k1.src.url;
+      sodium    = pkgs.libsodium-vrf.src.url;
+      blst      = pkgs.libblst.src.url;
+    };
+
+    dist = let
+      # For packaging, we can'd deal with split outputs.
+      mkSingleOutput = drv: drv.overrideDerivation (drv': { outputs = [ "out" ]; });
+      mkDebianPkg = prefix: drv: let
+        control = pkgs.writeText "control" ''
+        Package: ${drv.pname}
+        Version: ${if drv.version == "unstable-2022-02-06" then "2022-02-06-unstable" else drv.version}
+        Architecture: amd64
+        Maintainer: IOG <engineering@iog.io>
+        Description: ${drv.meta.description}
+        '';
+      in pkgs.stdenv.mkDerivation {
+        inherit (drv) version meta;
+        name = "${drv.name}-debian-pkg";
+        phases = [ "buildPhase" "installPhase" ];
+        buildInputs = with pkgs; [ rsync ];
+        buildPhase = ''
+          mkdir -p .${prefix}
+          rsync -a ${drv}/ .${prefix}
+
+          ls -lah
+
+          # replace any reference to the nix-path in the pkg-config files with
+          # references to the target prefix
+
+          for pc in $(find .${prefix} -name "*.pc"); do
+            substituteInPlace $pc --replace "${drv}" "${prefix}"
+            cat $pc
+          done
+
+          # create the data.tar.gz containing the install tree; we need to
+          # have data.tar.gz exist in . or tar will complain that . changed
+          # while creating the archive.
+          touch data.tar.gz
+          tar --exclude=env-vars --exclude=data.tar.gz -czf data.tar.gz .
+
+          # create the minimal control file, and control.tar.gz
+          substituteAll ${control} control
+          tar czf control.tar.gz control
+
+          # create the debian-binary file
+          echo 2.0 > debian-binary
+
+          # package it up. It's just ar.
+          ar r ${drv.name}.deb debian-binary control.tar.gz data.tar.gz
+        '';
+        installPhase = ''
+          mkdir -p $out
+          mv ${drv.name}.deb $out/
+
+          # make it downloadable from hydra.
+          mkdir -p $out/nix-support
+          for f in $out/*.deb; do
+            echo "file binary-dist \"''${f}\"" \
+                >> $out/nix-support/hydra-build-products
+          done
+        '';
+      };
+      mkDarwinPkg = prefix: drv: let
+        PackageInfo = pkgs.writeText "PackageInfo" ''
+        <?xml version="1.0" encoding="utf-8" standalone="no"?>
+        <pkg-info identifier="io.iog.${drv.pname}" version="${drv.version}" format-version="2" auth="root" install-location="${prefix}">
+          <payload numberOfFiles="@numfiles@" installKBytes="@kbsize@"/>
+          <scripts>
+            <postinstall file="./postinstall"/>
+          </scripts>
+        </pkg-info>
+        '';
+        PostInstall = pkgs.writeText "PostInstall" ''
+        #!/bin/bash
+        for lib in @libs@; do
+          chmod +w "${prefix}/$lib"
+          install_name_tool -id "${prefix}/$lib" "${prefix}/$lib"
+          chmod -w "${prefix}/$lib"
+        done
+        '';
+      in pkgs.stdenv.mkDerivation {
+        inherit (drv) version meta;
+        name = "${drv.name}-macos-pkg";
+        phases = [ "buildPhase" "installPhase" ];
+        buildInputs = with pkgs; [ rsync xar cpio bomutils ];
+        buildPhase = ''
+          mkdir -p pkg
+          rsync -a ${drv}/ pkg
+
+          # replace any reference to the nix-path in the pkg-config files with
+          # references to the target prefix
+
+          for pc in $(find pkg -name "*.pc"); do
+            substituteInPlace $pc --replace "${drv}" "${prefix}"
+            cat $pc
+          done
+
+          export numfiles=$(find pkg/ |wc -l)
+          export kbsize=$(du -ks pkg/ |cut -f1)
+
+          substituteAll ${PackageInfo} PackageInfo
 
+          (cd pkg; find . | cpio -o --format odc --owner 0:80 | gzip -c ) > Payload
+          mkbom -u 0 -g 80 pkg/ Bom
+
+          mkdir -p scripts
+
+          # ensure we drop the ./ from the found results, by using cut.
+          export libs=$(cd pkg; find . -name "*.dylib" -type f | cut -c 3-)
+
+          substituteAll ${PostInstall} scripts/postinstall
+          chmod +x scripts/postinstall
+
+          (cd scripts; find . | cpio -o --format odc --owner 0:80 | gzip -c ) > Scripts
+
+          xar --compression none -cf "${drv.name}.pkg" * --exclude="${drv.name}.pkg" --exclude=pkg/ --exclude=scripts/
+
+        '';
+        installPhase = ''
+          mkdir -p $out
+          mv ${drv.name}.pkg $out/
+
+          # make it downloadable from hydra.
+          mkdir -p $out/nix-support
+          for f in $out/*.pkg; do
+            echo "file binary-dist \"''${f}\"" \
+                >> $out/nix-support/hydra-build-products
+          done
+        '';
+      };
+      mkPacmanPkg = prefix: drv: let
+        pkgrel = 1; # pkg release version
+        PKGINFO = pkgs.writeText ".PKGINFO" ''
+        # Generated by packaging.nix
+        pkgname = mingw-w64-x86_64-${drv.pname}
+        pkgbase = mingw-w64-${drv.pname}
+        pkgver = ${drv.version}-${toString pkgrel}
+        pkgdesc = ${drv.meta.description} (mingw-w64)
+        url = ${drv.meta.homepage}
+        builddate = @date@
+        packager = packaging.nix
+        size = @size@
+        arch = any
+        license = custom:ISC
+        '';
+      in pkgs.stdenv.mkDerivation {
+        inherit (drv) version meta;
+        name = "${drv.name}-msys2-pkg";
+        phases = [ "buildPhase" "installPhase" ];
+        buildInputs = with pkgs; [ rsync tree libarchive zstd ];
+        buildPhase = ''
+          mkdir -p .${prefix}
+          rsync -a ${drv}/ .${prefix}
+
+          # replace any reference to the nix-path in the pkg-config files with
+          # references to the target prefix
+
+          for pc in $(find .${prefix} -name "*.pc"); do
+            substituteInPlace $pc --replace "${drv}" "${prefix}"
+            cat $pc
+          done
+
+          # this date is of course not correct, but we don't want the derivation
+          # to mutate all the time, just because it was re-built.
+          export date=0
+          export size=$(du -bs $out|cut -f1)
+
+          substituteAll ${PKGINFO} .PKGINFO
+
+          list_package_files() {
+            (
+              export LC_COLLATE=C
+              shopt -s dotglob globstar
+              # bash 5.0 only works with combo directory + file globs
+              printf '%s\0' **/*
+            )
+          }
+
+          list_package_files | LANG=C bsdtar -cnf - --format=mtree \
+        		--options='!all,use-set,type,uid,gid,mode,time,size,sha256,link' \
+        		--null --files-from - \
+            --exclude env-vars --exclude .MTREE \
+            | gzip -c -f -n > .MTREE
+
+          list_package_files | LANG=C bsdtar --no-fflags --no-read-sparse -cnf - \
+            --null --files-from - \
+            --exclude env-vars --exclude ${drv.name}.pkg.tar.zstd \
+            | zstd > ${drv.name}.pkg.tar.zstd
+        '';
+        installPhase = ''
+          mkdir -p $out
+          mv ${drv.name}.pkg.tar.zstd $out/
+
+          # make it downloadable from hydra.
+          mkdir -p $out/nix-support
+          for f in $out/*.zstd; do
+            echo "file binary-dist \"''${f}\"" \
+                >> $out/nix-support/hydra-build-products
+          done
+        '';
+      };
+    in {
+      msys2 = {
+        libsodium-vrf= mkPacmanPkg "/mingw64/opt/cardano" (mkSingleOutput pkgs.pkgsCross.mingwW64.libsodium-vrf);
+        libsodium    = mkPacmanPkg "/mingw64/opt/cardano" (mkSingleOutput pkgs.pkgsCross.mingwW64.libsodium);
+        libblst      = mkPacmanPkg "/mingw64/opt/cardano" (mkSingleOutput pkgs.pkgsCross.mingwW64.libblst);
+        libsecp256k1 = mkPacmanPkg "/mingw64/opt/cardano" (mkSingleOutput pkgs.pkgsCross.mingwW64.secp256k1);
+      };
+      macos = {
+        libsodium-vrf= mkDarwinPkg "/usr/local/opt/cardano" (mkSingleOutput darwin-pkgs.libsodium-vrf);
+        libsodium    = mkDarwinPkg "/usr/local/opt/cardano" (mkSingleOutput darwin-pkgs.libsodium);
+        libblst      = mkDarwinPkg "/usr/local/opt/cardano" (mkSingleOutput darwin-pkgs.libblst);
+        libsecp256k1 = mkDarwinPkg "/usr/local/opt/cardano" (mkSingleOutput darwin-pkgs.secp256k1);
+      };
+      debian = {
+        libsodium-vrf= mkDebianPkg "/usr/local/opt/cardano" (mkSingleOutput pkgs.libsodium-vrf);
+        libsodium    = mkDebianPkg "/usr/local/opt/cardano" (mkSingleOutput pkgs.libsodium);
+        libblst      = mkDebianPkg "/usr/local/opt/cardano" (mkSingleOutput pkgs.libblst);
+        libsecp256k1 = mkDebianPkg "/usr/local/opt/cardano" (mkSingleOutput pkgs.secp256k1);
+      };
+    };
+    hydraJobs = dist;
   };
 }

overlays/crypto/libblst.nix

@@ -1,7 +1,7 @@
 { stdenv, lib, fetchFromGitHub, autoreconfHook }:
 
 stdenv.mkDerivation rec {
-  name = "blst-0.3.10";
+  pname = "blst";
   version = "0.3.10";
 
   src = fetchFromGitHub {
@@ -17,12 +17,14 @@ stdenv.mkDerivation rec {
     ./build.sh -shared ${lib.optionalString stdenv.targetPlatform.isWindows "flavour=mingw64"}
   '';
   installPhase = ''
-    mkdir -p $out/lib
+    mkdir -p $out/{lib,include}
     for lib in libblst.{a,so,dylib}; do
       if [ -f $lib ]; then
         cp $lib $out/lib/
       fi
     done
+    cp bindings/{blst.h,blst_aux.h} $out/include
+
     for lib in blst.dll; do
       if [ -f $lib ]; then
         mkdir -p $out/bin
@@ -34,13 +36,17 @@ stdenv.mkDerivation rec {
     cat <<EOF > $out/lib/pkgconfig/libblst.pc
     prefix=$out
     exec_prefix=''\\''${prefix}
-    libdir=$out/lib
+    libdir=''\\''${exec_prefix}/lib
+    includedir=''\\''${prefix}/include
 
     Name: libblst
-    Version: ${version}
     Description: ${meta.description}
+    URL: ${meta.homepage}
+    Version: ${version}
 
+    Cflags: -I''\\''${includedir}
     Libs: -L''\\''${libdir} -lblst
+    Libs.private:
     EOF
   '';
 

overlays/crypto/libsodium.nix

@@ -1,7 +1,8 @@
 { stdenv, lib, fetchFromGitHub, autoreconfHook }:
 
 stdenv.mkDerivation rec {
-  name = "libsodium-1.0.18";
+  pname = "libsodium-vrf";
+  version = "1.0.18";
 
   src = fetchFromGitHub {
     owner = "input-output-hk";