diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 36a5c9843c2f0..572fcbc116217 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -156,6 +156,7 @@ in { couchdb = handleTest ./couchdb.nix {}; cri-o = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cri-o.nix {}; cups-pdf = handleTest ./cups-pdf.nix {}; + curl-impersonate = handleTest ./curl-impersonate.nix {}; custom-ca = handleTest ./custom-ca.nix {}; croc = handleTest ./croc.nix {}; deluge = handleTest ./deluge.nix {}; diff --git a/nixos/tests/curl-impersonate.nix b/nixos/tests/curl-impersonate.nix new file mode 100644 index 0000000000000..7954e9e5584c4 --- /dev/null +++ b/nixos/tests/curl-impersonate.nix @@ -0,0 +1,157 @@ +/* + Test suite for curl-impersonate + + Abstract: + Uses the test suite from the curl-impersonate source repo which: + + 1. Performs requests with libcurl and captures the TLS client-hello + packets with tcpdump to compare against known-good signatures + 2. Spins up an nghttpd2 server to test client HTTP/2 headers against + known-good headers + + See https://github.com/lwthiker/curl-impersonate/tree/main/tests/signatures + for details. + + Notes: + - We need to have our own web server running because the tests expect to be able + to hit domains like wikipedia.org and the sandbox has no internet + - We need to be able to do (verifying) TLS handshakes without internet access. + We do that by creating a trusted CA and issuing a cert that includes + all of the test domains as subject-alternative names and then spoofs the + hostnames in /etc/hosts. +*/ + +import ./make-test-python.nix ({ pkgs, lib, ... }: let + # Update with domains in TestImpersonate.TEST_URLS if needed from: + # https://github.com/lwthiker/curl-impersonate/blob/main/tests/test_impersonate.py + domains = [ + "www.wikimedia.org" + "www.wikipedia.org" + "www.mozilla.org" + "www.apache.org" + "www.kernel.org" + "git-scm.com" + ]; + + tls-certs = let + # Configure CA with X.509 v3 extensions that would be trusted by curl + ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" '' + basicConstraints = critical, CA:TRUE + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always, issuer:always + keyUsage = critical, cRLSign, digitalSignature, keyCertSign + ''; + + # Configure leaf certificate with X.509 v3 extensions that would be trusted + # by curl and set subject-alternative names for test domains + tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" '' + basicConstraints = critical, CA:FALSE + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always, issuer:always + keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement + extendedKeyUsage = critical, serverAuth + subjectAltName = @alt_names + + [alt_names] + ${lib.concatStringsSep "\n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)} + ''; + in pkgs.runCommand "curl-impersonate-test-certs" { + nativeBuildInputs = [ pkgs.openssl ]; + } '' + # create CA certificate and key + openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test' + openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500 + openssl x509 -in ca.pem -text + + # create server certificate and key + openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test' + openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500 + openssl x509 -in cert.pem -text + + # output CA cert and server cert and key + mkdir -p $out + cp key.pem cert.pem ca.pem $out + ''; + + # Test script + curl-impersonate-test = let + # Build miniature libcurl client used by test driver + minicurl = pkgs.runCommandCC "minicurl" { + buildInputs = [ pkgs.curl ]; + } '' + mkdir -p $out/bin + $CC -Wall -Werror -o $out/bin/minicurl ${pkgs.curl-impersonate.src}/tests/minicurl.c `curl-config --libs` + ''; + in pkgs.writeShellScript "curl-impersonate-test" '' + set -euxo pipefail + + # Test driver requirements + export PATH="${with pkgs; lib.makeBinPath [ + bash + coreutils + python3Packages.pytest + nghttp2 + tcpdump + ]}" + export PYTHONPATH="${with pkgs.python3Packages; makePythonPath [ + pyyaml + pytest-asyncio + dpkt + ]}" + + # Prepare test root prefix + mkdir -p usr/{bin,lib} + cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/ + + cp -r ${pkgs.curl-impersonate.src}/tests ./ + + # Run tests + cd tests + pytest . --install-dir ../usr --capture-interface eth1 + ''; +in { + name = "curl-impersonate"; + + meta = with lib.maintainers; { + maintainers = [ lilyinstarlight ]; + }; + + nodes = { + web = { nodes, pkgs, lib, config, ... }: { + networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services = { + nginx = { + enable = true; + virtualHosts."curl-impersonate.nixos.test" = { + default = true; + addSSL = true; + sslCertificate = "${tls-certs}/cert.pem"; + sslCertificateKey = "${tls-certs}/key.pem"; + }; + }; + }; + }; + + curl = { nodes, pkgs, lib, config, ... }: { + networking.extraHosts = lib.concatStringsSep "\n" (map (domain: "${nodes.web.networking.primaryIPAddress} ${domain}") domains); + + security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ]; + }; + }; + + testScript = { nodes, ... }: '' + start_all() + + with subtest("Wait for network"): + web.wait_for_unit("network-online.target") + curl.wait_for_unit("network-online.target") + + with subtest("Wait for web server"): + web.wait_for_unit("nginx.service") + web.wait_for_open_port(443) + + with subtest("Run curl-impersonate tests"): + curl.succeed("${curl-impersonate-test}") + ''; +}) diff --git a/pkgs/tools/networking/curl-impersonate/curl-impersonate-0.5.2-fix-shebangs.patch b/pkgs/tools/networking/curl-impersonate/curl-impersonate-0.5.2-fix-shebangs.patch new file mode 100644 index 0000000000000..7082c25ac148b --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/curl-impersonate-0.5.2-fix-shebangs.patch @@ -0,0 +1,13 @@ +diff --git a/Makefile.in b/Makefile.in +index 877c54f..3e39ed1 100644 +--- a/Makefile.in ++++ b/Makefile.in +@@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz: + + $(nss_static_libs): $(NSS_VERSION).tar.gz + tar xf $(NSS_VERSION).tar.gz ++ sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh ++ sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi + + ifeq ($(host),$(build)) + # Native build, use NSS' build script. diff --git a/pkgs/tools/networking/curl-impersonate/default.nix b/pkgs/tools/networking/curl-impersonate/default.nix index e00b1a77ef2bb..a753322504bfe 100644 --- a/pkgs/tools/networking/curl-impersonate/default.nix +++ b/pkgs/tools/networking/curl-impersonate/default.nix @@ -1,27 +1,186 @@ -#TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries. -{ lib, stdenv, fetchzip, zlib, autoPatchelfHook }: -stdenv.mkDerivation rec { - pname = "curl-impersonate-bin"; - version = "v0.5.3"; - - src = fetchzip { - url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz"; - sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g="; - stripRoot = false; +{ lib +, stdenv +, fetchFromGitHub +, fetchpatch +, callPackage +, buildGoModule +, installShellFiles +, symlinkJoin +, zlib +, sqlite +, cmake +, python3 +, ninja +, perl +, autoconf +, automake +, libtool +, darwin +, cacert +, unzip +, go +, p11-kit +, nixosTests +}: + +let + makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec { + pname = "curl-impersonate-${name}"; + version = "0.5.4"; + + src = fetchFromGitHub { + owner = "lwthiker"; + repo = "curl-impersonate"; + rev = "v${version}"; + hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A="; + }; + + patches = [ + # Fix shebangs in the NSS build script + # (can't just patchShebangs since makefile unpacks it) + ./curl-impersonate-0.5.2-fix-shebangs.patch + ]; + + strictDeps = true; + + nativeBuildInputs = lib.optionals stdenv.isDarwin [ + # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize' + darwin.cctools + ] ++ [ + installShellFiles + cmake + python3 + python3.pkgs.gyp + ninja + perl + autoconf + automake + libtool + unzip + go + ]; + + buildInputs = [ + zlib + sqlite + ]; + + configureFlags = [ + "--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}" + "--with-ca-path=${cacert}/etc/ssl/certs" + ]; + + buildFlags = [ "${target}-build" ]; + checkTarget = "${target}-checkbuild"; + installTargets = [ "${target}-install" ]; + + doCheck = true; + + dontUseCmakeConfigure = true; + dontUseNinjaBuild = true; + dontUseNinjaInstall = true; + dontUseNinjaCheck = true; + + postUnpack = lib.concatStringsSep "\n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps)); + + preConfigure = '' + export GOCACHE=$TMPDIR/go-cache + export GOPATH=$TMPDIR/go + export GOPROXY=file://${passthru.boringssl-go-modules} + export GOSUMDB=off + + # Need to get value of $out for this flag + configureFlagsArray+=("--with-libnssckbi=$out/lib") + ''; + + postInstall = '' + # Remove vestigial *-config script + rm $out/bin/curl-impersonate-${name}-config + + # Patch all shebangs of installed scripts + patchShebangs $out/bin + + # Build and install completions for each curl binary + + # Patch in correct binary name and alias it to all scripts + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh + substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \ + --replace \ + '#compdef curl' \ + "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')" + + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish + substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \ + --replace \ + '--command curl' \ + "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')" + + # Install zsh and fish completions + installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish} + ''; + + preFixup = let + libext = stdenv.hostPlatform.extensions.sharedLibrary; + in '' + # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure + if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then + # NOTE: "p11-kit-trust" always ends in ".so" even when on darwin + ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext} + ${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"} + fi + ''; + + disallowedReferences = [ go ]; + + passthru = { + deps = callPackage ./deps.nix {}; + + boringssl-go-modules = (buildGoModule { + inherit (passthru.deps."boringssl.zip") name; + + src = passthru.deps."boringssl.zip"; + vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q="; + + nativeBuildInputs = [ unzip ]; + + proxyVendor = true; + }).go-modules; + }; + + meta = with lib; { + description = "A special build of curl that can impersonate Chrome & Firefox"; + homepage = "https://github.com/lwthiker/curl-impersonate"; + license = with licenses; [ curl mit ]; + maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ]; + platforms = platforms.unix; + knownVulnerabilities = [ + "CVE-2023-32001" # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html + "CVE-2022-43551" # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html + "CVE-2022-42916" # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html + ]; + }; }; +in + +symlinkJoin rec { + pname = "curl-impersonate"; + inherit (passthru.curl-impersonate-ff) version meta; + + name = "${pname}-${version}"; + + paths = [ + passthru.curl-impersonate-ff + passthru.curl-impersonate-chrome + ]; + + passthru = { + curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; }; + curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; }; - nativeBuildInputs = [ autoPatchelfHook zlib ]; + updateScript = ./update.sh; - installPhase = '' - mkdir -p $out/bin - cp * $out/bin - ''; + inherit (passthru.curl-impersonate-ff) src; - meta = with lib; { - description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox "; - homepage = "https://github.com/lwthiker/curl-impersonate"; - license = with licenses; [ curl mit ]; - maintainers = with maintainers; [ deliciouslytyped ]; - platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere. + tests = { inherit (nixosTests) curl-impersonate; }; }; } diff --git a/pkgs/tools/networking/curl-impersonate/deps.nix b/pkgs/tools/networking/curl-impersonate/deps.nix new file mode 100644 index 0000000000000..498616247dce9 --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/deps.nix @@ -0,0 +1,29 @@ +# Generated by update.sh +{ fetchurl }: + +{ + "curl-7.84.0.tar.xz" = fetchurl { + url = "https://curl.se/download/curl-7.84.0.tar.xz"; + hash = "sha256-LRGLQ/VHv+W66AbY1HtOWW6lslpsHwgK70n7zYF8Xbg="; + }; + + "brotli-1.0.9.tar.gz" = fetchurl { + url = "https://github.com/google/brotli/archive/refs/tags/v1.0.9.tar.gz"; + hash = "sha256-+ejYHQQFumbRgVKa9CozVPg4yTkJX/mZMNpqqc32/kY="; + }; + + "nss-3.87.tar.gz" = fetchurl { + url = "https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_87_RTM/src/nss-3.87-with-nspr-4.35.tar.gz"; + hash = "sha256-63DqC1jc5pqkkOnp/s0TKn1kTh2j1jHhYzdqDcwRoCI="; + }; + + "boringssl.zip" = fetchurl { + url = "https://github.com/google/boringssl/archive/3a667d10e94186fd503966f5638e134fe9fb4080.zip"; + hash = "sha256-HsDIkd1x5IH49fUF07dJaabMIMsQygW+NI7GneULpA8="; + }; + + "nghttp2-1.46.0.tar.bz2" = fetchurl { + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2"; + hash = "sha256-moKXjIcAcbdp8n0riBkct3/clFpRwdaFx/YafhP8Ryk="; + }; +} diff --git a/pkgs/tools/networking/curl-impersonate/update.sh b/pkgs/tools/networking/curl-impersonate/update.sh new file mode 100755 index 0000000000000..3930c0768478a --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/update.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts +set -euo pipefail + +nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs repo\nAre we running from within the nixpkgs git repo?\n' >&2; exit 1))" + +stripwhitespace() { + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +narhash() { + nix --extra-experimental-features nix-command store prefetch-file --json "$1" | jq -r .hash +} + +nixeval() { + nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1" | jq -r . +} + +vendorhash() { + (nix --extra-experimental-features nix-command build --no-link -f "$nixpkgs" --no-link "$1" 2>&1 >/dev/null | tail -n3 | grep -F got: | cut -d: -f2- | stripwhitespace) 2>/dev/null || true +} + +findpath() { + path="$(nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1.meta.position" | jq -r . | cut -d: -f1)" + outpath="$(nix --extra-experimental-features nix-command eval --json --impure --expr "builtins.fetchGit \"$nixpkgs\"")" + + if [ -n "$outpath" ]; then + path="${path/$(echo "$outpath" | jq -r .)/$nixpkgs}" + fi + + echo "$path" +} + +getvar() { + echo "$2" | grep -F "$1" | sed -e 's/:=/:/g' | cut -d: -f2- | stripwhitespace +} + +attr="${UPDATE_NIX_ATTR_PATH:-curl-impersonate}" +version="$(curl -sSL "https://api.github.com/repos/lwthiker/curl-impersonate/releases/latest" | jq -r .tag_name | sed -e 's/^v//')" + +pkgpath="$(findpath "$attr")" + +updated="$(cd "$nixpkgs" && update-source-version "$attr" "$version" --file="$pkgpath" --print-changes | jq -r length)" + +if [ "$updated" -eq 0 ]; then + echo 'update.sh: Package version not updated, nothing to do.' + exit 0 +fi + +vars="$(curl -sSL "https://github.com/lwthiker/curl-impersonate/raw/v$version/Makefile.in" | grep '^ *[^ ]*_\(VERSION\|URL\|COMMIT\) *:=')" + +cat >"$(dirname "$pkgpath")"/deps.nix <