Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

python311: enable windows cross-compilation, make win-dll-hook link to distant dependancies #227818

Merged
merged 1 commit into from
Sep 8, 2023
Merged

python311: enable windows cross-compilation, make win-dll-hook link to distant dependancies #227818

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 46 additions & 13 deletions pkgs/development/interpreters/python/cpython/default.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ lib, stdenv, fetchurl, fetchpatch
{ lib, stdenv, fetchurl, fetchpatch, fetchgit
, bzip2
, expat
, libffi
Expand All @@ -18,6 +18,7 @@
, self
, configd
, darwin
, windows
, autoreconfHook
, autoconf-archive
, pkg-config
Expand All @@ -44,6 +45,9 @@
, static ? stdenv.hostPlatform.isStatic
, enableFramework ? false
, enableOptimizations ? false
# these dont build for windows
, withGdbm ? !stdenv.hostPlatform.isWindows
, withReadline ? !stdenv.hostPlatform.isWindows
Comment on lines +48 to +50
Copy link
Member

@Artturin Artturin Sep 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fedora has windows patches for these too

https://src.fedoraproject.org/rpms/mingw-gdbm/tree/rawhide
https://src.fedoraproject.org/rpms/mingw-readline/tree/rawhide

Copy link
Member

@Artturin Artturin Sep 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for readline

diff --git a/pkgs/development/libraries/readline/8.2.nix b/pkgs/development/libraries/readline/8.2.nix
index 1c53da3cdfa4..eca19897e73e 100644
--- a/pkgs/development/libraries/readline/8.2.nix
+++ b/pkgs/development/libraries/readline/8.2.nix
@@ -1,4 +1,4 @@
-{ fetchurl, stdenv, lib, ncurses
+{ fetchurl, stdenv, lib, ncurses, fetchgit
 }:
 
 stdenv.mkDerivation rec {
@@ -33,6 +33,23 @@ stdenv.mkDerivation rec {
     ]
     ++ upstreamPatches;
 
+  postPatch = if (stdenv.hostPlatform.isMinGW) then (
+    let
+      # https://src.fedoraproject.org/rpms/mingw-gdbm
+      readline-patch = fetchgit {
+        name = "mingw-readline-patches";
+        url = "https://src.fedoraproject.org/rpms/mingw-readline.git";
+        rev = "c15e77591106244a9f1c318226167d5bcea7bf04"; # patches for readline 8.2
+        sha256 = "sha256-QcMJ/nq2T4QlLkw0tAutMohAB0zWa1mRUYuN7D/Kvh0=";
+      };
+    in
+    ''
+      for p in ${readline-patch}/*.patch; do
+        patch -p1 -d . -i $p
+      done
+    ''
+  ) else null;
+
   meta = with lib; {
     description = "Library for interactive line editing";
 
@@ -57,7 +74,7 @@ stdenv.mkDerivation rec {
 
     maintainers = with maintainers; [ dtzWill ];
 
-    platforms = platforms.unix;
+    platforms = platforms.unix ++ platforms.windows;
     branch = "8.2";
   };
 }
readline-x86_64-w64-mingw32> /nix/store/8gfd58vf0arh22p7bc8da0qidrydfi4b-x86_64-w64-mingw32-binutils-2.40/bin/x86_64-w64-mingw32-ld: cannot
find -lncurses: No such file or directory
readline-x86_64-w64-mingw32> /nix/store/8gfd58vf0arh22p7bc8da0qidrydfi4b-x86_64-w64-mingw32-binutils-2.40/bin/x86_64-w64-mingw32-ld: cannot
find -ltermcap: No such file or directory

fedora has mingw-termcap https://src.fedoraproject.org/rpms/mingw-termcap/blob/rawhide/f/mingw-termcap.spec

which has a comment

# Note: Termcap was deprecated and removed from Fedora after F-8.  It
	
# has been replaced by ncurses.  However ncurses cannot be compiled on
	
# Windows so we have to supply termcap.

but pkgsCross.mingwW64.ncurses builds for us and is already in propagatedBuildInputs but does not work.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the gdbm patch is for 1.11 and doesn't apply to our 1.23

# enableNoSemanticInterposition is a subset of the enableOptimizations flag that doesn't harm reproducibility.
# clang starts supporting `-fno-sematic-interposition` with version 10
, enableNoSemanticInterposition ? (!stdenv.cc.isClang || (stdenv.cc.isClang && lib.versionAtLeast stdenv.cc.version "10"))
Expand Down Expand Up @@ -128,12 +132,17 @@ let
];

buildInputs = filter (p: p != null) ([
zlib bzip2 expat xz libffi libxcrypt gdbm sqlite readline ncurses openssl' ]
zlib bzip2 expat xz libffi libxcrypt ]
++ optional withGdbm gdbm
++ [ sqlite ]
++ optional withReadline readline
++ [ ncurses openssl' ]
++ optionals x11Support [ tcl tk libX11 xorgproto ]
++ optionals (bluezSupport && stdenv.isLinux) [ bluez ]
++ optionals stdenv.isDarwin [ configd ])

++ optionals enableFramework [ Cocoa ]
++ optionals stdenv.hostPlatform.isMinGW [ windows.mingw_w64_pthreads windows.dlfcn ]
++ optionals tzdataSupport [ tzdata ]; # `zoneinfo` module

hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
Expand All @@ -160,6 +169,8 @@ let
# are not documented, and must be derived from the configure script (see links
# below).
sysconfigdataHook = with stdenv.hostPlatform; with passthru; let
machdep = if isWindows then "win32" else parsed.kernel.name; # win32 is added by Fedora’s patch

# https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L428
# The configure script uses "arm" as the CPU name for all 32-bit ARM
# variants when cross-compiling, but native builds include the version
Expand All @@ -176,7 +187,7 @@ let
powerpc64 = "ppc64";
powerpc64le = "ppc64le";
}.${parsed.cpu.name} or parsed.cpu.name;
in "${parsed.kernel.name}-${cpu}";
in "${machdep}-${cpu}";

# https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L724
multiarchCpu =
Expand Down Expand Up @@ -205,12 +216,13 @@ let
else "gnu";
multiarch =
if isDarwin then "darwin"
else "${multiarchCpu}-${parsed.kernel.name}-${pythonAbiName}";
else if isWindows then ""
else "${multiarchCpu}-${machdep}-${pythonAbiName}";

abiFlags = optionalString isPy37 "m";

# https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L78
pythonSysconfigdataName = "_sysconfigdata_${abiFlags}_${parsed.kernel.name}_${multiarch}";
pythonSysconfigdataName = "_sysconfigdata_${abiFlags}_${machdep}_${multiarch}";
in ''
sysconfigdataHook() {
if [ "$1" = '${placeholder "out"}' ]; then
Expand All @@ -222,12 +234,13 @@ let
addEnvHooks "$hostOffset" sysconfigdataHook
'';

execSuffix = stdenv.hostPlatform.extensions.executable;
in with passthru; stdenv.mkDerivation {
pname = "python3";
inherit src version;

inherit nativeBuildInputs;
buildInputs = [ bash ] ++ buildInputs; # bash is only for patchShebangs
buildInputs = lib.optionals (!stdenv.hostPlatform.isWindows) [ bash ] ++ buildInputs; # bash is only used for patchShebangs


prePatch = optionalString stdenv.isDarwin ''
Expand Down Expand Up @@ -295,9 +308,19 @@ in with passthru; stdenv.mkDerivation {
] ++ optionals stdenv.hostPlatform.isLoongArch64 [
# https://github.com/python/cpython/issues/90656
./loongarch-support.patch
];
] ++ optionals (stdenv.hostPlatform.isMinGW) (let
# https://src.fedoraproject.org/rpms/mingw-python3
mingw-patch = fetchgit {
name = "mingw-python-patches";
url = "https://src.fedoraproject.org/rpms/mingw-python3.git";
rev = "45c45833ab9e5480ad0ae00778a05ebf35812ed4"; # for python 3.11.5 at the time of writing.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I updated the rev to python 3.11.5 even though the patch files haven't changed.

sha256 = "sha256-KIyNvO6MlYTrmSy9V/DbzXm5OsIuyT/BEpuo7Umm9DI=";
};
in [
"${mingw-patch}/*.patch"
]);

postPatch = ''
postPatch = optionalString (!stdenv.hostPlatform.isWindows) ''
substituteInPlace Lib/subprocess.py \
--replace "'/bin/sh'" "'${bash}/bin/sh'"
'' + optionalString mimetypesSupport ''
Expand Down Expand Up @@ -360,8 +383,9 @@ in with passthru; stdenv.mkDerivation {
"ac_cv_have_long_long_format=yes"
"ac_cv_have_size_t_format=yes"
"ac_cv_computed_gotos=yes"
"ac_cv_file__dev_ptmx=yes"
"ac_cv_file__dev_ptc=yes"
# Both fail when building for windows, normally configure checks this by itself but on other platforms this is set to yes always.
"ac_cv_file__dev_ptmx=${if stdenv.hostPlatform.isWindows then "no" else "yes"}"
"ac_cv_file__dev_ptc=${if stdenv.hostPlatform.isWindows then "no" else "yes"}"
Comment on lines +386 to +388
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed the in-tree patch and did this instead.

] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform && pythonAtLeast "3.11") [
"--with-build-python=${pythonForBuildInterpreter}"
] ++ optionals stdenv.hostPlatform.isLinux [
Expand All @@ -370,7 +394,8 @@ in with passthru; stdenv.mkDerivation {
"ac_cv_func_lchmod=no"
] ++ optionals tzdataSupport [
"--with-tzpath=${tzdata}/share/zoneinfo"
] ++ optional static "LDFLAGS=-static";
] ++ optional static "LDFLAGS=-static"
++ optional (execSuffix != "") "--with-suffix=${execSuffix}";

preConfigure = optionalString (pythonOlder "3.12") ''
for i in /usr /sw /opt /pkg; do # improve purity
Expand Down Expand Up @@ -438,7 +463,7 @@ in with passthru; stdenv.mkDerivation {
# Use Python3 as default python
ln -s "$out/bin/idle3" "$out/bin/idle"
ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
ln -s "$out/bin/python3" "$out/bin/python"
ln -s "$out/bin/python3${execSuffix}" "$out/bin/python${execSuffix}"
ln -s "$out/bin/python3-config" "$out/bin/python-config"
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
ln -sL "$out/share/man/man1/python3.1.gz" "$out/share/man/man1/python.1.gz"
Expand Down Expand Up @@ -504,6 +529,14 @@ in with passthru; stdenv.mkDerivation {
To use Python with Nix and nixpkgs, have a look at the online documentation:
<https://nixos.org/manual/nixpkgs/stable/#python>.
EXTERNALLY_MANAGED
'' + optionalString stdenv.hostPlatform.isWindows ''
# Shebang files that link against the build python. Shebang don’t work on windows
rm $out/bin/2to3*
rm $out/bin/idle*
rm $out/bin/pydoc*

echo linking DLLs for python’s compiled librairies
linkDLLsInfolder $out/lib/python*/lib-dynload/
'';

preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
Expand Down Expand Up @@ -573,7 +606,7 @@ in with passthru; stdenv.mkDerivation {
high level dynamic data types.
'';
license = licenses.psfl;
platforms = platforms.linux ++ platforms.darwin;
platforms = platforms.linux ++ platforms.darwin ++ platforms.windows;
maintainers = with maintainers; [ fridh ];
mainProgram = executable;
};
Expand Down