Merge pull request openslide#230 from bgilbert/zstd

Allow enabling extra deps when overriding OpenSlide; add zstd to dev builds

.github/ISSUE_TEMPLATE/release.md

@@ -1,5 +1,6 @@
 # openslide-bin release process
 
+- [ ] If there's a new OpenSlide release, remove `dev_deps` condition from any subprojects used by the new release
 - [ ] Run [workflow](https://github.com/openslide/openslide-bin/actions/workflows/update-check.yml) to check for updates
 - [ ] Merge any resulting PR; perform any needed manual updates reported by the workflow
 - [ ] Submit PR to update `CHANGELOG.md` and `_PROJECT_VERSION`

.github/workflows/build.yml

@@ -144,13 +144,13 @@ jobs:
         run: |
           import json, os
           sys_map = {
-            '${{ inputs.linux_builder_repo_and_digest }}': 'linux',
-            '${{ inputs.macos_enable && 'true' || '' }}': 'macos',
-            '${{ inputs.windows_builder_repo_and_digest }}': 'windows',
+              '${{ inputs.linux_builder_repo_and_digest }}': 'linux',
+              '${{ inputs.macos_enable && 'true' || '' }}': 'macos',
+              '${{ inputs.windows_builder_repo_and_digest }}': 'windows',
           }
           systems = json.dumps(sorted(v for k, v in sys_map.items() if k))
           with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
-            fh.write(f'systems={systems}\n')
+              fh.write(f'systems={systems}\n')
 
   bdist:
     name: Binaries

bintool

@@ -169,7 +169,7 @@ class BuildParams:
         overrides directory.'''
         assert self.locked
         for proj in Project.get_all():
-            override = self.root / 'override' / proj.id
+            override = proj.override_path
             dir = self.root / 'subprojects' / proj.id
             wrap = proj.wrap_path
             overridden = wrap.with_suffix('.wrap.overridden')
@@ -287,12 +287,20 @@ class MesonPlatform(Platform):
             # we wipe
             args.append('--wipe')
 
-        openslide_src = Project.get('openslide').source_dir
+        openslide = Project.get('openslide')
+        # we can't check for the existence of the wrap file; sdist needs
+        # dev_deps but doesn't activate overrides
+        dev_deps = openslide.override_path.is_dir()
+        args.append(f'-Ddev_deps={str(dev_deps).lower()}')
+        if dev_deps:
+            # OpenSlide is overridden; enable deps used by its Git main
+            log('Enabling development dependencies...')
+
         version_suffix = (
             subprocess.check_output(
-                ['git', 'rev-parse', 'HEAD'], cwd=openslide_src
+                ['git', 'rev-parse', 'HEAD'], cwd=openslide.source_dir
             ).decode()[:7]
-            if (openslide_src / '.git').exists()
+            if (openslide.source_dir / '.git').exists()
             else ''
         )
         args.append(f'-Dopenslide:version_suffix={version_suffix}')
@@ -322,7 +330,7 @@ class MesonPlatform(Platform):
             ['meson', 'subprojects', 'purge', '--confirm'],
             cwd=self.params.root,
         )
-        dir = self._setup('sdist', ['-Dall_subprojects=true'])
+        dir = self._setup('sdist', ['-Dall_systems=true'])
         # avoid spurious complaints about a dirty work tree
         subprocess.check_call(
             ['git', 'update-index', '-q', '--refresh'], cwd=self.params.root

common/software.py

@@ -112,6 +112,10 @@ def wrap(self) -> configparser.RawConfigParser:
     def wrap_path(self) -> Path:
         return meson_source_root() / 'subprojects' / f'{self.id}.wrap'
 
+    @property
+    def override_path(self) -> Path:
+        return meson_source_root() / 'override' / self.id
+
     @cached_property
     def version(self) -> str:
         try:
@@ -321,6 +325,14 @@ def _sqlite3_license(proj: Project) -> tuple[str, str]:
         update_url='https://zlib.net/',
         update_regex=re.compile('source code, version ([0-9.]+)'),
     ),
+    Project(
+        id='zstd',
+        display='Zstandard',
+        # Dual-licensed BSD or GPLv2.  Elect BSD.
+        licenses=['LICENSE'],
+        update_url='https://github.com/facebook/zstd/tags',
+        update_regex=re.compile('archive/refs/tags/v([0-9.]+)\\.tar'),
+    ),
 )
 
 # gvdb is a copylib bundled with glib, without a stable API

deps/meson.build

@@ -1,13 +1,15 @@
-# should we enable all subprojects?  this is needed when building source
-# distributions so we don't leave anything out
-force_subprojects = get_option('all_subprojects')
+# should we enable all subprojects that are needed by any OS?  this is
+# needed when building source distributions so we don't leave anything out
+all_systems = get_option('all_systems')
+# should we enable subprojects needed by OpenSlide Git main?
+dev_deps = get_option('dev_deps')
 
 add_global_arguments(
   '-I' + meson.current_source_dir(),
   language: ['c', 'cpp']
 )
 
-if system == 'darwin' and not force_subprojects
+if not all_systems and system == 'darwin'
   cc = meson.get_compiler('c')
   # [dependency name, library name, version] or
   # [dependency name, library name, header with version, version define]
@@ -57,10 +59,19 @@ else
   subproject('sqlite3')
 endif
 
-if system != 'linux' or force_subprojects
+if all_systems or system != 'linux'
   subproject('proxy-libintl')
 endif
 
+if dev_deps
+  subproject(
+    'zstd',
+    default_options : [
+      'bin_programs=false',
+    ],
+  )
+endif
+
 subproject('libpng')
 subproject(
   'libjpeg-turbo',

meson.options

@@ -4,8 +4,14 @@
 # -Dopenslide-java:werror=true   - fail OpenSlide Java build on warnings
 
 option(
-  'all_subprojects',
+  'all_systems',
   type : 'boolean',
   value : false,
-  description : 'Enable all subprojects (for building the source tarball)',
+  description : 'Enable subprojects for all OSes (for building source tarball)',
+)
+option(
+  'dev_deps',
+  type : 'boolean',
+  value : false,
+  description : 'Enable subprojects for OpenSlide Git main',
 )

subprojects/packagefiles/zstd-pthread.patch

@@ -0,0 +1,48 @@
+From d4ec0e6b84cc5e81753921c15285603dc0aca82c Mon Sep 17 00:00:00 2001
+From: Benjamin Gilbert <bgilbert@backtick.net>
+Date: Thu, 7 Mar 2024 21:40:27 +0900
+Subject: [PATCH] meson: don't link with -pthread on Windows
+
+Meson always returns -pthread in dependency('threads') on non-MSVC
+compilers.  Fix a link error when building on MinGW without winpthreads.
+---
+ meson.build       | 9 +++++++--
+ meson_options.txt | 2 +-
+ 2 files changed, 8 insertions(+), 3 deletions(-)
+
+diff --git a/meson.build b/meson.build
+index 15c56c2b13aa..7ddca2e79236 100644
+--- a/meson.build
++++ b/meson.build
+@@ -88,8 +88,13 @@ feature_lz4 = get_option('lz4')
+ # =============================================================================
+
+ libm_dep = cc.find_library('m', required: false)
+-thread_dep = dependency('threads', required: feature_multi_thread)
+-use_multi_thread = thread_dep.found()
++if host_machine_os == os_windows
++  thread_dep = dependency('', required: false)
++  use_multi_thread = not feature_multi_thread.disabled()
++else
++  thread_dep = dependency('threads', required: feature_multi_thread)
++  use_multi_thread = thread_dep.found()
++endif
+ # Arguments in dependency should be equivalent to those passed to pkg-config
+ zlib_dep = dependency('zlib', required: feature_zlib)
+ use_zlib = zlib_dep.found()
+diff --git a/meson_options.txt b/meson_options.txt
+index f35cd5fc8b11..4705178279f8 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -27,7 +27,7 @@ option('bin_contrib', type: 'boolean', value: false,
+   description: 'Enable contrib build')
+
+ option('multi_thread', type: 'feature', value: 'enabled',
+-  description: 'Enable multi-threading when pthread is detected')
++  description: 'Enable multi-threading when pthread or Windows is detected')
+ option('zlib', type: 'feature', value: 'auto',
+   description: 'Enable zlib support')
+ option('lzma', type: 'feature', value: 'auto',
+-- 
+2.44.0
+

subprojects/zstd.wrap

@@ -0,0 +1,16 @@
+[wrap-file]
+directory = zstd-1.5.5
+source_url = https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz
+source_filename = zstd-1.5.5.tar.gz
+source_hash = 9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4
+patch_filename = zstd_1.5.5-1_patch.zip
+patch_url = https://wrapdb.mesonbuild.com/v2/zstd_1.5.5-1/get_patch
+patch_hash = 0a076f6e60c4288193c2d9604670f2d8b5cb05511c2ac43eb521cb7363665278
+source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/zstd_1.5.5-1/zstd-1.5.5.tar.gz
+wrapdb_version = 1.5.5-1
+
+# https://github.com/facebook/zstd/pull/3931
+diff_files = zstd-pthread.patch
+
+[provide]
+libzstd = libzstd_dep