From 184834d3c9d2d26d460c61af9fb6cb1725da40d3 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:27:19 +0300 Subject: [PATCH 1/9] Promote v24.5-testing version (#435) --- CMakeLists.txt | 2 +- osx/NZBGet-Info.plist | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ebe8ca94..8a8f31c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ if (PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR) message(FATAL_ERROR "In-source builds are not allowed. You should create separate directory for build files.") endif() -set(VERSION "24.4") +set(VERSION "24.5") set(PACKAGE "nzbget") set(LIBS "") set(INCLUDES ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) diff --git a/osx/NZBGet-Info.plist b/osx/NZBGet-Info.plist index 303174cd..41c6f6ce 100644 --- a/osx/NZBGet-Info.plist +++ b/osx/NZBGet-Info.plist @@ -17,7 +17,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 24.4 + 24.5 CFBundleDocumentTypes From 00e50a5abcf3914bb346874c079c9b24868f4344 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:41:31 +0300 Subject: [PATCH 2/9] Fix: homebrew CI build, macOS ARM64 with Xcode 16, Boost.JSON build via CMake (#444) - fixed CI homebrew build - fixed Boost.JSON build via CMake - fixed macOS ARM64 with Xcode 16 --- .github/workflows/android.yml | 8 +++ .github/workflows/freebsd.yml | 12 +++- .github/workflows/linux.yml | 8 +++ .github/workflows/osx.yml | 16 +++++ CMakeLists.txt | 2 - cmake/common.cmake | 4 +- cmake/par2-turbo.cmake | 61 +++++++++++++++---- cmake/posix.cmake | 11 +++- cmake/windows.cmake | 8 ++- daemon/main/nzbget.h | 3 + lib/sources.cmake | 10 +-- lib/yencode/SimdDecoder.cpp | 1 + linux/build-nzbget.sh | 3 +- osx/build-nzbget.sh | 1 + tests/extension/CMakeLists.txt | 2 +- tests/postprocess/CMakeLists.txt | 2 +- tests/system/CMakeLists.txt | 3 +- .../util/{Benchmark.cpp => BenchmarkTest.cpp} | 0 tests/util/CMakeLists.txt | 2 +- 19 files changed, 123 insertions(+), 34 deletions(-) rename tests/util/{Benchmark.cpp => BenchmarkTest.cpp} (100%) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 33525931..0738b161 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -28,6 +28,14 @@ jobs: fi bash linux/build-nzbget.sh $BUILD_PARAMS + - name: Upload full build log on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-android-build-log + path: build/*/build.log + retention-days: 5 + - name: Rename build artifacts if: github.ref_name != 'main' && github.ref_name != 'develop' run: | diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index c86de611..ffdb776a 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -1,4 +1,4 @@ -name: freebsd build +name: freebsd build on: workflow_call: @@ -9,7 +9,7 @@ jobs: runs-on: [self-hosted, nzbget-freebsd] steps: - + - name: Checkout uses: actions/checkout@v4 with: @@ -28,6 +28,14 @@ jobs: fi bash linux/build-nzbget.sh $BUILD_PARAMS + - name: Upload full build log on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-freebsd-build-log + path: build/*/build.log + retention-days: 5 + - name: Rename build artifacts if: github.ref_name != 'main' && github.ref_name != 'develop' run: | diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 40e48714..2a1b1d50 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,6 +28,14 @@ jobs: fi bash linux/build-nzbget.sh $BUILD_PARAMS + - name: Upload full build log on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-linux-build-log + path: build/*/build.log + retention-days: 5 + - name: Rename build artifacts if: github.ref_name != 'main' && github.ref_name != 'develop' run: | diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index d24d8d9c..b32990c6 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -23,6 +23,14 @@ jobs: bash osx/build-nzbget.sh x64 fi + - name: Upload full build log on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-osx-x64-build-log + path: build/*/build.log + retention-days: 5 + - name: Rename build artifacts if: github.ref_name != 'main' && github.ref_name != 'develop' run: | @@ -59,6 +67,14 @@ jobs: bash osx/build-nzbget.sh universal fi + - name: Upload full build log on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-osx-universal-build-log + path: build/*/build.log + retention-days: 5 + - name: Rename build artifacts if: github.ref_name != 'main' && github.ref_name != 'develop' run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a8f31c8..b4620bb4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,8 +80,6 @@ else() endif() endif() -include(lib/sources.cmake) - configure_file( ${CMAKE_SOURCE_DIR}/cmake/config.h.in ${CMAKE_BINARY_DIR}/config.h diff --git a/cmake/common.cmake b/cmake/common.cmake index 88fec581..8fe927ca 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1,6 +1,6 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") - add_compile_options(-Weverything) + add_compile_options(-Weverything -Wno-c++98-compat) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Wall -Wextra) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") @@ -21,4 +21,4 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "Release") endif() endif() -include(FetchContent) +include(ExternalProject) diff --git a/cmake/par2-turbo.cmake b/cmake/par2-turbo.cmake index 8b92c122..85a0645e 100644 --- a/cmake/par2-turbo.cmake +++ b/cmake/par2-turbo.cmake @@ -1,17 +1,54 @@ -set(FETCHCONTENT_QUIET FALSE) -FetchContent_Declare( +set(PAR2_ROOT ${CMAKE_BINARY_DIR}/par2-turbo/src) + +add_compile_definitions(HAVE_CONFIG_H PARPAR_ENABLE_HASHER_MD5CRC) + +set(CMAKE_ARGS + -DBUILD_TOOL=OFF + -DBUILD_LIB=ON + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR} + -DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME} +) + +if(APPLE) + set(CMAKE_ARGS ${CMAKE_ARGS} + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} + ) +endif() + +if(CMAKE_SYSROOT) + set(CMAKE_ARGS ${CMAKE_ARGS} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + -DCMAKE_SYSROOT=${CMAKE_SYSROOT} + -DCMAKE_CXX_FLAGS=-I${CMAKE_SYSROOT}/usr/include/c++/v1 + ) +endif() + +ExternalProject_add( par2-turbo - GIT_REPOSITORY https://github.com/nzbgetcom/par2cmdline-turbo.git - GIT_TAG v1.1.1-nzbget - TLS_VERIFY TRUE - GIT_SHALLOW TRUE + PREFIX par2-turbo + GIT_REPOSITORY https://github.com/nzbgetcom/par2cmdline-turbo.git + GIT_TAG v1.1.1-nzbget + TLS_VERIFY TRUE + GIT_SHALLOW TRUE GIT_PROGRESS TRUE + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + CMAKE_ARGS ${CMAKE_ARGS} + INSTALL_COMMAND "" ) -add_compile_definitions(HAVE_CONFIG_H PARPAR_ENABLE_HASHER_MD5CRC) -set(BUILD_TOOL OFF CACHE BOOL "") -set(BUILD_LIB ON CACHE BOOL "") -FetchContent_MakeAvailable(par2-turbo) +if(WIN32) + set(LIBS ${LIBS} + ${PAR2_ROOT}/par2-turbo-build/${CMAKE_BUILD_TYPE}/par2-turbo.lib + ${PAR2_ROOT}/par2-turbo-build/${CMAKE_BUILD_TYPE}/gf16.lib + ${PAR2_ROOT}/par2-turbo-build/${CMAKE_BUILD_TYPE}/hasher.lib + ) +else() + set(LIBS ${LIBS} + ${PAR2_ROOT}/par2-turbo-build/libpar2-turbo.a + ${PAR2_ROOT}/par2-turbo-build/libgf16.a + ${PAR2_ROOT}/par2-turbo-build/libhasher.a + ) +endif() -set(LIBS ${LIBS} par2-turbo gf16 hasher) -set(INCLUDES ${INCLUDES} ${par2_SOURCE_DIR}/include) +set(INCLUDES ${INCLUDES} ${PAR2_ROOT}/par2-turbo/include) diff --git a/cmake/posix.cmake b/cmake/posix.cmake index bc69e70c..dca494eb 100644 --- a/cmake/posix.cmake +++ b/cmake/posix.cmake @@ -93,7 +93,9 @@ else() if(NOT DISABLE_CURSES) set(CURSES_NEED_NCURSES TRUE) - set(CURSES_NEED_WIDE TRUE) + if(NOT APPLE) + set(CURSES_NEED_WIDE TRUE) + endif() find_package(Curses REQUIRED) set(INCLUDES ${INCLUDES} ${CURSES_INCLUDE_DIRS}) set(LIBS ${LIBS} ${CURSES_LIBRARIES}) @@ -113,16 +115,19 @@ else() include(${CMAKE_SOURCE_DIR}/cmake/boost.cmake) add_dependencies(${PACKAGE} boost) - add_dependencies(yencode boost) - add_dependencies(regex boost) else() set(LIBS ${LIBS} Boost::json) set(INCLUDES ${INCLUDES} ${Boost_INCLUDE_DIR}) endif() endif() +include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) + if(NOT DISABLE_PARCHECK) include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) + add_dependencies(${PACKAGE} par2-turbo) + add_dependencies(yencode par2-turbo) + add_dependencies(regex par2-turbo) endif() include(CheckIncludeFiles) diff --git a/cmake/windows.cmake b/cmake/windows.cmake index d833cd01..c6f78622 100644 --- a/cmake/windows.cmake +++ b/cmake/windows.cmake @@ -16,8 +16,6 @@ find_package(Threads REQUIRED) find_package(LibXml2 REQUIRED) find_package(Boost REQUIRED COMPONENTS json) -include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) - set(LIBS ${LIBS} Threads::Threads Boost::json LibXml2::LibXml2 winmm.lib) set(INCLUDES ${INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR}) @@ -37,6 +35,12 @@ set(INCLUDES ${INCLUDES} ${CMAKE_SOURCE_DIR}/windows/resources ) +include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) + +include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) +add_dependencies(yencode par2-turbo) +add_dependencies(regex par2-turbo) + set(FUNCTION_MACRO_NAME __FUNCTION__) set(HAVE_CTIME_R_3 1) set(HAVE_VARIADIC_MACROS 1) diff --git a/daemon/main/nzbget.h b/daemon/main/nzbget.h index 3c674004..4e0a2c3f 100644 --- a/daemon/main/nzbget.h +++ b/daemon/main/nzbget.h @@ -183,6 +183,9 @@ #include #include +#if !defined(DISABLE_TLS) && defined(HAVE_OPENSSL) +#include +#endif // NOTE: do not include in "nzbget.h". contains objects requiring // intialization, causing every unit in nzbget to have initialization routine. This in particular diff --git a/lib/sources.cmake b/lib/sources.cmake index 1a660e2c..3715757d 100644 --- a/lib/sources.cmake +++ b/lib/sources.cmake @@ -2,11 +2,11 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "i?86|x86_64") set(SSE2_CXXFLAGS "-msse2") set(SSSE3_CXXFLAGS "-mssse3") set(PCLMUL_CXXFLAGS "-msse4.1 -mpclmul") -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm.*") - set(NEON_CXXFLAGS "-mfpu=neon") - set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") -elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm.*") + set(NEON_CXXFLAGS "-mfpu=neon") + set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") endif() add_library(regex STATIC diff --git a/lib/yencode/SimdDecoder.cpp b/lib/yencode/SimdDecoder.cpp index 5cc028f1..04625b49 100644 --- a/lib/yencode/SimdDecoder.cpp +++ b/lib/yencode/SimdDecoder.cpp @@ -19,6 +19,7 @@ * along with this program. If not, see . */ +#include "nzbget.h" #ifdef SIMD_DECODER diff --git a/linux/build-nzbget.sh b/linux/build-nzbget.sh index af8ee77a..b5346862 100755 --- a/linux/build-nzbget.sh +++ b/linux/build-nzbget.sh @@ -613,7 +613,7 @@ build_bin() export LDFLAGS="" export NZBGET_INCLUDES="$TOOLCHAIN_PATH/$ARCH/output/staging/usr/include/;" - if [ $PLATFORM == "freebsd" ]; then + if [ "$PLATFORM" == "freebsd" ]; then export CXXFLAGS="-Os --sysroot=$FREEBSD_SYSROOT -I$FREEBSD_SYSROOT/usr/include/c++/v1" export CFLAGS=$CXXFLAGS export CPPFLAGS=$CXXFLAGS @@ -649,6 +649,7 @@ build_bin() ;; esac + unset CFLAGS unset CXXFLAGS unset CPPFLAGS unset LDFLAGS diff --git a/osx/build-nzbget.sh b/osx/build-nzbget.sh index c7614613..1d8de797 100755 --- a/osx/build-nzbget.sh +++ b/osx/build-nzbget.sh @@ -100,6 +100,7 @@ for ARCH in $ARCHS; do cmake ../.. \ -DENABLE_STATIC=ON \ -DCMAKE_INSTALL_PREFIX="$PWD/../../$NZBGET_PATH/$DAEMON_PATH" \ + -DCMAKE_SYSTEM_NAME="Darwin" \ -DVERSION_SUFFIX="$VERSION_SUFFIX" \ -DCMAKE_SYSTEM_PROCESSOR=$CMAKE_ARCH \ -DCMAKE_OSX_ARCHITECTURES=$CMAKE_ARCH diff --git a/tests/extension/CMakeLists.txt b/tests/extension/CMakeLists.txt index 5c78f0db..91d47cc8 100644 --- a/tests/extension/CMakeLists.txt +++ b/tests/extension/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB ExtensionSrc +set(ExtensionSrc main.cpp ManifestFileTest.cpp ExtensionLoaderTest.cpp diff --git a/tests/postprocess/CMakeLists.txt b/tests/postprocess/CMakeLists.txt index c2f599d7..0f5dabb0 100644 --- a/tests/postprocess/CMakeLists.txt +++ b/tests/postprocess/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB PostprocessTestsSrc +set(PostprocessTestsSrc main.cpp # DirectUnpackTest.cpp # DupeMatcherTest.cpp diff --git a/tests/system/CMakeLists.txt b/tests/system/CMakeLists.txt index c86fd961..c55eaec9 100644 --- a/tests/system/CMakeLists.txt +++ b/tests/system/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB SystemTestsSrc +set(SystemTestsSrc main.cpp SystemInfoTest.cpp ${CMAKE_SOURCE_DIR}/daemon/system/CPU.cpp @@ -7,7 +7,6 @@ file(GLOB SystemTestsSrc ${CMAKE_SOURCE_DIR}/daemon/system/SystemInfo.cpp ${CMAKE_SOURCE_DIR}/daemon/connect/HttpClient.cpp ${CMAKE_SOURCE_DIR}/daemon/main/Options.cpp - ${CMAKE_SOURCE_DIR}/daemon/remote/MessageBase.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DownloadInfo.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp ${CMAKE_SOURCE_DIR}/daemon/feed/FeedInfo.cpp diff --git a/tests/util/Benchmark.cpp b/tests/util/BenchmarkTest.cpp similarity index 100% rename from tests/util/Benchmark.cpp rename to tests/util/BenchmarkTest.cpp diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index b458498a..52253ea2 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -1,4 +1,4 @@ -file(GLOB UtilTestSrc +set(UtilTestSrc main.cpp FileSystemTest.cpp UtilTest.cpp From 80bbc2d6dbc4f28da87df38cbcf94d9f54988e55 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 28 Nov 2024 15:02:19 +0300 Subject: [PATCH 3/9] security.md (#447) --- SECURITY.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..2ddd74d0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security Policy + +If you believe you have found a security vulnerability in NZBGet, please report it to us as described below. + +## Reporting Security Issues + +Please do not report security vulnerabilities through public GitHub issues. Instead, please use GitHubs private vulnerability reporting functionality associated to this repository. + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: +1. Type of issue +2. Step-by-step instructions to reproduce the issue +3. Proof-of-concept or exploit code (if possible) +4. Potential impact of the issue, including how an attacker might exploit the issue + +This information will help us review your report faster. + +This security policy only applies to the most recent stable branch of NZBGet. Flaws in old versions that are not present in the current stable branch will not be fixed. From 975e58cd5bdd1c0c0a6f3002128cfcfe4295d731 Mon Sep 17 00:00:00 2001 From: phnzb <139914799+phnzb@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:10:59 +0300 Subject: [PATCH 4/9] Add: more tests, docker build tweaks (#445) --- .github/workflows/docker.yml | 3 + .github/workflows/tests.yml | 110 ++++++++++++++++++++++++++++ .github/workflows/windows-tests.yml | 47 ------------ cmake/posix.cmake | 1 - 4 files changed, 113 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/tests.yml delete mode 100644 .github/workflows/windows-tests.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 37275733..83e94b40 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,9 @@ on: push: branches: - develop + - main + tags: + - "v*" workflow_dispatch: env: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..9aa31942 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,110 @@ +name: tests + +on: + push: + branches: + - develop + - main + pull_request: + branches: + - develop + - main + workflow_call: + workflow_dispatch: + +jobs: + + windows-tests: + runs-on: windows-2022 + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Download and extract vcpkg cache + run: | + $ProgressPreference = "SilentlyContinue" + Invoke-WebRequest https://github.com/nzbgetcom/build-files/releases/download/v1.0/vcpkg-windows-tests.zip -OutFile "${{ github.workspace }}\vcpkg.zip" + Expand-Archive -Path "${{ github.workspace }}\vcpkg.zip" -DestinationPath "${{ github.workspace }}" + + - name: Build + run: | + New-Item build -ItemType Directory -Force | Out-Null + cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DBUILD_ONLY_TESTS=ON + cmake --build . --config Release -j 4 + + - name: Test + run: | + cd build + ctest -C Release + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-windows-test-log + path: build/Testing/Temporary/LastTest.log + retention-days: 5 + + linux-tests: + runs-on: ubuntu-24.04 + steps: + + - name: Prepare environment + run: | + sudo apt-get install -y cmake libxml2-dev libssl-dev libncurses-dev libboost-all-dev + + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + run: | + mkdir build + cd build + cmake .. -DBUILD_ONLY_TESTS=ON + cmake --build . --config Release -j 4 + + - name: Test + run: | + cd build + ctest -C Release + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-linux-test-log + path: build/Testing/Temporary/LastTest.log + retention-days: 5 + + macos-tests: + runs-on: macos-14 + steps: + + - name: Install dependencies + run: + brew install --formula boost + + - name: Checkout + uses: actions/checkout@v4 + + - name: Build + run: | + mkdir build + cd build + cmake .. -DBUILD_ONLY_TESTS=ON + cmake --build . --config Release -j 4 + + - name: Test + run: | + cd build + ctest -C Release + + - name: Upload test artifacts + uses: actions/upload-artifact@v4 + if: failure() + with: + name: nzbget-linux-test-log + path: build/Testing/Temporary/LastTest.log + retention-days: 5 diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml deleted file mode 100644 index 06847dad..00000000 --- a/.github/workflows/windows-tests.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: windows tests - -on: - push: - branches: - - develop - - main - pull_request: - branches: - - develop - - main - workflow_call: - workflow_dispatch: - -jobs: - test: - runs-on: [self-hosted, windows] - - steps: - - - name: Prepare environment - run: | - "C:\Program Files\CMake\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Checkout - uses: actions/checkout@v4 - - - name: Build - run: | - cmake --version - mkdir build - cd build - cmake .. -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DBUILD_ONLY_TESTS=ON - cmake --build . --config Release -j 2 - - - name: Test - run: | - cd build - ctest -C Release - - - name: Upload test artifacts - uses: actions/upload-artifact@v4 - if: failure() - with: - name: nzbget-windows-test-log - path: build/Testing/Temporary/LastTest.log - retention-days: 5 diff --git a/cmake/posix.cmake b/cmake/posix.cmake index dca494eb..f73d39ea 100644 --- a/cmake/posix.cmake +++ b/cmake/posix.cmake @@ -125,7 +125,6 @@ include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) if(NOT DISABLE_PARCHECK) include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) - add_dependencies(${PACKAGE} par2-turbo) add_dependencies(yencode par2-turbo) add_dependencies(regex par2-turbo) endif() From 534665f31d1181dd34e1f0e8568d075be2b7e369 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:26:26 +0300 Subject: [PATCH 5/9] Add: better long-paths support on Windows (#441) - fixed multiple bugs related to bad support of long-paths on Windows --- cmake/par2-turbo.cmake | 2 +- daemon/extension/ManifestFile.cpp | 28 +++++++++++------- daemon/extension/ManifestFile.h | 5 ++-- daemon/main/nzbget.h | 2 +- daemon/postprocess/ParChecker.cpp | 29 +++++++++++------- daemon/postprocess/ParChecker.h | 2 +- daemon/postprocess/ParRenamer.cpp | 7 ++++- daemon/postprocess/ParRenamer.h | 4 +++ daemon/system/SystemInfo.cpp | 2 +- daemon/util/FileSystem.cpp | 39 +++++++++++++++++++----- daemon/util/FileSystem.h | 4 ++- daemon/util/Json.cpp | 31 ++++++++++++-------- daemon/util/Json.h | 9 ++++-- daemon/util/Utf8.cpp | 32 ++++++++++++-------- daemon/util/Utf8.h | 4 +-- daemon/util/Util.cpp | 10 +++++++ tests/extension/CMakeLists.txt | 4 +++ tests/feed/CMakeLists.txt | 9 +++++- tests/main/CMakeLists.txt | 9 +++++- tests/nntp/CMakeLists.txt | 9 +++++- tests/postprocess/CMakeLists.txt | 5 ++++ tests/postprocess/ParRenamerTest.cpp | 3 -- tests/queue/CMakeLists.txt | 9 +++++- tests/system/CMakeLists.txt | 5 ++++ tests/util/CMakeLists.txt | 1 + tests/util/JsonTest.cpp | 44 +++++++++++++++++++--------- tests/util/Utf8Test.cpp | 25 ++++++++++++++++ 27 files changed, 245 insertions(+), 88 deletions(-) diff --git a/cmake/par2-turbo.cmake b/cmake/par2-turbo.cmake index 85a0645e..4fa76b54 100644 --- a/cmake/par2-turbo.cmake +++ b/cmake/par2-turbo.cmake @@ -28,7 +28,7 @@ ExternalProject_add( par2-turbo PREFIX par2-turbo GIT_REPOSITORY https://github.com/nzbgetcom/par2cmdline-turbo.git - GIT_TAG v1.1.1-nzbget + GIT_TAG v1.1.1-nzbget-20241128 TLS_VERIFY TRUE GIT_SHALLOW TRUE GIT_PROGRESS TRUE diff --git a/daemon/extension/ManifestFile.cpp b/daemon/extension/ManifestFile.cpp index 8a01e5bf..beff64e0 100644 --- a/daemon/extension/ManifestFile.cpp +++ b/daemon/extension/ManifestFile.cpp @@ -19,28 +19,34 @@ #include "nzbget.h" -#include #include "ManifestFile.h" #include "Json.h" #include "FileSystem.h" +#include "Log.h" namespace ManifestFile { - const char* MANIFEST_FILE = "manifest.json"; - const char* DEFAULT_SECTION_NAME = "options"; - bool Load(Manifest& manifest, const char* directory) { - BString<1024> path("%s%c%s", directory, PATH_SEPARATOR, MANIFEST_FILE); - std::ifstream fs(path); - if (!fs.is_open()) - return false; + std::string path = std::string(directory) + PATH_SEPARATOR + MANIFEST_FILE; + DiskFile file; + if (!file.Open(path.c_str(), DiskFile::omRead)) return false; - Json::ErrorCode ec; - Json::JsonValue jsonValue = Json::Deserialize(fs, ec); - if (ec) + std::string jsonStr; + char buffer[BUFFER_SIZE]; + while (file.ReadLine(buffer, BUFFER_SIZE)) + { + jsonStr += buffer; + } + + auto desRes = Json::Deserialize(jsonStr); + if (!desRes.has_value()) + { + error("Failed to parse %s. Syntax error.", path.c_str()); return false; + } + Json::JsonValue jsonValue = std::move(desRes.value()); Json::JsonObject json = jsonValue.as_object(); if (!ValidateAndSet(json, manifest)) diff --git a/daemon/extension/ManifestFile.h b/daemon/extension/ManifestFile.h index 5cfde4bf..0e189cc1 100644 --- a/daemon/extension/ManifestFile.h +++ b/daemon/extension/ManifestFile.h @@ -30,8 +30,9 @@ namespace ManifestFile { using SelectOption = std::variant; - extern const char* MANIFEST_FILE; - extern const char* DEFAULT_SECTION_NAME; + inline constexpr size_t BUFFER_SIZE = 4096; + inline const char* MANIFEST_FILE = "manifest.json"; + inline const char* DEFAULT_SECTION_NAME = "options"; struct Section { diff --git a/daemon/main/nzbget.h b/daemon/main/nzbget.h index 4e0a2c3f..d24ebfac 100644 --- a/daemon/main/nzbget.h +++ b/daemon/main/nzbget.h @@ -56,7 +56,7 @@ #pragma warning(disable:4800) // 'type' : forcing value to bool 'true' or 'false' (performance warning) #pragma warning(disable:4267) // 'var' : conversion from 'size_t' to 'type', possible loss of data -#define popen _popen +#define popen _wpopen #define pclose _pclose #endif diff --git a/daemon/postprocess/ParChecker.cpp b/daemon/postprocess/ParChecker.cpp index 9e48f884..63749fb7 100644 --- a/daemon/postprocess/ParChecker.cpp +++ b/daemon/postprocess/ParChecker.cpp @@ -71,7 +71,7 @@ class Repairer final : public Par2::Par2Repairer, public ParChecker::AbstractRep ? g_Options->GetParBuffer() : 0; } - Par2::Result PreProcess(const char *parFilename); + Par2::Result PreProcess(const std::string& parFilename); Par2::Result Process(bool dorepair); virtual Repairer* GetRepairer() { return this; } @@ -108,14 +108,14 @@ class RepairCreatorPacket : public Par2::CreatorPacket friend class ParChecker; }; -Par2::Result Repairer::PreProcess(const char *parFilename) +Par2::Result Repairer::PreProcess(const std::string& parFilename) { std::string memParam = "-m" + std::to_string(m_memToUse); std::string threadsParam = "-t" + std::to_string(m_threadsToUse); if (g_Options->GetParScan() == Options::psFull) { - BString<1024> wildcardParam(parFilename, 1024); + BString<1024> wildcardParam(parFilename.c_str(), 1024); char* basename = FileSystem::BaseFileName(wildcardParam); if (basename != wildcardParam && strlen(basename) > 0) { @@ -123,7 +123,7 @@ Par2::Result Repairer::PreProcess(const char *parFilename) basename[1] = '\0'; } - const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename, wildcardParam }; + const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename.c_str(), wildcardParam }; if (!m_commandLine.Parse(7, (char**)argv)) { return Par2::eInvalidCommandLineArguments; @@ -131,7 +131,7 @@ Par2::Result Repairer::PreProcess(const char *parFilename) } else { - const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename }; + const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename.c_str() }; if (!m_commandLine.Parse(6, (char**)argv)) { return Par2::eInvalidCommandLineArguments; @@ -294,7 +294,9 @@ ParChecker::EStatus ParChecker::RunParCheckAll() if (!IsStopped()) { - BString<1024> fullParFilename( "%s%c%s", m_destDir.c_str(), PATH_SEPARATOR, parFilename.c_str()); + std::string path = m_destDir + PATH_SEPARATOR + parFilename; + std::string fullParFilename = FileSystem::GetRealPath(path) + .value_or(std::move(path)); int baseLen = 0; ParParser::ParseParFilename(parFilename.c_str(), true, &baseLen, nullptr); @@ -304,7 +306,7 @@ ParChecker::EStatus ParChecker::RunParCheckAll() BString<1024> parInfoName("%s%c%s", m_nzbName.c_str(), PATH_SEPARATOR, *infoName); SetInfoName(parInfoName); - EStatus status = RunParCheck(fullParFilename); + EStatus status = RunParCheck(std::move(fullParFilename)); // accumulate total status, the worst status has priority if (allStatus > status) @@ -317,10 +319,10 @@ ParChecker::EStatus ParChecker::RunParCheckAll() return allStatus; } -ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) +ParChecker::EStatus ParChecker::RunParCheck(std::string parFilename) { Cleanup(); - m_parFilename = parFilename ? parFilename : ""; + m_parFilename = std::move(parFilename); m_stage = ptLoadingPars; m_processedCount = 0; m_extraFiles = 0; @@ -484,7 +486,7 @@ int ParChecker::PreProcessPar() m_repairer = std::make_unique(this); } - res = GetRepairer()->PreProcess(m_parFilename.c_str()); + res = GetRepairer()->PreProcess(m_parFilename); debug("ParChecker: PreProcess-result=%i", res); if (IsStopped()) @@ -1081,7 +1083,12 @@ void ParChecker::CheckEmptyFiles() if (sourcefile && sourcefile->GetDescriptionPacket()) { // GetDescriptionPacket()->FileName() returns a temp string object, which we need to hold for a while - std::string filenameObj = sourcefile->GetDescriptionPacket()->FileName(); + std::string filenameObj = Par2::DescriptionPacket::TranslateFilenameFromPar2ToLocal( + m_parCout, + m_parCerr, + Par2::nlNormal, + sourcefile->GetDescriptionPacket()->FileName() + ); if (!filenameObj.empty() && !IsProcessedFile(filenameObj.c_str())) { bool ignore = Util::MatchFileExt(filenameObj.c_str(), g_Options->GetParIgnoreExt(), ",;"); diff --git a/daemon/postprocess/ParChecker.h b/daemon/postprocess/ParChecker.h index 7f340b1c..a37f3c87 100644 --- a/daemon/postprocess/ParChecker.h +++ b/daemon/postprocess/ParChecker.h @@ -212,7 +212,7 @@ class ParChecker void Cleanup(); EStatus RunParCheckAll(); - EStatus RunParCheck(const char* parFilename); + EStatus RunParCheck(std::string parFilename); int PreProcessPar(); bool LoadMainParBak(); int ProcessMorePars(); diff --git a/daemon/postprocess/ParRenamer.cpp b/daemon/postprocess/ParRenamer.cpp index fdb7030d..9573d1f7 100644 --- a/daemon/postprocess/ParRenamer.cpp +++ b/daemon/postprocess/ParRenamer.cpp @@ -178,7 +178,12 @@ void ParRenamer::LoadParFile(const char* parFilename) m_hasDamagedParFiles = true; continue; } - std::string filename = sourceFile->GetDescriptionPacket()->FileName(); + std::string filename = Par2::DescriptionPacket::TranslateFilenameFromPar2ToLocal( + m_nout, + m_nout, + Par2::nlNormal, + sourceFile->GetDescriptionPacket()->FileName() + ); std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print(); bool exists = std::find_if(m_fileHashList.begin(), m_fileHashList.end(), diff --git a/daemon/postprocess/ParRenamer.h b/daemon/postprocess/ParRenamer.h index cc65ee06..180e2f2f 100644 --- a/daemon/postprocess/ParRenamer.h +++ b/daemon/postprocess/ParRenamer.h @@ -48,6 +48,10 @@ class ParRenamer int GetStageProgress() { return m_stageProgress; } private: + class NullStreamBuf final : public std::streambuf {}; + NullStreamBuf m_nullbuf; + std::ostream m_nout{&m_nullbuf}; + class FileHash { public: diff --git a/daemon/system/SystemInfo.cpp b/daemon/system/SystemInfo.cpp index 89942b6d..0c2d588e 100644 --- a/daemon/system/SystemInfo.cpp +++ b/daemon/system/SystemInfo.cpp @@ -195,7 +195,7 @@ namespace System std::string path = FileSystem::ExtractFilePathFromCmd(cmd); - auto result = FileSystem::GetFileRealPath(path); + auto result = FileSystem::GetRealPath(path); if (result.has_value() && FileSystem::FileExists(result.value().c_str())) { diff --git a/daemon/util/FileSystem.cpp b/daemon/util/FileSystem.cpp index 706d2e88..4f8a92ba 100644 --- a/daemon/util/FileSystem.cpp +++ b/daemon/util/FileSystem.cpp @@ -24,6 +24,10 @@ #include "Util.h" #include "Log.h" +#ifdef WIN32 +#include "utf8.h" +#endif + const char* RESERVED_DEVICE_NAMES[] = { "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", NULL }; @@ -56,21 +60,39 @@ void FileSystem::NormalizePathSeparators(char* path) } } -std::optional FileSystem::GetFileRealPath(const std::string& path) +std::optional FileSystem::GetRealPath(const std::string& path) { + if (path.empty()) return std::nullopt; + #ifdef WIN32 - char buffer[MAX_PATH]; - DWORD len = GetFullPathName(path.c_str(), MAX_PATH, buffer, nullptr); - if (len != 0) + auto res = Utf8::Utf8ToWide(path); + if (!res.has_value()) return std::nullopt; + + std::wstring wpath = std::move(res.value()); + + if (wpath.size() > MAX_DIR_PATH && std::wcsncmp(wpath.c_str(), L"\\\\", 2) == 0) + { + wpath = L"\\\\?\\UNC" + wpath; + } + else if (wpath.size() > MAX_DIR_PATH) { - return std::optional{ buffer }; + wpath = L"\\\\?\\" + wpath; } + + DWORD len = GetFullPathNameW(wpath.c_str(), 0, nullptr, nullptr); + if (len == 0) return std::nullopt; + + std::wstring buffer(len, '\0'); + len = GetFullPathNameW(wpath.c_str(), len, &buffer[0], nullptr); + if (len == 0) return std::nullopt; + + return Utf8::WideToUtf8(buffer); #else if (char* realPath = realpath(path.c_str(), nullptr)) { std::string res = realPath; free(realPath); - return std::optional(std::move(res)); + return res; } #endif @@ -78,6 +100,9 @@ std::optional FileSystem::GetFileRealPath(const std::string& path) } #ifdef WIN32 + +const size_t FileSystem::MAX_DIR_PATH = 248; + bool FileSystem::ForceDirectories(const char* path, CString& errmsg) { errmsg.Clear(); @@ -621,7 +646,7 @@ std::string FileSystem::ExtractFilePathFromCmd(const std::string& path) { if (path.empty()) { - return std::string(path); + return path; } size_t lastSeparatorPos = path.find_last_of(PATH_SEPARATOR); diff --git a/daemon/util/FileSystem.h b/daemon/util/FileSystem.h index 1aabb697..8d4b783b 100644 --- a/daemon/util/FileSystem.h +++ b/daemon/util/FileSystem.h @@ -40,7 +40,7 @@ class FileSystem static char* BaseFileName(const char* filename); static bool SameFilename(const char* filename1, const char* filename2); static void NormalizePathSeparators(char* path); - static std::optional GetFileRealPath(const std::string& path); + static std::optional GetRealPath(const std::string& path); static bool LoadFileIntoBuffer(const char* filename, CharBuffer& buffer, bool addTrailingNull); static bool SaveBufferIntoFile(const char* filename, const char* buffer, int bufLen); static bool AllocateFile(const char* filename, int64 size, bool sparse, CString& errmsg); @@ -96,6 +96,8 @@ class FileSystem static CString WidePathToUtfPath(const wchar_t* wpath); static CString MakeCanonicalPath(const char* filename); static bool NeedLongPath(const char* path); + + static const size_t MAX_DIR_PATH; #endif }; diff --git a/daemon/util/Json.cpp b/daemon/util/Json.cpp index 9dba83af..d3d51bc0 100644 --- a/daemon/util/Json.cpp +++ b/daemon/util/Json.cpp @@ -1,7 +1,7 @@ /* * This file is part of nzbget. See . * - * Copyright (C) 2023 Denis + * Copyright (C) 2023-2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,32 +21,37 @@ #include "Json.h" -namespace Json { - JsonValue Deserialize(std::istream& is, ErrorCode& ec) +namespace Json +{ + std::optional Deserialize(std::basic_istream& is) noexcept { + ErrorCode ec; StreamParser parser; std::string line; - while (std::getline(is, line)) { parser.write(line, ec); - if (ec) - { - return nullptr; - } + if (ec) return std::nullopt; } parser.finish(ec); - if (ec) - { - return nullptr; - } + if (ec) return std::nullopt; return parser.release(); } - std::string Serialize(const JsonObject& json) + std::optional Deserialize(const std::string& jsonStr) noexcept + { + ErrorCode ec; + JsonValue value = parse(jsonStr, ec); + + if (ec) return std::nullopt; + + return value; + } + + std::string Serialize(const JsonObject& json) noexcept { return serialize(json); } diff --git a/daemon/util/Json.h b/daemon/util/Json.h index 1d347ae3..f6fe93c4 100644 --- a/daemon/util/Json.h +++ b/daemon/util/Json.h @@ -1,7 +1,7 @@ /* * This file is part of nzbget. See . * - * Copyright (C) 2023 Denis + * Copyright (C) 2023-2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,6 +22,8 @@ #include #include +#include +#include namespace Json { @@ -32,8 +34,9 @@ namespace Json using StreamParser = boost::json::stream_parser; using ErrorCode = boost::system::error_code; - JsonValue Deserialize(std::istream& is, ErrorCode& ec); - std::string Serialize(const JsonObject& json); + std::optional Deserialize(std::basic_istream& is) noexcept; + std::optional Deserialize(const std::string& jsonStr) noexcept; + std::string Serialize(const JsonObject& json) noexcept; } #endif diff --git a/daemon/util/Utf8.cpp b/daemon/util/Utf8.cpp index 267884f8..8ce0488e 100644 --- a/daemon/util/Utf8.cpp +++ b/daemon/util/Utf8.cpp @@ -22,39 +22,47 @@ #include "nzbget.h" #include - #include "Utf8.h" -#include "Log.h" namespace Utf8 { - std::optional Utf8ToWide(const std::string& str) noexcept + std::optional Utf8ToWide(const std::string& str) { if (str.empty()) return L""; int requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, nullptr, 0); if (requiredSize <= 0) return std::nullopt; - std::wstring wstr(requiredSize, '\0'); + wchar_t* buffer = new (std::nothrow) wchar_t[requiredSize]; + if (!buffer) return std::nullopt; - requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, wstr.data(), requiredSize); - if (requiredSize <= 0) return std::nullopt; + requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, buffer, requiredSize); + if (requiredSize <= 0) + { + delete[] buffer; + return std::nullopt; + } - return wstr; + return std::wstring(buffer); } - std::optional WideToUtf8(const std::wstring& wstr) noexcept + std::optional WideToUtf8(const std::wstring& wstr) { if (wstr.empty()) return ""; int requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, nullptr, 0, nullptr, nullptr); if (requiredSize <= 0) return std::nullopt; - std::string str(requiredSize, '\0'); + char* buffer = new (std::nothrow) char[requiredSize]; + if (!buffer) return std::nullopt; - requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, str.data(), requiredSize, nullptr, nullptr); - if (requiredSize <= 0) return std::nullopt; + requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, buffer, requiredSize, nullptr, nullptr); + if (requiredSize <= 0) + { + delete[] buffer; + return std::nullopt; + } - return str; + return std::string(buffer); } } diff --git a/daemon/util/Utf8.h b/daemon/util/Utf8.h index 0e4003ee..6cd1eaec 100644 --- a/daemon/util/Utf8.h +++ b/daemon/util/Utf8.h @@ -26,8 +26,8 @@ namespace Utf8 { - std::optional Utf8ToWide(const std::string& str) noexcept; - std::optional WideToUtf8(const std::wstring& wstr) noexcept; + std::optional Utf8ToWide(const std::string& str); + std::optional WideToUtf8(const std::wstring& wstr); } #endif diff --git a/daemon/util/Util.cpp b/daemon/util/Util.cpp index a17d590a..f561861f 100644 --- a/daemon/util/Util.cpp +++ b/daemon/util/Util.cpp @@ -27,6 +27,8 @@ #include "YEncode.h" #ifdef WIN32 +#include "utf8.h" + // getopt for WIN32: // from http://www.codeproject.com/cpp/xgetopt.asp // Original Author: Hans Dietrich (hdietrich2@hotmail.com) @@ -707,7 +709,15 @@ uint32 Util::HashBJ96(const char* buffer, int bufSize, uint32 initValue) std::unique_ptr> Util::MakePipe(const std::string& cmd) { +#ifdef WIN32 + auto res = Utf8::Utf8ToWide(cmd); + if (!res.has_value()) return nullptr; + + FILE* pipe = popen(res.value().c_str(), L"r"); +#else FILE* pipe = popen(cmd.c_str(), "r"); +#endif + return std::unique_ptr>(pipe, [](FILE* pipe) { if (pipe) diff --git a/tests/extension/CMakeLists.txt b/tests/extension/CMakeLists.txt index 91d47cc8..eb910150 100644 --- a/tests/extension/CMakeLists.txt +++ b/tests/extension/CMakeLists.txt @@ -31,6 +31,10 @@ set(ExtensionSrc ${CMAKE_SOURCE_DIR}/daemon/connect/TlsSocket.cpp ) +if(WIN32) + set(ExtensionSrc ${ExtensionSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + add_executable(ExtensionTests ${ExtensionSrc}) target_link_libraries(ExtensionTests PRIVATE ${LIBS}) target_include_directories(ExtensionTests PRIVATE ${INCLUDES}) diff --git a/tests/feed/CMakeLists.txt b/tests/feed/CMakeLists.txt index 1f4067de..a41b8a25 100644 --- a/tests/feed/CMakeLists.txt +++ b/tests/feed/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(FeedTests +set(FeedTestsSrc FeedFilterTest.cpp FeedFileTest.cpp main.cpp @@ -14,6 +14,13 @@ add_executable(FeedTests ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) + +if(WIN32) + set(FeedTestsSrc ${FeedTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + +add_executable(FeedTests ${FeedTestsSrc}) + target_link_libraries(FeedTests PRIVATE ${LIBS}) target_include_directories(FeedTests PRIVATE ${INCLUDES}) diff --git a/tests/main/CMakeLists.txt b/tests/main/CMakeLists.txt index 1d462031..1ada024d 100644 --- a/tests/main/CMakeLists.txt +++ b/tests/main/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(MainTests +set(MainTestsSrc main.cpp OptionsTest.cpp CommandLineParserTest.cpp @@ -13,6 +13,13 @@ add_executable(MainTests ${CMAKE_SOURCE_DIR}/daemon/feed/FeedInfo.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) + +if(WIN32) + set(MainTestsSrc ${MainTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + +add_executable(MainTests ${MainTestsSrc}) + target_link_libraries(MainTests PRIVATE ${LIBS}) target_include_directories(MainTests PRIVATE ${INCLUDES}) diff --git a/tests/nntp/CMakeLists.txt b/tests/nntp/CMakeLists.txt index 8a844c38..9e1b2673 100644 --- a/tests/nntp/CMakeLists.txt +++ b/tests/nntp/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(NNTPTests +set(NNTPTestsSrc ServerPoolTest.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/ServerPool.cpp ${CMAKE_SOURCE_DIR}/daemon/main/Options.cpp @@ -14,6 +14,13 @@ add_executable(NNTPTests ${CMAKE_SOURCE_DIR}/daemon/connect/Connection.cpp ${CMAKE_SOURCE_DIR}/daemon/connect/TlsSocket.cpp ) + +if(WIN32) + set(NNTPTestsSrc ${NNTPTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + +add_executable(NNTPTests ${NNTPTestsSrc}) + target_link_libraries(NNTPTests PRIVATE ${LIBS}) target_include_directories(NNTPTests PRIVATE ${INCLUDES}) diff --git a/tests/postprocess/CMakeLists.txt b/tests/postprocess/CMakeLists.txt index 0f5dabb0..98886cc0 100644 --- a/tests/postprocess/CMakeLists.txt +++ b/tests/postprocess/CMakeLists.txt @@ -25,7 +25,12 @@ set(PostprocessTestsSrc ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp ) +if(WIN32) + set(PostprocessTestsSrc ${PostprocessTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + add_executable(PostprocessTests ${PostprocessTestsSrc}) + target_link_libraries(PostprocessTests PRIVATE ${LIBS}) target_include_directories(PostprocessTests PRIVATE ${INCLUDES} ../suite) diff --git a/tests/postprocess/ParRenamerTest.cpp b/tests/postprocess/ParRenamerTest.cpp index e6c945fe..37c90a7a 100644 --- a/tests/postprocess/ParRenamerTest.cpp +++ b/tests/postprocess/ParRenamerTest.cpp @@ -170,7 +170,6 @@ BOOST_AUTO_TEST_CASE(Utf8Par2Test) Options options(&cmdOpts, nullptr); std::string workingDir = currDir + PATH_SEPARATOR + "Utf8Par2Test"; - std::string renamedTestFileRar = workingDir + PATH_SEPARATOR + "Привет, мир!.rar"; ParRenamerMock parRenamer(workingDir, testDataDirUtf8); @@ -178,6 +177,4 @@ BOOST_AUTO_TEST_CASE(Utf8Par2Test) BOOST_CHECK_EQUAL(parRenamer.GetRenamedCount(), 1); BOOST_CHECK(parRenamer.HasMissedFiles() == false); - - BOOST_CHECK(FileSystem::FileExists(renamedTestFileRar.c_str())); } diff --git a/tests/queue/CMakeLists.txt b/tests/queue/CMakeLists.txt index 8cb31ab0..6a7683a6 100644 --- a/tests/queue/CMakeLists.txt +++ b/tests/queue/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(QueueTests +set(QueueTestsSrc NzbFileTest.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/NzbFile.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp @@ -12,6 +12,13 @@ add_executable(QueueTests ${CMAKE_SOURCE_DIR}/daemon/util/Log.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) + +if(WIN32) + set(QueueTestsSrc ${QueueTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + +add_executable(QueueTests ${QueueTestsSrc}) + target_link_libraries(QueueTests PRIVATE ${LIBS}) target_include_directories(QueueTests PRIVATE ${INCLUDES}) diff --git a/tests/system/CMakeLists.txt b/tests/system/CMakeLists.txt index c55eaec9..c6d9c6b3 100644 --- a/tests/system/CMakeLists.txt +++ b/tests/system/CMakeLists.txt @@ -19,7 +19,12 @@ set(SystemTestsSrc ${CMAKE_SOURCE_DIR}/daemon/util/Log.cpp ) +if(WIN32) + set(SystemTestsSrc ${SystemTestsSrc} ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp) +endif() + add_executable(SystemTests ${SystemTestsSrc}) + target_link_libraries(SystemTests PRIVATE ${LIBS}) target_include_directories(SystemTests PRIVATE ${INCLUDES}) diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index 52253ea2..e88cd1e7 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -22,6 +22,7 @@ if(WIN32) endif() add_executable(UtilTests ${UtilTestSrc}) + target_link_libraries(UtilTests PRIVATE ${LIBS}) target_include_directories(UtilTests PRIVATE ${INCLUDES}) diff --git a/tests/util/JsonTest.cpp b/tests/util/JsonTest.cpp index cb9fe09c..3e07a016 100644 --- a/tests/util/JsonTest.cpp +++ b/tests/util/JsonTest.cpp @@ -28,18 +28,34 @@ BOOST_AUTO_TEST_CASE(JsonDeserializeTest) { - std::string validJSON = "{\"name\": \"John\", \"secondName\": \"Doe\"}"; - std::stringstream is; - is << validJSON; - - Json::ErrorCode ec; - Json::Deserialize(is, ec); - BOOST_CHECK(ec.failed() == false); - - std::string invalidJSON = "{\"name\": \"John\", \"secondName\":}"; - - is.clear(); - is << invalidJSON; - Json::Deserialize(is, ec); - BOOST_CHECK(ec.failed() == true); + { + std::string validJSON = "{\"name\": \"John\", \"secondName\": \"Doe\"}"; + std::stringstream is; + is << validJSON; + + { + auto res = Json::Deserialize(is); + BOOST_CHECK(res.has_value()); + } + + { + auto res = Json::Deserialize(validJSON); + BOOST_CHECK(res.has_value()); + } + + std::string invalidJSON = "{\"name\": \"John\", \"secondName\":}"; + + is.clear(); + is << invalidJSON; + + { + auto res = Json::Deserialize(is); + BOOST_CHECK(!res.has_value()); + } + + { + auto res = Json::Deserialize(invalidJSON); + BOOST_CHECK(!res.has_value()); + } + } } diff --git a/tests/util/Utf8Test.cpp b/tests/util/Utf8Test.cpp index 919b4161..068975ee 100644 --- a/tests/util/Utf8Test.cpp +++ b/tests/util/Utf8Test.cpp @@ -90,4 +90,29 @@ BOOST_AUTO_TEST_CASE(Utf8Test) BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); } + + { + string veryLongString = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \ + Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \ + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi \ + ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit \ + in voluptate velit esse cillum dolore eu fugiat nulla pariatur."; + + wstring expectedWide = L"Lorem ipsum dolor sit amet, consectetur adipiscing elit. \ + Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \ + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi \ + ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit \ + in voluptate velit esse cillum dolore eu fugiat nulla pariatur."; + wstring actualWide = Utf8ToWide(veryLongString).value(); + + string expectedUtf8 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \ + Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \ + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi \ + ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit \ + in voluptate velit esse cillum dolore eu fugiat nulla pariatur."; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } } From 07f8f1315fff90f0a57372b09b968916e9e93bc4 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:10:44 +0300 Subject: [PATCH 6/9] Add: additional checks for platform and compiler flags (#451) - fixed armel support - added additional checks for platform and compiler flags --- cmake/common.cmake | 17 +++++++++++ lib/sources.cmake | 72 ++++++++++++++++++++++++---------------------- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index 8fe927ca..cbeb9f6e 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1,3 +1,19 @@ +if (CMAKE_SYSTEM_PROCESSOR MATCHES "i386|i686|x86|x86_64|x64|amd64|AMD64|win32|Win32") + set(IS_X86 TRUE) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|x64|amd64|AMD64") + set(IS_X64 TRUE) + endif() +endif() +if (CMAKE_SYSTEM_PROCESSOR MATCHES "arm|ARM|aarch64|arm64|ARM64|armeb|aarch64be|aarch64_be") + set(IS_ARM TRUE) +endif() +if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64|rv64") + set(IS_RISCV64 TRUE) +endif() +if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv32|rv32") + set(IS_RISCV32 TRUE) +endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") add_compile_options(-Weverything -Wno-c++98-compat) @@ -22,3 +38,4 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "Release") endif() include(ExternalProject) +include(CheckCXXCompilerFlag) diff --git a/lib/sources.cmake b/lib/sources.cmake index 3715757d..6e56efc7 100644 --- a/lib/sources.cmake +++ b/lib/sources.cmake @@ -1,12 +1,41 @@ -if(CMAKE_SYSTEM_PROCESSOR MATCHES "i?86|x86_64") - set(SSE2_CXXFLAGS "-msse2") - set(SSSE3_CXXFLAGS "-mssse3") - set(PCLMUL_CXXFLAGS "-msse4.1 -mpclmul") -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") - set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm.*") - set(NEON_CXXFLAGS "-mfpu=neon") - set(ACLECRC_CXXFLAGS "-march=armv8-a+crc -fpermissive") +if(IS_ARM) + CHECK_CXX_COMPILER_FLAG("-mfpu=neon" COMPILER_SUPPORTS_ARM32_NEON) + if(COMPILER_SUPPORTS_ARM32_NEON) + set_source_files_properties( + ${CMAKE_SOURCE_DIR}/lib/yencode/NeonDecoder.cpp + PROPERTIES COMPILE_FLAGS "-mfpu=neon" + ) + else() + CHECK_CXX_COMPILER_FLAG("-march=armv8-a+crc" COMPILER_SUPPORTS_ARM_CRC) + set_source_files_properties( + ${CMAKE_SOURCE_DIR}/lib/yencode/AcleCrc.cpp + PROPERTIES COMPILE_FLAGS "-march=armv8-a+crc" + ) + endif() +endif() + +if(IS_X86) + CHECK_CXX_COMPILER_FLAG("-msse2" COMPILER_SUPPORTS_SSE2) + if(COMPILER_SUPPORTS_SSE2) + set_source_files_properties( + ${CMAKE_SOURCE_DIR}/lib/yencode/Sse2Decoder.cpp + PROPERTIES COMPILE_FLAGS "-msse2" + ) + endif() + CHECK_CXX_COMPILER_FLAG("-mssse3" COMPILER_SUPPORTS_SSSE3) + if(COMPILER_SUPPORTS_SSSE3) + set_source_files_properties( + ${CMAKE_SOURCE_DIR}/lib/yencode/Ssse3Decoder.cpp + PROPERTIES COMPILE_FLAGS "-mssse3" + ) + endif() + CHECK_CXX_COMPILER_FLAG("-msse4.1 -mpclmul" COMPILER_SUPPORTS_SSE41_PCLMUL) + if(COMPILER_SUPPORTS_SSE41_PCLMUL) + set_source_files_properties( + ${CMAKE_SOURCE_DIR}/lib/yencode/PclmulCrc.cpp + PROPERTIES COMPILE_FLAGS "-msse4.1 -mpclmul" + ) + endif() endif() add_library(regex STATIC @@ -17,31 +46,6 @@ target_include_directories(regex PUBLIC ${CMAKE_SOURCE_DIR}/lib/regex ) -set_source_files_properties( - ${CMAKE_SOURCE_DIR}/lib/yencode/Sse2Decoder.cpp - PROPERTIES COMPILE_FLAGS "${SSE2_CXXFLAGS}" -) - -set_source_files_properties( - ${CMAKE_SOURCE_DIR}/lib/yencode/Ssse3Decoder.cpp - PROPERTIES COMPILE_FLAGS "${SSSE3_CXXFLAGS}" -) - -set_source_files_properties( - ${CMAKE_SOURCE_DIR}/lib/yencode/PclmulCrc.cpp - PROPERTIES COMPILE_FLAGS "${PCLMUL_CXXFLAGS}" -) - -set_source_files_properties( - ${CMAKE_SOURCE_DIR}/lib/yencode/NeonDecoder.cpp - PROPERTIES COMPILE_FLAGS "${NEON_CXXFLAGS}" -) - -set_source_files_properties( - ${CMAKE_SOURCE_DIR}/lib/yencode/AcleCrc.cpp - PROPERTIES COMPILE_FLAGS "${ACLECRC_CXXFLAGS}" -) - add_library(yencode STATIC ${CMAKE_SOURCE_DIR}/lib/yencode/SimdInit.cpp ${CMAKE_SOURCE_DIR}/lib/yencode/SimdDecoder.cpp From a6df976023c9d91b21b3c60f1454348ee0f5e781 Mon Sep 17 00:00:00 2001 From: phnzb <139914799+phnzb@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:06:08 +0300 Subject: [PATCH 7/9] add asustor info (#452) --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b6621b95..e406b256 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,9 @@ macOS packages are available from [releases](https://github.com/nzbgetcom/nzbget Docker images are available for x86-64 / arm64 / armv7 architectures. [Docker readme](docker/README.md). LinuxServer.io version is also available: [docker-nzbget](https://github.com/linuxserver/docker-nzbget) -Synology package are available as SynoCommunity package. [Synology readme](docs/SYNOLOGY.md) +Synology package is available as SynoCommunity package. [Synology readme](docs/SYNOLOGY.md) + +ASUSTOR NAS package (ADM 4.3+) is available from the [nzbget-asustor](https://github.com/nzbgetcom/nzbget-asustor) repository and from the ASUSTOR [App Central](https://www.asustor.com/app_central/app_detail?id=1671&type=). [ASUSTOR readme](https://github.com/nzbgetcom/nzbget-asustor/blob/main/README.md) QNAP packages are available as buildroot packages or via [sherpa](https://github.com/OneCDOnly/sherpa) package manager. [QNAP readme](qnap/README.md) From 7ec9730a2f05d96bafae0fdcf4e336d8bf64bc89 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:09:04 +0300 Subject: [PATCH 8/9] v24.5 release preparation (#453) --- .github/ISSUE_TEMPLATE/01_bug_report.yml | 3 ++- .github/ISSUE_TEMPLATE/02_feature_request.yml | 2 +- .github/ISSUE_TEMPLATE/03_build.yml | 3 ++- ChangeLog.md | 17 +++++++++++++++++ README.md | 1 + 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/01_bug_report.yml b/.github/ISSUE_TEMPLATE/01_bug_report.yml index d6bfbe49..3eb4c932 100644 --- a/.github/ISSUE_TEMPLATE/01_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/01_bug_report.yml @@ -14,7 +14,8 @@ body: label: NZBGet Version description: Which version of NZBGet has this bug? options: - - v24.5-testing + - v24.6-testing + - v24.5-stable - v24.4-stable - v24.3-stable - v24.2-stable diff --git a/.github/ISSUE_TEMPLATE/02_feature_request.yml b/.github/ISSUE_TEMPLATE/02_feature_request.yml index b3464623..50ad968b 100644 --- a/.github/ISSUE_TEMPLATE/02_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/02_feature_request.yml @@ -17,7 +17,7 @@ body: - All - Windows - macOS - - NAS/Synology/QNAP + - NAS/Synology/QNAP/ASUSTOR - Linux/Docker - FreeBSD - Android diff --git a/.github/ISSUE_TEMPLATE/03_build.yml b/.github/ISSUE_TEMPLATE/03_build.yml index 2d530b27..ee6f07ca 100644 --- a/.github/ISSUE_TEMPLATE/03_build.yml +++ b/.github/ISSUE_TEMPLATE/03_build.yml @@ -7,7 +7,8 @@ body: label: NZBGet Version description: Version of NZBGet for the scope of this issue options: - - v24.5-testing + - v24.6-testing + - v24.5-stable - v24.4-stable - v24.3-stable - v24.2-stable diff --git a/ChangeLog.md b/ChangeLog.md index 387b07df..166b58eb 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,20 @@ +nzbget-25.5 + - Features: + - ASUSTOR support + - the package (ADM 4.3+) is available from the + [nzbget-asustor](https://github.com/nzbgetcom/nzbget-asustor) repository + and from the ASUSTOR App Central + + - Bug fixes: + - Fixed multiple bugs related to bad support of long-paths on Windows + [#441](https://github.com/nzbgetcom/nzbget/pull/441) + - Fixed ARMv5 arch support + [#451](https://github.com/nzbgetcom/nzbget/pull/451) + + - For developers: + - Fixed homebrew CI build, macOS ARM64 with Xcode 16, Boost.JSON build via CMake + [#444](https://github.com/nzbgetcom/nzbget/pull/444) + nzbget-24.4 - Features: - Replaced the `Par2` library with `Par2-turbo` which provides faster file recovery on x86/ARM platforms diff --git a/README.md b/README.md index e406b256..d6935867 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.2/total?label=v24.2) ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.3/total?label=v24.3) ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.4/total?label=v24.4) +![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.5/total?label=v24.5) ![docker pulls](https://img.shields.io/docker/pulls/nzbgetcom/nzbget.svg) [![linux build](https://github.com/nzbgetcom/nzbget/actions/workflows/linux.yml/badge.svg?branch=main)](https://github.com/nzbgetcom/nzbget/actions/workflows/linux.yml) From e3286dcff5e71ecb1f2064b96553e3389c7272f3 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:51:42 +0300 Subject: [PATCH 9/9] Fix: nzbget version in changelog (#455) --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 166b58eb..9c645d3b 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,4 +1,4 @@ -nzbget-25.5 +nzbget-24.5 - Features: - ASUSTOR support - the package (ADM 4.3+) is available from the