From 7348b19cf8a81739922303e3f9047d06e59670d0 Mon Sep 17 00:00:00 2001 From: Denis <146707790+dnzbk@users.noreply.github.com> Date: Mon, 5 Feb 2024 05:45:15 -0800 Subject: [PATCH] Feature/extension manager (#76) #### The new nzbget extension system, which makes it easy to download/update/delete extensions with backward compatibility with the old system. Extensions [master list](https://github.com/nzbgetcom/nzbget-extensions/). - Changed - RPC request "configtemplates" - no longer returns script templates, but only the config template. - Added - new RPC requests: - "loadextensions" - loads all extensions from {ScriptDIR} and returns an array of structures in JSON/XML formats. - "updateextension" - downloads by url and name and installs the extension. Returns 'true' or error response in JSON/XML formats. - "deleteextension" - deletes extension by name. Returns 'true' or error response in JSON/XML formats. - "downloadextension" - downloads by url and installs the extension. Returns 'true' or error response in JSON/XML formats. - "testextension" - tries to find the right executor program for the extension, e.g. Python. Returns 'true' or error response in JSON/XML formats. - "EXTENSION MANAGER" section in webui to download/delete/update extensions - Boost.Json library to work with JSON - more unit tests - Refactored - replaced raw pointers with smart pointers and const refs where possible for memory safty reasons - Updated - INSTALALTION.md - Removed - testdata_FILES from Makefile.am - EMail and Logger scripts --------- Co-authored-by: phnzb --- CMakeLists.txt | 1 + INSTALLATION.md | 22 +- Makefile.am | 68 +- configure.ac | 3 + daemon/extension/CommandScript.cpp | 8 +- daemon/extension/CommandScript.h | 2 +- daemon/extension/Extension.cpp | 436 +++++++ daemon/extension/Extension.h | 117 ++ daemon/extension/ExtensionLoader.cpp | 422 +++++++ daemon/extension/ExtensionLoader.h | 81 ++ daemon/extension/ExtensionManager.cpp | 373 ++++++ daemon/extension/ExtensionManager.h | 82 ++ daemon/extension/FeedScript.cpp | 4 +- daemon/extension/FeedScript.h | 2 +- daemon/extension/ManifestFile.cpp | 232 ++++ daemon/extension/ManifestFile.h | 82 ++ daemon/extension/NzbScript.cpp | 6 +- daemon/extension/NzbScript.h | 5 +- daemon/extension/PostScript.cpp | 22 +- daemon/extension/PostScript.h | 4 +- daemon/extension/QueueScript.cpp | 34 +- daemon/extension/QueueScript.h | 12 +- daemon/extension/ScanScript.cpp | 6 +- daemon/extension/ScanScript.h | 2 +- daemon/extension/SchedulerScript.cpp | 4 +- daemon/extension/SchedulerScript.h | 2 +- daemon/extension/ScriptConfig.cpp | 345 +----- daemon/extension/ScriptConfig.h | 61 +- daemon/main/Options.h | 8 +- daemon/main/nzbget.cpp | 16 +- daemon/main/nzbget.h | 4 +- daemon/postprocess/Unpack.cpp | 17 +- daemon/postprocess/Unpack.h | 2 +- daemon/queue/Scanner.cpp | 10 +- daemon/remote/XmlRpc.cpp | 246 +++- daemon/util/Json.cpp | 53 + daemon/util/Json.h | 39 + daemon/util/ScriptController.cpp | 59 +- daemon/util/ScriptController.h | 1 - daemon/util/Util.cpp | 109 +- daemon/util/Util.h | 4 + daemon/util/Xml.cpp | 42 + daemon/util/Xml.h | 31 + docker/Dockerfile | 4 +- nzbget.vcxproj | 21 +- osx/build-info.md | 2 +- osx/build-nzbget.sh | 28 +- posix/ax_boost_json.m4 | 123 ++ scripts/.gitkeep | 0 scripts/EMail.py | 291 ----- scripts/Logger.py | 95 -- synology/README.md | 2 +- synology/build-nzbget.sh | 16 +- synology/package/SynoBuildConf/build | 27 +- synology/package/SynoBuildConf/install | 3 +- tests/CMakeLists.txt | 3 +- tests/extension/CMakeLists.txt | 63 + tests/extension/ExtensionLoaderTest.cpp | 150 +++ tests/extension/ExtensionManagerTest.cpp | 89 ++ tests/extension/ExtensionTest.cpp | 150 +++ tests/extension/ManifestFileTest.cpp | 84 ++ tests/extension/main.cpp | 3 + tests/feed/FeedFileTest.cpp | 2 + tests/queue/NzbFileTest.cpp | 2 + tests/testdata/extension/V1/Extension.py | 54 + .../extension/manifest/invalid/manifest.json | 9 + .../extension/manifest/valid/manifest.json | 39 + .../testdata/extension/scripts/Email/email.py | 1 + .../extension/scripts/Email/manifest.json | 32 + .../testdata/extension/scripts/Extension1.py | 34 + .../scripts/Extension2/Extension2.py | 34 + .../testdata/extension/scripts/Extension3.py | 34 + tests/util/CMakeLists.txt | 4 + tests/util/JsonTest.cpp | 45 + tests/util/main.cpp | 1 + webui/config.js | 1070 +++++++++++++++-- webui/edit.js | 24 +- webui/img/download-16.ico | Bin 0 -> 1150 bytes webui/img/house-16.ico | Bin 0 -> 1150 bytes webui/img/warning-16.ico | Bin 0 -> 1150 bytes webui/index.html | 52 +- webui/index.js | 23 +- webui/style.css | 36 +- windows/nzbget-setup.nsi | 1 - 84 files changed, 4592 insertions(+), 1138 deletions(-) create mode 100644 daemon/extension/Extension.cpp create mode 100644 daemon/extension/Extension.h create mode 100644 daemon/extension/ExtensionLoader.cpp create mode 100644 daemon/extension/ExtensionLoader.h create mode 100644 daemon/extension/ExtensionManager.cpp create mode 100644 daemon/extension/ExtensionManager.h create mode 100644 daemon/extension/ManifestFile.cpp create mode 100644 daemon/extension/ManifestFile.h create mode 100644 daemon/util/Json.cpp create mode 100644 daemon/util/Json.h create mode 100644 daemon/util/Xml.cpp create mode 100644 daemon/util/Xml.h create mode 100644 posix/ax_boost_json.m4 create mode 100644 scripts/.gitkeep delete mode 100755 scripts/EMail.py delete mode 100755 scripts/Logger.py create mode 100644 tests/extension/CMakeLists.txt create mode 100644 tests/extension/ExtensionLoaderTest.cpp create mode 100644 tests/extension/ExtensionManagerTest.cpp create mode 100644 tests/extension/ExtensionTest.cpp create mode 100644 tests/extension/ManifestFileTest.cpp create mode 100644 tests/extension/main.cpp create mode 100644 tests/testdata/extension/V1/Extension.py create mode 100644 tests/testdata/extension/manifest/invalid/manifest.json create mode 100644 tests/testdata/extension/manifest/valid/manifest.json create mode 100644 tests/testdata/extension/scripts/Email/email.py create mode 100644 tests/testdata/extension/scripts/Email/manifest.json create mode 100644 tests/testdata/extension/scripts/Extension1.py create mode 100644 tests/testdata/extension/scripts/Extension2/Extension2.py create mode 100644 tests/testdata/extension/scripts/Extension3.py create mode 100644 tests/util/JsonTest.cpp create mode 100644 webui/img/download-16.ico create mode 100644 webui/img/house-16.ico create mode 100644 webui/img/warning-16.ico diff --git a/CMakeLists.txt b/CMakeLists.txt index 765acc7ab..4165e27b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") find_package(OpenSSL REQUIRED) find_package(ZLIB REQUIRED) find_package(LibXml2 REQUIRED) +find_package(Boost REQUIRED COMPONENTS json) if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Weverything") diff --git a/INSTALLATION.md b/INSTALLATION.md index a08d0dab2..2dd0fdefc 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -1,4 +1,4 @@ -# NZBGet installation +# NZBGet installation This is a short documentation. For more information please visit NZBGet home page at @@ -62,29 +62,34 @@ NZBGet absolutely needs the following libraries: - libstdc++ (usually part of compiler) - [libxml2](https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home) + - [Boost.JSON](https://www.boost.org/doc/libs/1_84_0/libs/json/doc/html/index.html) + - [Boost.Optional](https://www.boost.org/doc/libs/1_84_0/libs/optional/doc/html/index.html) And the following libraries are optional: - - for curses-output-mode (enabled by default): + For curses-output-mode (enabled by default): - libcurses (usually part of commercial systems) or (better) - [libncurses](https://invisible-island.net/ncurses) - - for encrypted connections (TLS/SSL): + For encrypted connections (TLS/SSL): - [OpenSSL](https://www.openssl.org) or - [GnuTLS](https://gnutls.org) - - for gzip support in web-server and web-client (enabled by default): + For gzip support in web-server and web-client (enabled by default): - [zlib](https://www.zlib.net/) - - for configuration: + For configuration: - [autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) - [autoconf](https://www.gnu.org/software/autoconf/) - - for managing package dependencies: + For managing package dependencies: - [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/) + + For tests: + - [Boost.Test](https://www.boost.org/doc/libs/1_84_0/libs/test/doc/html/index.html) All these libraries are included in modern POSIX distributions and should be available as installable packages. Please note that you also @@ -219,6 +224,11 @@ Also required are: - [Regex](https://regexlib.com/) - [Zlib](https://gnuwin32.sourceforge.net/packages/zlib.htm) - [libxml2](https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home) + - [Boost.JSON](https://www.boost.org/doc/libs/1_84_0/libs/json/doc/html/index.html) + - [Boost.Optional](https://www.boost.org/doc/libs/1_84_0/libs/optional/doc/html/index.html) + +For tests: + - [Boost.Test](https://www.boost.org/doc/libs/1_84_0/libs/test/doc/html/index.html) We recommend using [vcpkg](https://vcpkg.io/) to install dependencies. diff --git a/Makefile.am b/Makefile.am index 473fb0fa8..b853f8f8a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -42,6 +42,14 @@ nzbget_SOURCES = \ daemon/extension/SchedulerScript.h \ daemon/extension/ScriptConfig.cpp \ daemon/extension/ScriptConfig.h \ + daemon/extension/Extension.cpp \ + daemon/extension/Extension.h \ + daemon/extension/ExtensionManager.cpp \ + daemon/extension/ExtensionManager.h \ + daemon/extension/ExtensionLoader.cpp \ + daemon/extension/ExtensionLoader.h \ + daemon/extension/ManifestFile.cpp \ + daemon/extension/ManifestFile.h \ daemon/feed/FeedCoordinator.cpp \ daemon/feed/FeedCoordinator.h \ daemon/feed/FeedFile.cpp \ @@ -158,6 +166,10 @@ nzbget_SOURCES = \ daemon/util/Service.h \ daemon/util/FileSystem.cpp \ daemon/util/FileSystem.h \ + daemon/util/Json.cpp \ + daemon/util/Json.h \ + daemon/util/Xml.cpp \ + daemon/util/Xml.h \ daemon/util/Util.cpp \ daemon/util/Util.h \ daemon/nserv/NServMain.h \ @@ -353,6 +365,8 @@ webui_FILES = \ webui/lib/raphael.min.js \ webui/lib/elycharts.js \ webui/lib/elycharts.min.js \ + webui/img/house-16.ico \ + webui/img/download-16.ico \ webui/img/icons.png \ webui/img/icons-2x.png \ webui/img/transmit.gif \ @@ -364,64 +378,13 @@ webui_FILES = \ webui/img/favicon-256x256-opaque.png \ webui/img/favicon-256x256.png -scripts_FILES = \ - scripts/EMail.py \ - scripts/Logger.py - -testdata_FILES = \ - tests/testdata/dupematcher1/testfile.part01.rar \ - tests/testdata/dupematcher1/testfile.part24.rar \ - tests/testdata/dupematcher2/testfile.part04.rar \ - tests/testdata/dupematcher2/testfile.part43.rar \ - tests/testdata/nzbfile/dotless.nzb \ - tests/testdata/nzbfile/dotless.txt \ - tests/testdata/nzbfile/plain.nzb \ - tests/testdata/nzbfile/plain.txt \ - tests/testdata/parchecker/crc.txt \ - tests/testdata/parchecker/testfile.dat \ - tests/testdata/parchecker/testfile.nfo \ - tests/testdata/parchecker/testfile.par2 \ - tests/testdata/parchecker/testfile.vol00+1.PAR2 \ - tests/testdata/parchecker/testfile.vol01+2.PAR2 \ - tests/testdata/parchecker/testfile.vol03+3.PAR2 \ - tests/testdata/parchecker2/crc.txt \ - tests/testdata/parchecker2/testfile.7z.001 \ - tests/testdata/parchecker2/testfile.7z.002 \ - tests/testdata/parchecker2/testfile.7z.003 \ - tests/testdata/parchecker2/testfile.7z.par2 \ - tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \ - tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \ - tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \ - tests/testdata/rarrenamer/testfile3.part01.rar \ - tests/testdata/rarrenamer/testfile3.part02.rar \ - tests/testdata/rarrenamer/testfile3.part03.rar \ - tests/testdata/rarrenamer/testfile5.part01.rar \ - tests/testdata/rarrenamer/testfile5.part02.rar \ - tests/testdata/rarrenamer/testfile5.part03.rar \ - tests/testdata/rarrenamer/testfile3oldnam.rar \ - tests/testdata/rarrenamer/testfile3oldnam.r00 \ - tests/testdata/rarrenamer/testfile3oldnam.r01 \ - tests/testdata/rarrenamer/testfile3encdata.part01.rar \ - tests/testdata/rarrenamer/testfile3encdata.part02.rar \ - tests/testdata/rarrenamer/testfile3encdata.part03.rar \ - tests/testdata/rarrenamer/testfile3encnam.part01.rar \ - tests/testdata/rarrenamer/testfile3encnam.part02.rar \ - tests/testdata/rarrenamer/testfile3encnam.part03.rar \ - tests/testdata/rarrenamer/testfile5encdata.part01.rar \ - tests/testdata/rarrenamer/testfile5encdata.part02.rar \ - tests/testdata/rarrenamer/testfile5encdata.part03.rar \ - tests/testdata/rarrenamer/testfile5encnam.part01.rar \ - tests/testdata/rarrenamer/testfile5encnam.part02.rar \ - tests/testdata/rarrenamer/testfile5encnam.part03.rar - # Install dist_doc_DATA = $(doc_FILES) exampleconfdir = $(datadir)/nzbget dist_exampleconf_DATA = $(exampleconf_FILES) webuidir = $(datadir)/nzbget nobase_dist_webui_DATA = $(webui_FILES) -scriptsdir = $(datadir)/nzbget -nobase_dist_scripts_SCRIPTS = $(scripts_FILES) +scriptsdir = $(datadir)/nzbget/scripts # Note about "sed": # We need to make some changes in installed files. @@ -434,6 +397,7 @@ nobase_dist_scripts_SCRIPTS = $(scripts_FILES) # Prepare example configuration file install-data-hook: + mkdir -p "$(DESTDIR)$(scriptsdir)" rm -f "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp" cp "$(DESTDIR)$(exampleconfdir)/nzbget.conf" "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp" sed 's:^ConfigTemplate=:ConfigTemplate=$(exampleconfdir)/nzbget.conf:' < "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp" > "$(DESTDIR)$(exampleconfdir)/nzbget.conf" diff --git a/configure.ac b/configure.ac index 78d9aa489..840058c51 100644 --- a/configure.ac +++ b/configure.ac @@ -31,7 +31,9 @@ AC_DEFINE(HAVE_CONFIG_H, 1, [Define to 1 use config.h]) AM_MAINTAINER_MODE m4_include([posix/ax_cxx_compile_stdcxx.m4]) +m4_include([posix/ax_boost_json.m4]) +AX_BOOST_JSON dnl dnl Check for programs. @@ -77,6 +79,7 @@ AC_SEARCH_LIBS([socket], [socket]) AC_SEARCH_LIBS([inet_addr], [nsl]) AC_SEARCH_LIBS([hstrerror], [resolv]) +AC_CHECK_LIB([boost_json], [main], [], [AC_MSG_ERROR([Could not find Boost.JSON library])]) dnl dnl Android NDK restrictions diff --git a/daemon/extension/CommandScript.cpp b/daemon/extension/CommandScript.cpp index 9b096ed66..1f00a5819 100644 --- a/daemon/extension/CommandScript.cpp +++ b/daemon/extension/CommandScript.cpp @@ -41,9 +41,9 @@ bool CommandScriptController::StartScript(const char* scriptName, const char* co scriptController->Start(); - for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts()) + for (const auto script : g_ExtensionManager->GetExtensions()) { - if (FileSystem::SameFilename(scriptName, script.GetName())) + if (strcmp(scriptName, script->GetName()) == 0) { return true; } @@ -56,11 +56,11 @@ void CommandScriptController::Run() ExecuteScriptList(m_script); } -void CommandScriptController::ExecuteScript(ScriptConfig::Script* script) +void CommandScriptController::ExecuteScript(std::shared_ptr script) { PrintMessage(Message::mkInfo, "Executing script %s with command %s", script->GetName(), *m_command); - SetArgs({script->GetLocation()}); + SetArgs({script->GetEntry()}); BString<1024> infoName("script %s with command %s", script->GetName(), *m_command); SetInfoName(infoName); diff --git a/daemon/extension/CommandScript.h b/daemon/extension/CommandScript.h index 676432b07..ed20380db 100644 --- a/daemon/extension/CommandScript.h +++ b/daemon/extension/CommandScript.h @@ -31,7 +31,7 @@ class CommandScriptController : public Thread, public NzbScriptController static bool StartScript(const char* scriptName, const char* command, std::unique_ptr modifiedOptions); protected: - virtual void ExecuteScript(ScriptConfig::Script* script); + virtual void ExecuteScript(std::shared_ptr script); virtual void AddMessage(Message::EKind kind, const char* text); virtual const char* GetOptValue(const char* name, const char* value); diff --git a/daemon/extension/Extension.cpp b/daemon/extension/Extension.cpp new file mode 100644 index 000000000..044fbad22 --- /dev/null +++ b/daemon/extension/Extension.cpp @@ -0,0 +1,436 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include "Xml.h" +#include "Extension.h" + +namespace Extension +{ + void Script::SetEntry(std::string entry) + { + m_entry = std::move(entry); + } + + const char* Script::GetEntry() const + { + return m_entry.c_str(); + } + + void Script::SetLocation(std::string location) + { + m_location = std::move(location); + } + + void Script::SetRootDir(std::string dir) + { + m_rootDir = std::move(dir); + } + + const char* Script::GetRootDir() const + { + return m_rootDir.c_str(); + } + + const char* Script::GetLocation() const + { + return m_location.c_str(); + } + + void Script::SetAuthor(std::string author) + { + m_author = std::move(author); + } + + const char* Script::GetAuthor() const + { + return m_author.c_str(); + } + + void Script::SetHomepage(std::string homepage) + { + m_homepage = std::move(homepage); + } + + const char* Script::GetHomepage() const + { + return m_homepage.c_str(); + } + + void Script::SetVersion(std::string version) + { + m_version = std::move(version); + } + + const char* Script::GetVersion() const + { + return m_version.c_str(); + } + + void Script::SetLicense(std::string license) + { + m_license = std::move(license); + } + + const char* Script::GetLicense() const + { + return m_license.c_str(); + } + + void Script::SetName(std::string name) + { + m_name = std::move(name); + } + + const char* Script::GetName() const + { + return m_name.c_str(); + } + + void Script::SetDisplayName(std::string displayName) + { + m_displayName = std::move(displayName); + } + + const char* Script::GetDisplayName() const + { + return m_displayName.c_str(); + } + + void Script::SetAbout(std::string about) + { + m_about = std::move(about); + } + + const char* Script::GetAbout() const + { + return m_about.c_str(); + } + + void Script::SetDescription(std::vector description) + { + m_description = std::move(description); + }; + + const std::vector& Script::GetDescription() const + { + return m_description; + } + + void Script::SetKind(Kind kind) + { + m_kind = std::move(kind); + }; + + bool Script::GetPostScript() const + { + return m_kind.post; + } + + bool Script::GetScanScript() const + { + return m_kind.scan; + } + + bool Script::GetQueueScript() const + { + return m_kind.queue; + } + bool Script::GetSchedulerScript() const + { + return m_kind.scheduler; + } + bool Script::GetFeedScript() const + { + return m_kind.feed; + } + void Script::SetQueueEvents(std::string queueEvents) + { + m_queueEvents = std::move(queueEvents); + } + + const char* Script::GetQueueEvents() const + { + return m_queueEvents.c_str(); + } + + + void Script::SetTaskTime(std::string taskTime) + { + m_taskTime = std::move(taskTime); + } + + const char* Script::GetTaskTime() const + { + return m_taskTime.c_str(); + } + + void Script::SetOptions(std::vector options) + { + m_options = std::move(options); + } + + void Script::SetRequirements(std::vector requirements) + { + m_requirements = std::move(requirements); + } + + const std::vector& Script::GetRequirements() const + { + return m_requirements; + } + + const std::vector& Script::GetOptions() const + { + return m_options; + } + + void Script::SetCommands(std::vector commands) + { + m_commands = std::move(commands); + } + + const std::vector& Script::GetCommands() const + { + return m_commands; + } + + std::string ToJsonStr(const Script& script) + { + Json::JsonObject json; + Json::JsonArray descriptionJson; + Json::JsonArray requirementsJson; + Json::JsonArray optionsJson; + Json::JsonArray commandsJson; + + json["Entry"] = script.GetEntry(); + json["Location"] = script.GetLocation(); + json["RootDir"] = script.GetRootDir(); + json["Name"] = script.GetName(); + json["DisplayName"] = script.GetDisplayName(); + json["About"] = script.GetAbout(); + json["Author"] = script.GetAuthor(); + json["Homepage"] = script.GetHomepage(); + json["License"] = script.GetLicense(); + json["Version"] = script.GetVersion(); + json["PostScript"] = script.GetPostScript(); + json["ScanScript"] = script.GetScanScript(); + json["QueueScript"] = script.GetQueueScript(); + json["SchedulerScript"] = script.GetSchedulerScript(); + json["FeedScript"] = script.GetFeedScript(); + json["QueueEvents"] = script.GetQueueEvents(); + json["TaskTime"] = script.GetTaskTime(); + + for (const auto& line : script.GetDescription()) + { + descriptionJson.push_back(Json::JsonValue(line)); + } + + for (const auto& line : script.GetRequirements()) + { + requirementsJson.push_back(Json::JsonValue(line)); + } + + for (const auto& option : script.GetOptions()) + { + Json::JsonObject optionJson; + Json::JsonArray descriptionJson; + Json::JsonArray selectJson; + + optionJson["Name"] = option.name; + optionJson["DisplayName"] = option.displayName; + + if (const std::string* val = boost::variant2::get_if(&option.value)) + { + optionJson["Value"] = *val; + } + else if (const double* val = boost::variant2::get_if(&option.value)) + { + optionJson["Value"] = *val; + } + + for (const auto& line : option.description) + { + descriptionJson.push_back(Json::JsonValue(line)); + } + + for (const auto& value : option.select) + { + if (const std::string* val = boost::variant2::get_if(&value)) + { + selectJson.push_back(Json::JsonValue(*val)); + } + else if (const double* val = boost::variant2::get_if(&value)) + { + selectJson.push_back(Json::JsonValue(*val)); + } + } + + optionJson["Description"] = std::move(descriptionJson); + optionJson["Select"] = std::move(selectJson); + optionsJson.push_back(std::move(optionJson)); + } + + for (const auto& command : script.GetCommands()) + { + Json::JsonObject commandJson; + Json::JsonArray descriptionJson; + + commandJson["Name"] = command.name; + commandJson["DisplayName"] = command.displayName; + commandJson["Action"] = command.action; + + + for (const auto& line : command.description) + { + descriptionJson.push_back(Json::JsonValue(line)); + } + + commandJson["Description"] = std::move(descriptionJson); + commandsJson.push_back(std::move(commandJson)); + } + + json["Description"] = std::move(descriptionJson); + json["Requirements"] = std::move(requirementsJson); + json["Options"] = std::move(optionsJson); + json["Commands"] = std::move(commandsJson); + + return Json::Serialize(json); + } + + std::string ToXmlStr(const Script& script) + { + xmlNodePtr rootNode = xmlNewNode(NULL, BAD_CAST "value"); + xmlNodePtr structNode = xmlNewNode(NULL, BAD_CAST "struct"); + + AddNewNode(structNode, "Entry", "string", script.GetEntry()); + AddNewNode(structNode, "Location", "string", script.GetLocation()); + AddNewNode(structNode, "RootDir", "string", script.GetRootDir()); + AddNewNode(structNode, "Name", "string", script.GetName()); + AddNewNode(structNode, "DisplayName", "string", script.GetDisplayName()); + AddNewNode(structNode, "About", "string", script.GetAbout()); + AddNewNode(structNode, "Author", "string", script.GetAuthor()); + AddNewNode(structNode, "Homepage", "string", script.GetHomepage()); + AddNewNode(structNode, "License", "string", script.GetLicense()); + AddNewNode(structNode, "Version", "string", script.GetVersion()); + + AddNewNode(structNode, "PostScript", "boolean", BoolToStr(script.GetPostScript())); + AddNewNode(structNode, "ScanScript", "boolean", BoolToStr(script.GetScanScript())); + AddNewNode(structNode, "QueueScript", "boolean", BoolToStr(script.GetQueueScript())); + AddNewNode(structNode, "SchedulerScript", "boolean", BoolToStr(script.GetSchedulerScript())); + AddNewNode(structNode, "FeedScript", "boolean", BoolToStr(script.GetFeedScript())); + + AddNewNode(structNode, "QueueEvents", "string", script.GetQueueEvents()); + AddNewNode(structNode, "TaskTime", "string", script.GetTaskTime()); + + xmlNodePtr descriptionNode = xmlNewNode(NULL, BAD_CAST "Description"); + for (const std::string& line : script.GetDescription()) + { + AddNewNode(descriptionNode, "Value", "string", line.c_str()); + } + + xmlNodePtr requirementsNode = xmlNewNode(NULL, BAD_CAST "Requirements"); + for (const std::string& line : script.GetRequirements()) + { + AddNewNode(requirementsNode, "Value", "string", line.c_str()); + } + + xmlNodePtr commandsNode = xmlNewNode(NULL, BAD_CAST "Commands"); + for (const ManifestFile::Command& command : script.GetCommands()) + { + AddNewNode(commandsNode, "Name", "string", command.name.c_str()); + AddNewNode(commandsNode, "DisplayName", "string", command.displayName.c_str()); + AddNewNode(commandsNode, "Action", "string", command.action.c_str()); + + xmlNodePtr descriptionNode = xmlNewNode(NULL, BAD_CAST "Description"); + for (const std::string& line : command.description) + { + AddNewNode(descriptionNode, "Value", "string", line.c_str()); + } + xmlAddChild(commandsNode, descriptionNode); + } + + xmlNodePtr optionsNode = xmlNewNode(NULL, BAD_CAST "Options"); + for (const ManifestFile::Option& option : script.GetOptions()) + { + AddNewNode(optionsNode, "Name", "string", option.name.c_str()); + AddNewNode(optionsNode, "DisplayName", "string", option.displayName.c_str()); + + if (const std::string* val = boost::variant2::get_if(&option.value)) + { + AddNewNode(optionsNode, "Value", "string", val->c_str()); + } + else if (const double* val = boost::variant2::get_if(&option.value)) + { + AddNewNode(optionsNode, "Value", "number", std::to_string(*val).c_str()); + } + + xmlNodePtr selectNode = xmlNewNode(NULL, BAD_CAST "Select"); + for (const auto& selectOption : option.select) + { + if (const std::string* val = boost::variant2::get_if(&selectOption)) + { + AddNewNode(selectNode, "Value", "string", val->c_str()); + } + else if (const double* val = boost::variant2::get_if(&selectOption)) + { + AddNewNode(selectNode, "Value", "number", std::to_string(*val).c_str()); + } + } + + xmlNodePtr descriptionNode = xmlNewNode(NULL, BAD_CAST "Description"); + for (const std::string& line : option.description) + { + AddNewNode(descriptionNode, "Value", "string", line.c_str()); + } + + xmlAddChild(optionsNode, descriptionNode); + xmlAddChild(optionsNode, selectNode); + } + + xmlAddChild(structNode, descriptionNode); + xmlAddChild(structNode, requirementsNode); + xmlAddChild(structNode, commandsNode); + xmlAddChild(structNode, optionsNode); + xmlAddChild(rootNode, structNode); + + std::string result = Xml::Serialize(rootNode); + xmlFreeNode(rootNode); + return result; + } + + namespace + { + void AddNewNode(xmlNodePtr rootNode, const char* name, const char* type, const char* value) + { + xmlNodePtr memberNode = xmlNewNode(NULL, BAD_CAST "member"); + xmlNodePtr valueNode = xmlNewNode(NULL, BAD_CAST "value"); + xmlNewChild(memberNode, NULL, BAD_CAST "name", BAD_CAST name); + xmlNewChild(valueNode, NULL, BAD_CAST type, BAD_CAST value); + xmlAddChild(memberNode, valueNode); + xmlAddChild(rootNode, memberNode); + } + + const char* BoolToStr(bool value) + { + return value ? "true" : "false"; + } + } +} diff --git a/daemon/extension/Extension.h b/daemon/extension/Extension.h new file mode 100644 index 000000000..6e3e7ef11 --- /dev/null +++ b/daemon/extension/Extension.h @@ -0,0 +1,117 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EXTENSION_H +#define EXTENSION_H + +#include "ManifestFile.h" + +namespace Extension +{ + struct Kind + { + bool post = false; + bool scan = false; + bool queue = false; + bool scheduler = false; + bool feed = false; + }; + + class Script + { + public: + Script() = default; + ~Script() = default; + + Script& operator=(const Script&) = delete; + Script(const Script&) = delete; + + Script(Script&& other) = default; + Script& operator=(Script&& other) = default; + + void SetEntry(std::string entry); + const char* GetEntry() const; + void SetLocation(std::string location); + const char* GetLocation() const; + void SetRootDir(std::string dir); + const char* GetRootDir() const; + void SetAuthor(std::string author); + const char* GetAuthor() const; + void SetVersion(std::string version); + const char* GetVersion() const; + void SetLicense(std::string license); + const char* GetLicense() const; + void SetHomepage(std::string homepage); + const char* GetHomepage() const; + void SetName(std::string name); + const char* GetName() const; + void SetDisplayName(std::string displayName); + const char* GetDisplayName() const; + void SetAbout(std::string about); + const char* GetAbout() const; + void SetDescription(std::vector description); + const std::vector& GetDescription() const; + void SetKind(Kind kind); + bool GetPostScript() const; + bool GetScanScript() const; + bool GetQueueScript() const; + bool GetSchedulerScript() const; + bool GetFeedScript() const; + void SetQueueEvents(std::string queueEvents); + const char* GetQueueEvents() const; + void SetTaskTime(std::string taskTime); + const char* GetTaskTime() const; + void SetRequirements(std::vector requirements); + const std::vector& GetRequirements() const; + void SetOptions(std::vector options); + const std::vector& GetOptions() const; + void SetCommands(std::vector commands); + const std::vector& GetCommands() const; + + private: + Kind m_kind; + std::string m_entry; + std::string m_location; + std::string m_rootDir; + std::string m_author; + std::string m_version; + std::string m_homepage; + std::string m_license; + std::string m_name; + std::string m_displayName; + std::string m_about; + std::string m_queueEvents; + std::string m_taskTime; + std::vector m_description; + std::vector m_requirements; + std::vector m_options; + std::vector m_commands; + }; + + std::string ToJsonStr(const Script& script); + std::string ToXmlStr(const Script& script); + + namespace + { + void AddNewNode(xmlNodePtr rootNode, const char* name, const char* type, const char* value); + const char* BoolToStr(bool value); + } +} + +#endif diff --git a/daemon/extension/ExtensionLoader.cpp b/daemon/extension/ExtensionLoader.cpp new file mode 100644 index 000000000..804d2c1a5 --- /dev/null +++ b/daemon/extension/ExtensionLoader.cpp @@ -0,0 +1,422 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include +#include +#include "ExtensionLoader.h" +#include "ManifestFile.h" +#include "ScriptConfig.h" +#include "FileSystem.h" + +namespace ExtensionLoader +{ + const char* BEGIN_SCRIPT_SIGNATURE = "### NZBGET "; + const char* BEGIN_SCRIPT_COMMANDS_AND_OTPIONS = "### OPTIONS "; + const char* POST_SCRIPT_SIGNATURE = "POST-PROCESSING"; + const char* SCAN_SCRIPT_SIGNATURE = "SCAN"; + const char* QUEUE_SCRIPT_SIGNATURE = "QUEUE"; + const char* SCHEDULER_SCRIPT_SIGNATURE = "SCHEDULER"; + const char* FEED_SCRIPT_SIGNATURE = "FEED"; + const char* END_SCRIPT_SIGNATURE = " SCRIPT"; + const char* QUEUE_EVENTS_SIGNATURE = "### QUEUE EVENTS:"; + const char* TASK_TIME_SIGNATURE = "### TASK TIME:"; + const char* DEFINITION_SIGNATURE = "###"; + + const int BEGIN_SCRIPT_COMMANDS_AND_OTPIONS_LEN = strlen(BEGIN_SCRIPT_COMMANDS_AND_OTPIONS); + const int BEGIN_SINGNATURE_LEN = strlen(BEGIN_SCRIPT_SIGNATURE); + const int QUEUE_EVENTS_SIGNATURE_LEN = strlen(QUEUE_EVENTS_SIGNATURE); + const int TASK_TIME_SIGNATURE_LEN = strlen(TASK_TIME_SIGNATURE); + const int DEFINITION_SIGNATURE_LEN = strlen(DEFINITION_SIGNATURE); + + namespace V1 + { + bool Load(Extension::Script& script, const char* location, const char* rootDir) + { + std::ifstream file(script.GetEntry()); + if (!file.is_open()) + { + return false; + } + + std::string queueEvents; + std::string taskTime; + std::string about; + + Extension::Kind kind; + std::vector description; + std::vector requirements; + std::vector options; + std::vector commands; + + bool inBeforeConfig = false; + bool inConfig = false; + bool inAbout = false; + bool inDescription = false; + + // Declarations "QUEUE EVENT:" and "TASK TIME:" can be placed: + // - in script definition body (between opening and closing script signatures); + // - immediately before script definition (before opening script signature); + // - immediately after script definition (after closing script signature). + // The last two pissibilities are provided to increase compatibility of scripts with older + // nzbget versions which do not expect the extra declarations in the script defintion body. + std::string line; + while (std::getline(file, line)) + { + if (line.empty()) + { + continue; + } + if (!inBeforeConfig && !strncmp(line.c_str(), DEFINITION_SIGNATURE, DEFINITION_SIGNATURE_LEN)) + { + inBeforeConfig = true; + } + if (!inBeforeConfig && !inConfig) + { + continue; + } + + // if TASK TIME, e.g. ### TASK TIME: *;*:00;*:30 ### + if (!strncmp(line.c_str(), TASK_TIME_SIGNATURE, TASK_TIME_SIGNATURE_LEN)) + { + taskTime = line.substr(TASK_TIME_SIGNATURE_LEN + 1); + RemoveTailAndTrim(taskTime, "###"); + continue; + } + + if (!strncmp(line.c_str(), BEGIN_SCRIPT_SIGNATURE, BEGIN_SINGNATURE_LEN) && strstr(line.c_str(), END_SCRIPT_SIGNATURE)) + { + if (inConfig) + { + break; + } + inBeforeConfig = false; + inConfig = true; + kind = GetScriptKind(line.c_str()); + continue; + } + + // if QUEUE EVENTS, e.g. ### QUEUE EVENTS: NZB_ADDED, NZB_DOWNLOADED ### + if (!strncmp(line.c_str(), QUEUE_EVENTS_SIGNATURE, QUEUE_EVENTS_SIGNATURE_LEN)) + { + queueEvents = line.substr(QUEUE_EVENTS_SIGNATURE_LEN + 1); + RemoveTailAndTrim(queueEvents, "###"); + continue; + } + + // if about, e.g. # Sort movies and tv shows. + if (inConfig && !strncmp(line.c_str(), "# ", 2) && !inDescription) + { + inAbout = true; + about.append(line.substr(2)).push_back('\n'); + continue; + } + + if (inConfig && !strncmp(line.c_str(), "#", 1) && inAbout) + { + inAbout = false; + inDescription = true; + continue; + } + + // if requirements: e.g. NOTE: This script requires Python to be installed on your system. + if (inConfig && !strncmp(line.c_str(), "# NOTE: ", 8) && inDescription) + { + requirements.push_back(line.substr(strlen("# NOTE: "))); + continue; + } + + // if description: e.g # This is a script for downloaded TV shows and movies... + if (inConfig && !strncmp(line.c_str(), "# ", 2) && inDescription) + { + description.push_back(line.substr(2)); + continue; + } + + if (!strncmp(line.c_str(), BEGIN_SCRIPT_COMMANDS_AND_OTPIONS, BEGIN_SCRIPT_COMMANDS_AND_OTPIONS_LEN)) + { + ParseOptionsAndCommands(file, options, commands); + break; + } + } + + if (!(kind.post || kind.scan || kind.queue || kind.scheduler || kind.feed)) + { + return false; + } + + BuildDisplayName(script); + Util::TrimRight(about); + + requirements.shrink_to_fit(); + options.shrink_to_fit(); + commands.shrink_to_fit(); + + script.SetLocation(location); + script.SetRootDir(rootDir); + script.SetRequirements(std::move(requirements)); + script.SetKind(std::move(kind)); + script.SetQueueEvents(std::move(queueEvents)); + script.SetAbout(std::move(about)); + script.SetDescription(std::move(description)); + script.SetTaskTime(std::move(taskTime)); + script.SetOptions(std::move(options)); + script.SetCommands(std::move(commands)); + + return true; + } + namespace + { + void RemoveTailAndTrim(std::string& str, const char* tail) + { + size_t tailIdx = str.find(tail); + if (tailIdx != std::string::npos) + { + str = str.substr(0, tailIdx); + } + Util::TrimRight(str); + } + + void BuildDisplayName(Extension::Script& script) + { + BString<1024> shortName = script.GetName(); + if (char* ext = strrchr(shortName, '.')) *ext = '\0'; // strip file extension + + const char* displayName = FileSystem::BaseFileName(shortName); + + script.SetDisplayName(displayName); + } + + void ParseOptionsAndCommands( + std::ifstream& file, + std::vector& options, + std::vector& commands) + { + std::vector description; + std::string line; + while (getline(file, line)) + { + if (strstr(line.c_str(), END_SCRIPT_SIGNATURE)) + { + break; + } + if (line.empty()) + { + continue; + } + + size_t selectStartIdx = line.find("("); + size_t selectEndIdx = line.find(")"); + bool hasSelectOptions = selectStartIdx != std::string::npos + && selectEndIdx != std::string::npos + && selectEndIdx != std::string::npos + && !strncmp(line.c_str(), "# ", 2); + + // e.g. # When to send the message (Always, OnFailure) or # SMTP server port (1-65535). + if (hasSelectOptions) + { + std::string comma = ", "; + std::string dash = "-"; + bool foundComma = line.substr(selectStartIdx, selectEndIdx).find(comma) != std::string::npos; + bool foundDash = line.substr(selectStartIdx, selectEndIdx).find(dash) != std::string::npos; + if (!foundComma && !foundDash || !description.empty()) + { + description.push_back(line.substr(2)); + continue; + } + + std::string selectOptionsStr = line.substr(selectStartIdx + 1, selectEndIdx - selectStartIdx - 1); + + if (foundComma && !CheckCommaAfterEachWord(selectOptionsStr)) + { + description.push_back(line.substr(2)); + continue; + } + if (foundDash) + { + description.push_back(line.substr(2)); + } + else + { + description.push_back(line.substr(2, selectStartIdx - 3) + "."); + } + + ManifestFile::Option option; + std::vector selectOpts; + std::string delimiter = foundDash ? dash : comma; + ParseSelectOptions(selectOptionsStr, delimiter, selectOpts, foundDash); + option.select = std::move(selectOpts); + + while (std::getline(file, line)) + { + if (line == "#") + { + continue; + } + if (!strncmp(line.c_str(), "# ", 2)) + { + line = line.substr(2); + Util::TrimRight(line); + if (!line.empty()) + { + description.push_back(std::move(line)); + } + continue; + } + + if (strncmp(line.c_str(), "# ", 2)) + { + size_t delimPos = line.find("="); + if (delimPos != std::string::npos) + { + std::string name = line.substr(1, delimPos - 1); + std::string value = line.substr(delimPos + 1); + option.value = std::move(GetSelectOpt(value, foundDash)); + option.name = std::move(name); + option.displayName = option.name; + } + option.description = std::move(description); + options.push_back(std::move(option)); + break; + } + } + continue; + } + if (strncmp(line.c_str(), "# ", 2)) + { + // if command, e.g. #ConnectionTest@Send Test E-Mail + size_t eqPos = line.find("="); + size_t atPos = line.find("@"); + if (atPos != std::string::npos && eqPos == std::string::npos) + { + ManifestFile::Command command; + std::string name = line.substr(1, atPos - 1); + std::string action = line.substr(atPos + 1); + command.action = std::move(action); + command.name = std::move(name); + command.description = std::move(description); + command.displayName = command.name; + commands.push_back(std::move(command)); + } + else + { + if (eqPos != std::string::npos) + { + // if option, e.g. #Username=myaccount + ManifestFile::Option option; + std::string name = line.substr(1, eqPos - 1); + std::string value = line.substr(eqPos + 1); + option.value = std::move(value); + option.name = std::move(name); + option.description = std::move(description); + option.displayName = option.name; + options.push_back(std::move(option)); + } + } + } + else + { + description.push_back(line.substr(2)); + } + } + } + + void ParseSelectOptions( + std::string& line, + const std::string& delimiter, + std::vector& selectOpts, + bool isDash) + { + size_t pos = 0; + while ((pos = line.find(delimiter)) != std::string::npos) + { + std::string selectOpt = line.substr(0, pos); + selectOpts.push_back(std::move(GetSelectOpt(selectOpt, isDash))); + line.erase(0, pos + delimiter.length()); + } + selectOpts.push_back(std::move(GetSelectOpt(line, isDash))); + } + + ManifestFile::SelectOption GetSelectOpt(const std::string& val, bool isNumber) + { + if (isNumber) + return ManifestFile::SelectOption(std::stod(val)); + + return ManifestFile::SelectOption(val); + } + + bool CheckCommaAfterEachWord(const std::string& sentence) + { + std::stringstream ss(sentence); + std::string word; + + while (ss >> word) { + if (!word.empty() && word.back() != ',' && !ss.eof()) + { + return false; + } + } + + return true; + } + } + } + + bool V2::Load(Extension::Script& script, const char* location, const char* rootDir) + { + ManifestFile::Manifest manifest; + if (!ManifestFile::Load(manifest, location)) + return false; + + std::string entry = std::string(location) + PATH_SEPARATOR + manifest.main; + script.SetEntry(std::move(entry)); + script.SetLocation(location); + script.SetRootDir(rootDir); + script.SetAuthor(std::move(manifest.author)); + script.SetHomepage(std::move(manifest.homepage)); + script.SetLicense(std::move(manifest.license)); + script.SetVersion(std::move(manifest.version)); + script.SetDisplayName(std::move(manifest.displayName)); + script.SetName(std::move(manifest.name)); + script.SetAbout(std::move(manifest.about)); + script.SetDescription(std::move(manifest.description)); + script.SetKind(GetScriptKind(manifest.kind)); + script.SetQueueEvents(std::move(manifest.queueEvents)); + script.SetTaskTime(std::move(manifest.taskTime)); + script.SetRequirements(std::move(manifest.requirements)); + script.SetCommands(std::move(manifest.commands)); + script.SetOptions(std::move(manifest.options)); + return true; + } + + namespace + { + Extension::Kind GetScriptKind(const std::string& line) + { + Extension::Kind kind; + kind.post = strstr(line.c_str(), POST_SCRIPT_SIGNATURE) != nullptr; + kind.scan = strstr(line.c_str(), SCAN_SCRIPT_SIGNATURE) != nullptr; + kind.queue = strstr(line.c_str(), QUEUE_SCRIPT_SIGNATURE) != nullptr; + kind.scheduler = strstr(line.c_str(), SCHEDULER_SCRIPT_SIGNATURE) != nullptr; + kind.feed = strstr(line.c_str(), FEED_SCRIPT_SIGNATURE) != nullptr; + return kind; + } + } +} diff --git a/daemon/extension/ExtensionLoader.h b/daemon/extension/ExtensionLoader.h new file mode 100644 index 000000000..49ff4b31a --- /dev/null +++ b/daemon/extension/ExtensionLoader.h @@ -0,0 +1,81 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EXTENSION_LOADER_H +#define EXTENSION_LOADER_H + +#include +#include "Extension.h" +#include "Util.h" + +namespace ExtensionLoader +{ + extern const char* BEGIN_SCRIPT_SIGNATURE; + extern const char* BEGIN_SCRIPT_COMMANDS_AND_OTPIONS; + extern const char* POST_SCRIPT_SIGNATURE; + extern const char* SCAN_SCRIPT_SIGNATURE; + extern const char* QUEUE_SCRIPT_SIGNATURE; + extern const char* SCHEDULER_SCRIPT_SIGNATURE; + extern const char* FEED_SCRIPT_SIGNATURE; + extern const char* END_SCRIPT_SIGNATURE; + extern const char* QUEUE_EVENTS_SIGNATURE; + extern const char* TASK_TIME_SIGNATURE; + extern const char* DEFINITION_SIGNATURE; + + extern const int BEGIN_SCRIPT_COMMANDS_AND_OTPIONS_LEN; + extern const int BEGIN_SINGNATURE_LEN; + extern const int QUEUE_EVENTS_SIGNATURE_LEN; + extern const int TASK_TIME_SIGNATURE_LEN; + extern const int DEFINITION_SIGNATURE_LEN; + + namespace V1 + { + bool Load(Extension::Script& script, const char* location, const char* rootDir); + namespace + { + void ParseOptionsAndCommands( + std::ifstream& file, + std::vector& options, + std::vector& commands + ); + void ParseSelectOptions( + std::string& line, + const std::string& delimiter, + std::vector& selectOpts, + bool isDash + ); + ManifestFile::SelectOption GetSelectOpt(const std::string& val, bool isNumber); + void RemoveTailAndTrim(std::string& str, const char* tail); + void BuildDisplayName(Extension::Script& script); + bool CheckCommaAfterEachWord(const std::string& sentence); + } + } + + namespace V2 + { + bool Load(Extension::Script& script, const char* location, const char* rootDir); + } + + namespace + { + Extension::Kind GetScriptKind(const std::string& line); + } +} + +#endif diff --git a/daemon/extension/ExtensionManager.cpp b/daemon/extension/ExtensionManager.cpp new file mode 100644 index 000000000..2fbebc3b2 --- /dev/null +++ b/daemon/extension/ExtensionManager.cpp @@ -0,0 +1,373 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include +#include "Util.h" +#include "FileSystem.h" +#include "NString.h" +#include "Unpack.h" +#include "ExtensionLoader.h" +#include "ExtensionManager.h" + +namespace ExtensionManager +{ + const Extensions& Manager::GetExtensions() const + { + std::shared_lock lock{m_mutex}; + return m_extensions; + } + + std::pair + Manager::DownloadExtension(const std::string& url, const std::string& extName) + { + BString<1024> tmpFileName; + tmpFileName.Format("%s%cextension-%s.tmp.zip", g_Options->GetTempDir(), PATH_SEPARATOR, extName.c_str()); + + std::unique_ptr downloader = std::make_unique(); + downloader->SetUrl(url.c_str()); + downloader->SetForce(true); + downloader->SetRetry(false); + downloader->SetOutputFilename(tmpFileName); + downloader->SetInfoName(extName.c_str()); + + WebDownloader::EStatus status = downloader->DownloadWithRedirects(5); + downloader.reset(); + + return std::pair(status, tmpFileName.Str()); + } + + boost::optional + Manager::UpdateExtension(const std::string& filename, const std::string& extName) + { + std::unique_lock lock{m_mutex}; + + auto extensionIt = GetByName(extName); + if (extensionIt == std::end(m_extensions)) + { + return "Failed to find " + extName; + } + + if (extensionIt->use_count() > 1) + { + return "Failed to update: " + filename + " is executing"; + } + + const auto deleteExtError = DeleteExtension(*(*extensionIt)); + if (deleteExtError) + { + if (!FileSystem::DeleteFile(filename.c_str())) + { + return "Failed to delete extension and temp file: " + filename; + } + + return deleteExtError; + } + + const auto installExtError = InstallExtension(filename, (*extensionIt)->GetRootDir()); + if (installExtError) + { + return installExtError; + } + + m_extensions.erase(extensionIt); + return boost::none; + } + + boost::optional + Manager::InstallExtension(const std::string& filename, const std::string& dest) + { + if (Util::EmptyStr(g_Options->GetSevenZipCmd())) + { + + return std::string("\"SevenZipCmd\" is not specified"); + } + + UnpackController unpacker; + std::string outputDir = "-o" + dest; + UnpackController::ArgList args = { + g_Options->GetSevenZipCmd(), + "x", + filename.c_str(), + outputDir.c_str(), + "-y", + }; + unpacker.SetArgs(std::move(args)); + + int res = unpacker.Execute(); + + if (res != 0) + { + if (!FileSystem::DeleteFile(filename.c_str())) + { + return "Failed to unpack and delete temp file: " + filename; + } + + return "Failed to unpack " + filename; + } + + if (!FileSystem::DeleteFile(filename.c_str())) + { + return "Failed to delete temp file: " + filename; + } + + return boost::none; + } + + boost::optional + Manager::DeleteExtension(const std::string& name) + { + std::unique_lock lock{m_mutex}; + + auto extensionIt = GetByName(name); + if (extensionIt == std::end(m_extensions)) + { + return "Failed to find " + name; + } + + if (extensionIt->use_count() > 1) + { + return "Failed to delete: " + name + " is executing"; + } + + const auto err = DeleteExtension(*(*extensionIt)); + if (err) + { + return err; + } + + m_extensions.erase(extensionIt); + return boost::none; + } + + boost::optional + Manager::LoadExtensions() + { + const char* scriptDir = g_Options->GetScriptDir(); + if (Util::EmptyStr(scriptDir)) + { + return std::string("\"ScriptDir\" is not specified"); + } + + std::unique_lock lock{m_mutex}; + + m_extensions.clear(); + + Tokenizer tokDir(scriptDir, ",;"); + while (const char* extensionDir = tokDir.Next()) + { + LoadExtensionDir(extensionDir, false, extensionDir); + } + + Sort(g_Options->GetScriptOrder()); + CreateTasks(); + m_extensions.shrink_to_fit(); + + return boost::none; + } + + boost::optional + Manager::DeleteExtension(const Extension::Script& ext) + { + const char* location = ext.GetLocation(); + + CString err; + if (FileSystem::DirectoryExists(location) && FileSystem::DeleteDirectoryWithContent(location, err)) + { + if (!err.Empty()) + { + return boost::optional(err.Str()); + } + + return boost::none; + } + else if (FileSystem::FileExists(location) && FileSystem::DeleteFile(location)) + { + return boost::none; + } + + return std::string("Failed to delete ") + location; + } + + void Manager::LoadExtensionDir(const char* directory, bool isSubDir, const char* rootDir) + { + Extension::Script extension; + + if (ExtensionLoader::V2::Load(extension, directory, rootDir)) { + if (!Exists(extension.GetName())) + { + m_extensions.emplace_back(std::make_shared(std::move(extension))); + return; + } + } + + DirBrowser dir(directory); + while (const char* filename = dir.Next()) + { + if (filename[0] == '.' || filename[0] == '_') + continue; + + BString<1024> entry("%s%c%s", directory, PATH_SEPARATOR, filename); + if (!FileSystem::DirectoryExists(entry)) + { + std::string name = GetExtensionName(filename); + if (Exists(name)) + { + continue; + } + + const char* location = isSubDir ? directory : *entry; + extension.SetEntry(*entry); + extension.SetName(std::move(name)); + if (ExtensionLoader::V1::Load(extension, location, rootDir)) + { + m_extensions.emplace_back(std::make_shared(std::move(extension))); + } + } + else if (!isSubDir) + { + LoadExtensionDir(entry, true, rootDir); + } + } + } + + void Manager::CreateTasks() const + { + for (const auto extension : m_extensions) + { + if (!extension->GetSchedulerScript() || Util::EmptyStr(extension->GetTaskTime())) + { + continue; + } + Tokenizer tok(g_Options->GetExtensions(), ",;"); + while (const char* scriptName = tok.Next()) + { + if (strcmp(scriptName, extension->GetName()) == 0) + { + g_Options->CreateSchedulerTask( + 0, + extension->GetTaskTime(), + nullptr, + Options::scScript, + extension->GetName() + ); + break; + } + } + } + } + + bool Manager::Exists(const std::string& name) const + { + return GetByName(name) != std::end(m_extensions); + } + + void Manager::Sort(const char* orderStr) + { + auto comparator = [](const auto& a, const auto& b) -> bool + { + return strcmp(a->GetName(), b->GetName()) < 0; + }; + + if (Util::EmptyStr(orderStr)) + { + std::sort( + std::begin(m_extensions), + std::end(m_extensions), + comparator + ); + return; + } + + std::vector order; + Tokenizer tokOrder(orderStr, ",;"); + while (const char* extName = tokOrder.Next()) + { + auto pos = std::find( + std::begin(order), + std::end(order), + extName + ); + + if (pos == std::end(order)) + { + order.push_back(extName); + } + } + + if (order.empty()) + { + std::sort( + std::begin(m_extensions), + std::end(m_extensions), + comparator + ); + return; + } + + size_t count = 0; + for (size_t i = 0; i < order.size(); ++i) + { + const std::string& name = order[i]; + auto it = std::find_if( + std::begin(m_extensions), + std::end(m_extensions), + [&name](const auto& ext) + { + return name == ext->GetName(); + } + ); + if (it != std::end(m_extensions)) + { + std::iter_swap(std::begin(m_extensions) + count, it); + ++count; + } + } + + std::sort( + std::begin(m_extensions) + count, + std::end(m_extensions), + comparator + ); + } + + std::string Manager::GetExtensionName(const std::string& fileName) const + { + size_t lastIdx = fileName.find_last_of("."); + if (lastIdx != std::string::npos) + { + return fileName.substr(0, lastIdx); + } + + return fileName; + } + + Extensions::const_iterator Manager::GetByName(const std::string& name) const + { + return std::find_if( + std::begin(m_extensions), + std::end(m_extensions), + [&name](const auto& ext) + { + return ext->GetName() == name; + } + ); + } +} diff --git a/daemon/extension/ExtensionManager.h b/daemon/extension/ExtensionManager.h new file mode 100644 index 000000000..6a9c63f9a --- /dev/null +++ b/daemon/extension/ExtensionManager.h @@ -0,0 +1,82 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef EXTENSION_MANAGER_H +#define EXTENSION_MANAGER_H + +#include +#include +#include +#include +#include +#include "WebDownloader.h" +#include "Options.h" +#include "Extension.h" + +namespace ExtensionManager +{ + using Extensions = std::vector>; + + class Manager + { + public: + Manager() = default; + ~Manager() = default; + + Manager(const Manager&) = delete; + Manager& operator=(const Manager&) = delete; + + Manager(Manager&&) = delete; + Manager& operator=(Manager&&) = delete; + + boost::optional + InstallExtension(const std::string& filename, const std::string& dest); + + boost::optional + UpdateExtension(const std::string& filename, const std::string& extName); + + boost::optional + DeleteExtension(const std::string& name); + + boost::optional + LoadExtensions(); + + std::pair + DownloadExtension(const std::string& url, const std::string& info); + + const Extensions& GetExtensions() const; + + private: + void LoadExtensionDir(const char* directory, bool isSubDir, const char* rootDir); + void CreateTasks() const; + Extensions::const_iterator GetByName(const std::string& name) const; + bool Exists(const std::string& name) const; + void Sort(const char* order); + std::string GetExtensionName(const std::string& fileName) const; + boost::optional + DeleteExtension(const Extension::Script& ext); + + Extensions m_extensions; + mutable std::shared_timed_mutex m_mutex; + }; +} + +extern ExtensionManager::Manager* g_ExtensionManager; + +#endif diff --git a/daemon/extension/FeedScript.cpp b/daemon/extension/FeedScript.cpp index 03b1477db..be36a8892 100644 --- a/daemon/extension/FeedScript.cpp +++ b/daemon/extension/FeedScript.cpp @@ -40,7 +40,7 @@ void FeedScriptController::ExecuteScripts(const char* feedScript, const char* fe } } -void FeedScriptController::ExecuteScript(ScriptConfig::Script* script) +void FeedScriptController::ExecuteScript(std::shared_ptr script) { if (!script->GetFeedScript()) { @@ -49,7 +49,7 @@ void FeedScriptController::ExecuteScript(ScriptConfig::Script* script) PrintMessage(Message::mkInfo, "Executing feed-script %s for Feed%i", script->GetName(), m_feedId); - SetArgs({script->GetLocation()}); + SetArgs({script->GetEntry()}); BString<1024> infoName("feed-script %s for Feed%i", script->GetName(), m_feedId); SetInfoName(infoName); diff --git a/daemon/extension/FeedScript.h b/daemon/extension/FeedScript.h index 5dc6d0e6c..86a98b226 100644 --- a/daemon/extension/FeedScript.h +++ b/daemon/extension/FeedScript.h @@ -29,7 +29,7 @@ class FeedScriptController : public NzbScriptController static void ExecuteScripts(const char* feedScript, const char* feedFile, int feedId, bool* success); protected: - virtual void ExecuteScript(ScriptConfig::Script* script); + virtual void ExecuteScript(std::shared_ptr script); private: const char* m_feedFile; diff --git a/daemon/extension/ManifestFile.cpp b/daemon/extension/ManifestFile.cpp new file mode 100644 index 000000000..bed19e5ef --- /dev/null +++ b/daemon/extension/ManifestFile.cpp @@ -0,0 +1,232 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include +#include "ManifestFile.h" +#include "Json.h" +#include "FileSystem.h" + +namespace ManifestFile +{ + const char* MANIFEST_FILE = "manifest.json"; + + bool Load(Manifest& manifest, const char* directory) + { + BString<1024> path("%s%c%s", directory, PATH_SEPARATOR, MANIFEST_FILE); + std::ifstream fs(path); + if (!fs.is_open()) + return false; + + Json::ErrorCode ec; + Json::JsonValue jsonValue = Json::Deserialize(fs, ec); + if (ec) + return false; + + Json::JsonObject json = jsonValue.as_object(); + + if (!ValidateAndSet(json, manifest)) + return false; + + return true; + } + + namespace + { + bool ValidateAndSet(const Json::JsonObject& json, Manifest& manifest) + { + if (!CheckKeyAndSet(json, "author", manifest.author)) + return false; + + if (!CheckKeyAndSet(json, "main", manifest.main)) + return false; + + if (!CheckKeyAndSet(json, "homepage", manifest.homepage)) + return false; + + if (!CheckKeyAndSet(json, "about", manifest.about)) + return false; + + if (!CheckKeyAndSet(json, "version", manifest.version)) + return false; + + if (!CheckKeyAndSet(json, "name", manifest.name)) + return false; + + if (!CheckKeyAndSet(json, "displayName", manifest.displayName)) + return false; + + if (!CheckKeyAndSet(json, "kind", manifest.kind)) + return false; + + if (!CheckKeyAndSet(json, "license", manifest.license)) + return false; + + if (!CheckKeyAndSet(json, "taskTime", manifest.taskTime)) + return false; + + if (!CheckKeyAndSet(json, "queueEvents", manifest.queueEvents)) + return false; + + if (!ValidateOptionsAndSet(json, manifest.options)) + return false; + + if (!ValidateCommandsAndSet(json, manifest.commands)) + return false; + + if (!ValidateTxtAndSet(json, manifest.description, "description")) + return false; + + if (!ValidateTxtAndSet(json, manifest.requirements, "requirements")) + return false; + + return true; + } + + bool ValidateCommandsAndSet(const Json::JsonObject& json, std::vector& commands) + { + auto rawCommands = json.if_contains("commands"); + if (!rawCommands || !rawCommands->is_array()) + return false; + + for (auto& value : rawCommands->as_array()) + { + Json::JsonObject cmdJson = value.as_object(); + Command command; + + if (!CheckKeyAndSet(cmdJson, "name", command.name)) + continue; + + if (!CheckKeyAndSet(cmdJson, "displayName", command.displayName)) + continue; + + if (!ValidateTxtAndSet(cmdJson, command.description, "description")) + continue; + + if (!CheckKeyAndSet(cmdJson, "action", command.action)) + continue; + + commands.emplace_back(std::move(command)); + } + + commands.shrink_to_fit(); + + return true; + } + + bool ValidateOptionsAndSet(const Json::JsonObject& json, std::vector