Feature/extension manager (#76)

#### The new nzbget extension system, which makes it easy to download/update/delete extensions with backward compatibility with the old system.

Extensions [master list](https://github.com/nzbgetcom/nzbget-extensions/).

- Changed
  -  RPC request "configtemplates" -  no longer returns script templates, but only the config template.

- Added
  - new RPC requests:
    - "loadextensions" - loads all extensions from {ScriptDIR} and returns an array of structures in JSON/XML formats.
    - "updateextension" - downloads by url and name and installs the extension. Returns 'true' or error response in JSON/XML formats.
    - "deleteextension" - deletes extension by name. Returns 'true' or error response in JSON/XML formats.
    - "downloadextension" - downloads by url and installs the extension. Returns 'true' or error response in JSON/XML formats.
    - "testextension" - tries to find the right executor program for the extension, e.g. Python. Returns 'true' or error response in JSON/XML formats.
  - "EXTENSION MANAGER" section in webui to download/delete/update extensions
  - Boost.Json library to work with JSON
  - more unit tests

- Refactored
  - replaced raw pointers with smart pointers and const refs where possible for memory safty reasons
 
- Updated
  - INSTALALTION.md
 
- Removed
  - testdata_FILES from Makefile.am
  - EMail and Logger scripts

---------
Co-authored-by: phnzb <pavel@nzbget.com>

CMakeLists.txt

@@ -35,6 +35,7 @@ set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")
 find_package(OpenSSL REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(LibXml2 REQUIRED)
+find_package(Boost REQUIRED COMPONENTS json)
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Weverything")

INSTALLATION.md

@@ -1,4 +1,4 @@
-# NZBGet installation      
+# NZBGet installation
 
 This is a short documentation. For more information please
 visit NZBGet home page at 
@@ -62,29 +62,34 @@ NZBGet absolutely needs the following libraries:
 
  - libstdc++     (usually part of compiler)
  - [libxml2](https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home)
+ - [Boost.JSON](https://www.boost.org/doc/libs/1_84_0/libs/json/doc/html/index.html)
+ - [Boost.Optional](https://www.boost.org/doc/libs/1_84_0/libs/optional/doc/html/index.html)
 
 And the following libraries are optional:
 
- - for curses-output-mode (enabled by default):
+  For curses-output-mode (enabled by default):
    - libcurses   (usually part of commercial systems)
      or (better)
    - [libncurses](https://invisible-island.net/ncurses)
 
- - for encrypted connections (TLS/SSL):
+  For encrypted connections (TLS/SSL):
    - [OpenSSL](https://www.openssl.org)
 
      or
    - [GnuTLS](https://gnutls.org)
 
- - for gzip support in web-server and web-client (enabled by default):
+  For gzip support in web-server and web-client (enabled by default):
    - [zlib](https://www.zlib.net/)
 
- - for configuration:
+  For configuration:
    - [autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html)
    - [autoconf](https://www.gnu.org/software/autoconf/)
 
- - for managing package dependencies:
+  For managing package dependencies:
    - [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/)
+
+  For tests:
+   - [Boost.Test](https://www.boost.org/doc/libs/1_84_0/libs/test/doc/html/index.html)
 
 All these libraries are included in modern POSIX distributions and
 should be available as installable packages. Please note that you also 
@@ -219,6 +224,11 @@ Also required are:
    - [Regex](https://regexlib.com/)
    - [Zlib](https://gnuwin32.sourceforge.net/packages/zlib.htm)
    - [libxml2](https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home)
+   - [Boost.JSON](https://www.boost.org/doc/libs/1_84_0/libs/json/doc/html/index.html)
+   - [Boost.Optional](https://www.boost.org/doc/libs/1_84_0/libs/optional/doc/html/index.html)
+
+For tests:
+   - [Boost.Test](https://www.boost.org/doc/libs/1_84_0/libs/test/doc/html/index.html)
 
 We recommend using [vcpkg](https://vcpkg.io/) to install dependencies.
 

Makefile.am

@@ -42,6 +42,14 @@ nzbget_SOURCES = \
 	daemon/extension/SchedulerScript.h \
 	daemon/extension/ScriptConfig.cpp \
 	daemon/extension/ScriptConfig.h \
+	daemon/extension/Extension.cpp \
+	daemon/extension/Extension.h \
+	daemon/extension/ExtensionManager.cpp \
+	daemon/extension/ExtensionManager.h \
+	daemon/extension/ExtensionLoader.cpp \
+	daemon/extension/ExtensionLoader.h \
+	daemon/extension/ManifestFile.cpp \
+	daemon/extension/ManifestFile.h \
 	daemon/feed/FeedCoordinator.cpp \
 	daemon/feed/FeedCoordinator.h \
 	daemon/feed/FeedFile.cpp \
@@ -158,6 +166,10 @@ nzbget_SOURCES = \
 	daemon/util/Service.h \
 	daemon/util/FileSystem.cpp \
 	daemon/util/FileSystem.h \
+	daemon/util/Json.cpp \
+	daemon/util/Json.h \
+	daemon/util/Xml.cpp \
+	daemon/util/Xml.h \
 	daemon/util/Util.cpp \
 	daemon/util/Util.h \
 	daemon/nserv/NServMain.h \
@@ -353,6 +365,8 @@ webui_FILES = \
 	webui/lib/raphael.min.js \
 	webui/lib/elycharts.js \
 	webui/lib/elycharts.min.js \
+	webui/img/house-16.ico \
+	webui/img/download-16.ico \
 	webui/img/icons.png \
 	webui/img/icons-2x.png \
 	webui/img/transmit.gif \
@@ -364,64 +378,13 @@ webui_FILES = \
 	webui/img/favicon-256x256-opaque.png \
 	webui/img/favicon-256x256.png
 
-scripts_FILES = \
-	scripts/EMail.py \
-	scripts/Logger.py
-
-testdata_FILES = \
-	tests/testdata/dupematcher1/testfile.part01.rar \
-	tests/testdata/dupematcher1/testfile.part24.rar \
-	tests/testdata/dupematcher2/testfile.part04.rar \
-	tests/testdata/dupematcher2/testfile.part43.rar \
-	tests/testdata/nzbfile/dotless.nzb \
-	tests/testdata/nzbfile/dotless.txt \
-	tests/testdata/nzbfile/plain.nzb \
-	tests/testdata/nzbfile/plain.txt \
-	tests/testdata/parchecker/crc.txt \
-	tests/testdata/parchecker/testfile.dat \
-	tests/testdata/parchecker/testfile.nfo \
-	tests/testdata/parchecker/testfile.par2 \
-	tests/testdata/parchecker/testfile.vol00+1.PAR2 \
-	tests/testdata/parchecker/testfile.vol01+2.PAR2 \
-	tests/testdata/parchecker/testfile.vol03+3.PAR2 \
-	tests/testdata/parchecker2/crc.txt \
-	tests/testdata/parchecker2/testfile.7z.001 \
-	tests/testdata/parchecker2/testfile.7z.002 \
-	tests/testdata/parchecker2/testfile.7z.003 \
-	tests/testdata/parchecker2/testfile.7z.par2 \
-	tests/testdata/parchecker2/testfile.7z.vol0+1.PAR2 \
-	tests/testdata/parchecker2/testfile.7z.vol1+2.PAR2 \
-	tests/testdata/parchecker2/testfile.7z.vol3+3.PAR2 \
-	tests/testdata/rarrenamer/testfile3.part01.rar \
-	tests/testdata/rarrenamer/testfile3.part02.rar \
-	tests/testdata/rarrenamer/testfile3.part03.rar \
-	tests/testdata/rarrenamer/testfile5.part01.rar \
-	tests/testdata/rarrenamer/testfile5.part02.rar \
-	tests/testdata/rarrenamer/testfile5.part03.rar \
-	tests/testdata/rarrenamer/testfile3oldnam.rar \
-	tests/testdata/rarrenamer/testfile3oldnam.r00 \
-	tests/testdata/rarrenamer/testfile3oldnam.r01 \
-	tests/testdata/rarrenamer/testfile3encdata.part01.rar \
-	tests/testdata/rarrenamer/testfile3encdata.part02.rar \
-	tests/testdata/rarrenamer/testfile3encdata.part03.rar \
-	tests/testdata/rarrenamer/testfile3encnam.part01.rar \
-	tests/testdata/rarrenamer/testfile3encnam.part02.rar \
-	tests/testdata/rarrenamer/testfile3encnam.part03.rar \
-	tests/testdata/rarrenamer/testfile5encdata.part01.rar \
-	tests/testdata/rarrenamer/testfile5encdata.part02.rar \
-	tests/testdata/rarrenamer/testfile5encdata.part03.rar \
-	tests/testdata/rarrenamer/testfile5encnam.part01.rar \
-	tests/testdata/rarrenamer/testfile5encnam.part02.rar \
-	tests/testdata/rarrenamer/testfile5encnam.part03.rar
-
 # Install
 dist_doc_DATA = $(doc_FILES)
 exampleconfdir = $(datadir)/nzbget
 dist_exampleconf_DATA = $(exampleconf_FILES)
 webuidir = $(datadir)/nzbget
 nobase_dist_webui_DATA = $(webui_FILES)
-scriptsdir = $(datadir)/nzbget
-nobase_dist_scripts_SCRIPTS = $(scripts_FILES)
+scriptsdir = $(datadir)/nzbget/scripts
 
 # Note about "sed": 
 # We need to make some changes in installed files.
@@ -434,6 +397,7 @@ nobase_dist_scripts_SCRIPTS = $(scripts_FILES)
 
 # Prepare example configuration file
 install-data-hook:
+	mkdir -p "$(DESTDIR)$(scriptsdir)"
 	rm -f "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp"
 	cp "$(DESTDIR)$(exampleconfdir)/nzbget.conf" "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp"
 	sed 's:^ConfigTemplate=:ConfigTemplate=$(exampleconfdir)/nzbget.conf:' < "$(DESTDIR)$(exampleconfdir)/nzbget.conf.temp" > "$(DESTDIR)$(exampleconfdir)/nzbget.conf"

configure.ac

@@ -31,7 +31,9 @@ AC_DEFINE(HAVE_CONFIG_H, 1, [Define to 1 use config.h])
 AM_MAINTAINER_MODE
 
 m4_include([posix/ax_cxx_compile_stdcxx.m4])
+m4_include([posix/ax_boost_json.m4])
 
+AX_BOOST_JSON
 
 dnl
 dnl Check for programs.
@@ -77,6 +79,7 @@ AC_SEARCH_LIBS([socket], [socket])
 AC_SEARCH_LIBS([inet_addr], [nsl])
 AC_SEARCH_LIBS([hstrerror], [resolv])
 
+AC_CHECK_LIB([boost_json], [main], [], [AC_MSG_ERROR([Could not find Boost.JSON library])])
 
 dnl
 dnl Android NDK restrictions

daemon/extension/CommandScript.cpp

@@ -41,9 +41,9 @@ bool CommandScriptController::StartScript(const char* scriptName, const char* co
 
 	scriptController->Start();
 
-	for (ScriptConfig::Script& script : g_ScriptConfig->GetScripts())
+	for (const auto script : g_ExtensionManager->GetExtensions())
 	{
-		if (FileSystem::SameFilename(scriptName, script.GetName()))
+		if (strcmp(scriptName, script->GetName()) == 0)
 		{
 			return true;
 		}
@@ -56,11 +56,11 @@ void CommandScriptController::Run()
 	ExecuteScriptList(m_script);
 }
 
-void CommandScriptController::ExecuteScript(ScriptConfig::Script* script)
+void CommandScriptController::ExecuteScript(std::shared_ptr<const Extension::Script> script)
 {
 	PrintMessage(Message::mkInfo, "Executing script %s with command %s", script->GetName(), *m_command);
 
-	SetArgs({script->GetLocation()});
+	SetArgs({script->GetEntry()});
 
 	BString<1024> infoName("script %s with command %s", script->GetName(), *m_command);
 	SetInfoName(infoName);

daemon/extension/CommandScript.h

@@ -31,7 +31,7 @@ class CommandScriptController : public Thread, public NzbScriptController
 	static bool StartScript(const char* scriptName, const char* command, std::unique_ptr<Options::OptEntries> modifiedOptions);
 
 protected:
-	virtual void ExecuteScript(ScriptConfig::Script* script);
+	virtual void ExecuteScript(std::shared_ptr<const Extension::Script> script);
 	virtual void AddMessage(Message::EKind kind, const char* text);
 	virtual const char* GetOptValue(const char* name, const char* value);