From eba8d02acc747d9987c078ab76de041ef7983579 Mon Sep 17 00:00:00 2001 From: dnzbk Date: Fri, 29 Nov 2024 16:09:21 +0300 Subject: [PATCH] Improve deobfuscation --- daemon/main/Options.cpp | 7 ++ daemon/main/Options.h | 5 + daemon/postprocess/PostUnpackRenamer.cpp | 144 ++++++++++++++++++++++ daemon/postprocess/PostUnpackRenamer.h | 48 ++++++++ daemon/postprocess/PrePostProcessor.cpp | 17 +++ daemon/queue/Deobfuscation.cpp | 149 +++++++++++++++++++++++ daemon/queue/Deobfuscation.h | 51 ++++++++ daemon/queue/DirectRenamer.cpp | 16 +-- daemon/queue/DownloadInfo.cpp | 2 +- daemon/queue/DownloadInfo.h | 18 ++- daemon/queue/NzbFile.cpp | 119 ++---------------- daemon/queue/QueueCoordinator.cpp | 3 +- daemon/remote/XmlRpc.cpp | 3 +- daemon/sources.cmake | 2 + daemon/util/FileSystem.cpp | 9 ++ daemon/util/FileSystem.h | 1 + docs/api/LISTGROUPS.md | 1 + nzbget.conf | 12 ++ tests/queue/CMakeLists.txt | 3 + tests/queue/DeobfuscationTest.cpp | 108 ++++++++++++++++ tests/queue/NzbFileTest.cpp | 28 +---- tests/queue/main.cpp | 31 +++++ webui/downloads.js | 1 + 23 files changed, 626 insertions(+), 152 deletions(-) create mode 100644 daemon/postprocess/PostUnpackRenamer.cpp create mode 100644 daemon/postprocess/PostUnpackRenamer.h create mode 100644 daemon/queue/Deobfuscation.cpp create mode 100644 daemon/queue/Deobfuscation.h create mode 100644 tests/queue/DeobfuscationTest.cpp create mode 100644 tests/queue/main.cpp diff --git a/daemon/main/Options.cpp b/daemon/main/Options.cpp index 20c2e9934..969b4842d 100644 --- a/daemon/main/Options.cpp +++ b/daemon/main/Options.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -93,6 +94,8 @@ static const char* OPTION_PARSCAN = "ParScan"; static const char* OPTION_PARQUICK = "ParQuick"; static const char* OPTION_POSTSTRATEGY = "PostStrategy"; static const char* OPTION_FILENAMING = "FileNaming"; +static const char* OPTION_RENAMEAFTERUNPACK = "RenameAfterUnpack"; +static const char* OPTION_RENAMEIGNOREEXT = "RenameIgnoreExt"; static const char* OPTION_PARRENAME = "ParRename"; static const char* OPTION_PARBUFFER = "ParBuffer"; static const char* OPTION_PARTHREADS = "ParThreads"; @@ -480,6 +483,8 @@ void Options::InitDefaults() SetOption(OPTION_PARQUICK, "yes"); SetOption(OPTION_POSTSTRATEGY, "sequential"); SetOption(OPTION_FILENAMING, "article"); + SetOption(OPTION_RENAMEAFTERUNPACK, "yes"); + SetOption(OPTION_RENAMEIGNOREEXT, ".zip, .7z, .rar, .par2"); SetOption(OPTION_PARRENAME, "yes"); SetOption(OPTION_PARBUFFER, "16"); SetOption(OPTION_PARTHREADS, "0"); @@ -703,6 +708,7 @@ void Options::InitOptions() m_parIgnoreExt = GetOption(OPTION_PARIGNOREEXT); m_unpackIgnoreExt = GetOption(OPTION_UNPACKIGNOREEXT); m_shellOverride = GetOption(OPTION_SHELLOVERRIDE); + m_renameIgnoreExt = GetOption(OPTION_RENAMEIGNOREEXT); m_downloadRate = ParseIntValue(OPTION_DOWNLOADRATE, 10) * 1024; m_articleTimeout = ParseIntValue(OPTION_ARTICLETIMEOUT, 10); @@ -773,6 +779,7 @@ void Options::InitOptions() m_urlForce = (bool)ParseEnumValue(OPTION_URLFORCE, BoolCount, BoolNames, BoolValues); m_certCheck = (bool)ParseEnumValue(OPTION_CERTCHECK, BoolCount, BoolNames, BoolValues); m_reorderFiles = (bool)ParseEnumValue(OPTION_REORDERFILES, BoolCount, BoolNames, BoolValues); + m_renameAfterUnpack = (bool)ParseEnumValue(OPTION_RENAMEAFTERUNPACK, BoolCount, BoolNames, BoolValues); const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" }; const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses }; diff --git a/daemon/main/Options.h b/daemon/main/Options.h index f9e37ff4f..496d92682 100644 --- a/daemon/main/Options.h +++ b/daemon/main/Options.h @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -294,6 +295,8 @@ class Options bool GetUnpackPauseQueue() { return m_unpackPauseQueue; } const char* GetExtCleanupDisk() { return m_extCleanupDisk; } const char* GetParIgnoreExt() { return m_parIgnoreExt; } + const char* GetRenameIgnoreExt() { return m_renameIgnoreExt; } + bool GetRenameAfterUnpack() { return m_renameAfterUnpack; } const char* GetUnpackIgnoreExt() { return m_unpackIgnoreExt; } int GetFeedHistory() { return m_feedHistory; } bool GetUrlForce() { return m_urlForce; } @@ -444,6 +447,8 @@ class Options int m_dailyQuota = 0; bool m_reorderFiles = false; EFileNaming m_fileNaming = nfArticle; + bool m_renameAfterUnpack = true; + CString m_renameIgnoreExt; int m_downloadRate = 0; // Application mode diff --git a/daemon/postprocess/PostUnpackRenamer.cpp b/daemon/postprocess/PostUnpackRenamer.cpp new file mode 100644 index 000000000..dcaf71452 --- /dev/null +++ b/daemon/postprocess/PostUnpackRenamer.cpp @@ -0,0 +1,144 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include "PostUnpackRenamer.h" +#include "FileSystem.h" +#include "Deobfuscation.h" + +namespace PostUnpackRenamer +{ + void Controller::StartJob(PostInfo* postInfo) + { + Controller* controller = new (std::nothrow) Controller(); + + if (!controller) + { + error("Failed to allocate memory for PostUnpackRenamer::Controller"); + return; + } + + controller->m_postInfo = postInfo; + controller->SetAutoDestroy(false); + + postInfo->SetPostThread(controller); + + controller->Start(); + } + + void Controller::Run() + { + { + GuardedDownloadQueue guard = DownloadQueue::Guard(); + + m_name = m_postInfo->GetNzbInfo()->GetName(); + m_dstDir = m_postInfo->GetNzbInfo()->GetDestDir(); + } + + std::string infoName = "Post-unpack renaming for " + m_name; + SetInfoName(infoName.c_str()); + + if (Deobfuscation::IsExcessivelyObfuscated(m_name)) + { + PrintMessage(Message::mkWarning, + "Skipping Post-unpack renaming. NZB filename %s is excessively obfuscated which makes renaming unreliable.", + m_name.c_str() + ); + m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus( + NzbInfo::PostUnpackRenamingStatus::Skipped + ); + m_postInfo->SetWorking(false); + return; + } + + bool ok = RenameFiles(m_dstDir, m_name); + + GuardedDownloadQueue guard = DownloadQueue::Guard(); + if (ok) + { + PrintMessage(Message::mkInfo, "%s successful", infoName.c_str()); + m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus( + NzbInfo::PostUnpackRenamingStatus::Success + ); + } + else + { + PrintMessage(Message::mkError, "%s failed", infoName.c_str()); + m_postInfo->GetNzbInfo()->SetPostUnpackRenamingStatus( + NzbInfo::PostUnpackRenamingStatus::Failure + ); + } + + m_postInfo->SetWorking(false); + } + + bool Controller::RenameFiles(const std::string& dir, const std::string& newName) + { + DirBrowser dirBrowser(dir.c_str()); + while (const char* fileOrDir = dirBrowser.Next()) + { + std::string srcFileOrDir = dir + PATH_SEPARATOR + fileOrDir; + + if (FileSystem::DirectoryExists(srcFileOrDir.c_str())) + { + RenameFiles(srcFileOrDir, newName); + continue; + } + + if (!Deobfuscation::IsExcessivelyObfuscated(fileOrDir)) + { + PrintMessage(Message::mkInfo, + "Filename %s is not excessively obfuscated, no renaming needed.", + fileOrDir + ); + continue; + } + + std::string dstFile = dir + PATH_SEPARATOR + newName; + dstFile += FileSystem::GetFileExtension(srcFileOrDir).value_or(""); + + if (Util::MatchFileExt(dstFile.c_str(), g_Options->GetRenameIgnoreExt(), ",")) + { + continue; + } + + if (FileSystem::MoveFile(srcFileOrDir.c_str(), dstFile.c_str())) + { + PrintMessage(Message::mkInfo, "%s renamed to %s", srcFileOrDir.c_str(), dstFile.c_str()); + } + else + { + PrintMessage(Message::mkError, + "Could not rename file %s to %s: %s", + srcFileOrDir.c_str(), + dstFile.c_str(), + *FileSystem::GetLastErrorMessage() + ); + } + } + + return true; + } + + void Controller::AddMessage(Message::EKind kind, const char* text) + { + m_postInfo->GetNzbInfo()->AddMessage(kind, text); + } +} diff --git a/daemon/postprocess/PostUnpackRenamer.h b/daemon/postprocess/PostUnpackRenamer.h new file mode 100644 index 000000000..54ed74a8e --- /dev/null +++ b/daemon/postprocess/PostUnpackRenamer.h @@ -0,0 +1,48 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef POST_UNPACK_H +#define POST_UNPACK_H + +#include +#include "Thread.h" +#include "ScriptController.h" +#include "DownloadInfo.h" + +namespace PostUnpackRenamer +{ + class Controller final : public Thread, public ScriptController + { + public: + void Run() override; + static void StartJob(PostInfo* postInfo); + + protected: + void AddMessage(Message::EKind kind, const char* text) override; + + private: + PostInfo* m_postInfo; + std::string m_name; + std::string m_dstDir; + bool RenameFiles(const std::string& dir, const std::string& nameToRename); + }; +} + +#endif diff --git a/daemon/postprocess/PrePostProcessor.cpp b/daemon/postprocess/PrePostProcessor.cpp index 5eab30983..f28226980 100644 --- a/daemon/postprocess/PrePostProcessor.cpp +++ b/daemon/postprocess/PrePostProcessor.cpp @@ -37,6 +37,7 @@ #include "QueueScript.h" #include "ParParser.h" #include "DirectUnpack.h" +#include "PostUnpackRenamer.h" PrePostProcessor::PrePostProcessor() { @@ -844,6 +845,17 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo unpack = false; } + bool postUnpackRenaming = g_Options->GetRenameAfterUnpack() && + nzbInfo->GetPostUnpackRenamingStatus() == NzbInfo::PostUnpackRenamingStatus::None && + nzbInfo->GetDestDir() && + nzbInfo->GetName() && + nzbInfo->GetUnpackStatus() != NzbInfo::usFailure && + nzbInfo->GetUnpackStatus() != NzbInfo::usSpace && + nzbInfo->GetUnpackStatus() != NzbInfo::usPassword && + nzbInfo->GetParStatus() != NzbInfo::psFailure && + nzbInfo->GetParStatus() != NzbInfo::psManual && + nzbInfo->GetMoveStatus() == NzbInfo::msSuccess; + if (unpack) { EnterStage(downloadQueue, postInfo, PostInfo::ptUnpacking); @@ -859,6 +871,11 @@ void PrePostProcessor::StartJob(DownloadQueue* downloadQueue, PostInfo* postInfo EnterStage(downloadQueue, postInfo, PostInfo::ptMoving); MoveController::StartJob(postInfo); } + else if (postUnpackRenaming) + { + EnterStage(downloadQueue, postInfo, PostInfo::ptPostUnpackRenaming); + PostUnpackRenamer::Controller::StartJob(postInfo); + } else { EnterStage(downloadQueue, postInfo, PostInfo::ptExecutingScript); diff --git a/daemon/queue/Deobfuscation.cpp b/daemon/queue/Deobfuscation.cpp new file mode 100644 index 000000000..e07cf3681 --- /dev/null +++ b/daemon/queue/Deobfuscation.cpp @@ -0,0 +1,149 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include "nzbget.h" + +#include "Deobfuscation.h" + +namespace +{ + std::string ParseWithoutQuotes(const std::string& str) noexcept + { + if (str.size() < 8) return str; + + const std::string start = "Re: "; + size_t startPos = str.find(start); + size_t endPos = str.rfind(" ("); + if (endPos == std::string::npos) + { + return str; + } + + if (startPos != std::string::npos) + { + startPos += start.size(); + + size_t distance = endPos - startPos; + if (distance < 1) return str; + + return str.substr(startPos, distance); + } + + return str.substr(0, endPos); + } + + std::string ParseWtfNzb(const std::string& str) noexcept + { + const std::string begin = "[PRiVATE]-[WtFnZb]-"; + size_t beginPos = str.find(begin); + if (beginPos == std::string::npos) return str; + beginPos += begin.size(); + + std::string end = " - \"\""; + size_t endPos = str.rfind(end); + if (endPos == std::string::npos) return str; + + // can be + // [path/something[123].bin]-[1/10] + // or + // [1/10]-[path/something[123].bin] + size_t distance = endPos - beginPos; + std::string middle = str.substr(beginPos, distance); + std::string result; + result.reserve(middle.size()); + + bool foundExtOrWord = false; + int depth = 0; + for (unsigned char ch : middle) + { + if (ch == '[' && ++depth == 1) continue; + if (ch == ']' && --depth == 0) continue; + if (foundExtOrWord && !depth) break; + + if (!foundExtOrWord && !depth) + { + result.clear(); + continue; + } + + if (!depth && ch == '-') continue; + + if (depth == 1 && (ch == '/' || ch == '\\')) + { + result.clear(); + continue; + } + + if (depth == 1 && (ch == '.' || isalpha(ch) != 0)) + foundExtOrWord = true; + + result.push_back(ch); + } + + result.shrink_to_fit(); + + return result; + } +} + +namespace Deobfuscation +{ + bool IsExcessivelyObfuscated(const std::string& str) noexcept + { + if (str.empty()) + return false; + + for (auto& regex : EXCLUDED_HASHED_RELEASES_REGEXES) + { + if (std::regex_search(str, regex)) + return false; + } + + for (auto& regex : HASHED_RELEASES_REGEXES) + { + if (std::regex_search(str, regex)) + return true; + } + + return false; + } + + std::string Deobfuscate(const std::string& str) noexcept + { + if (str.size() < 3) return str; + + size_t firstQuotPos = str.find("\""); + + if (firstQuotPos == std::string::npos) + return ParseWithoutQuotes(str); + + firstQuotPos += 1; + size_t secondQuotPos = str.find("\"", firstQuotPos); + size_t distance = secondQuotPos - firstQuotPos; + + if (distance == 0) + return ParseWtfNzb(str); + + if (distance > 0) + return str.substr(firstQuotPos, distance); + + return str; + } +} diff --git a/daemon/queue/Deobfuscation.h b/daemon/queue/Deobfuscation.h new file mode 100644 index 000000000..35c9344d5 --- /dev/null +++ b/daemon/queue/Deobfuscation.h @@ -0,0 +1,51 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef DEOBFUSCATION_H +#define DEOBFUSCATION_H + +#include +#include +#include + +namespace Deobfuscation +{ + inline const std::array EXCLUDED_HASHED_RELEASES_REGEXES{ + std::regex{ "[0-9a-zA-Z]{24}.(7z(\\.\\d{2,3})?|rar|r\\d{2,3}|zip|par2)$" } + }; + + inline const std::array HASHED_RELEASES_REGEXES{ + std::regex{ "[0-9a-f.]{16}" }, + std::regex{ "^[0-9a-zA-Z]{24,}" }, + std::regex{ "^[a-z0-9]{24}$" }, + std::regex{ "^abc$" }, + std::regex{ "^abc[-_. ]xyz" }, + std::regex{ "^[A-Z]{11,}[0-9]{3}$" }, + std::regex{ "^Backup_[0-9]{5,}S[0-9]{2}-[0-9]{2}$" }, + std::regex{ "^123$" }, + std::regex{ "^b00bs$" }, + std::regex{ "^[0-9]{6}_[0-9]{2}$" }, + }; + + bool IsExcessivelyObfuscated(const std::string& str) noexcept; + std::string Deobfuscate(const std::string& str) noexcept; +} + +#endif diff --git a/daemon/queue/DirectRenamer.cpp b/daemon/queue/DirectRenamer.cpp index 714a5c02a..81bfef22c 100644 --- a/daemon/queue/DirectRenamer.cpp +++ b/daemon/queue/DirectRenamer.cpp @@ -362,39 +362,39 @@ void DirectRenamer::RenameFiles(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, // rename in-progress files for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - CString newName; + std::string newName; if (fileInfo->GetParFile() && renamePars) { - newName = BuildNewParName(fileInfo->GetFilename(), nzbInfo->GetDestDir(), fileInfo->GetParSetId(), vol); + newName = BuildNewParName(fileInfo->GetFilename(), nzbInfo->GetDestDir(), fileInfo->GetParSetId(), vol).Str(); } else if (!fileInfo->GetParFile()) { - newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k()); + newName = BuildNewRegularName(fileInfo->GetFilename(), parHashes, fileInfo->GetHash16k()).Str(); } - if (newName) + if (!newName.empty()) { bool written = fileInfo->GetOutputFilename() && !Util::EndsWith(fileInfo->GetOutputFilename(), ".out.tmp", true); if (!written) { nzbInfo->PrintMessage(Message::mkInfo, "Renaming in-progress file %s to %s", - fileInfo->GetFilename(), *newName); + fileInfo->GetFilename(), newName.c_str()); if (Util::EmptyStr(fileInfo->GetOrigname())) { fileInfo->SetOrigname(fileInfo->GetFilename()); } - fileInfo->SetFilename(newName); + fileInfo->SetFilename(std::move(newName)); fileInfo->SetFilenameConfirmed(true); renamedCount++; } - else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName)) + else if (RenameCompletedFile(nzbInfo, fileInfo->GetFilename(), newName.c_str())) { if (Util::EmptyStr(fileInfo->GetOrigname())) { fileInfo->SetOrigname(fileInfo->GetFilename()); } - fileInfo->SetFilename(newName); + fileInfo->SetFilename(std::move(newName)); fileInfo->SetFilenameConfirmed(true); renamedCount++; } diff --git a/daemon/queue/DownloadInfo.cpp b/daemon/queue/DownloadInfo.cpp index ea0f2746d..03642a178 100644 --- a/daemon/queue/DownloadInfo.cpp +++ b/daemon/queue/DownloadInfo.cpp @@ -821,7 +821,7 @@ void FileInfo::SetExtraPriority(bool extraPriority) void FileInfo::MakeValidFilename() { - m_filename = FileSystem::MakeValidFilename(m_filename); + m_filename = FileSystem::MakeValidFilename(m_filename.c_str()).Str(); } void FileInfo::SetActiveDownloads(int activeDownloads) diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index b10f4d359..38994eea1 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -141,8 +141,8 @@ class FileInfo Groups* GetGroups() { return &m_groups; } const char* GetSubject() { return m_subject; } void SetSubject(const char* subject) { m_subject = subject; } - const char* GetFilename() { return m_filename; } - void SetFilename(const char* filename) { m_filename = filename; } + const char* GetFilename() { return m_filename.c_str(); } + void SetFilename(std::string filename) { m_filename = std::move(filename); } void SetOrigname(const char* origname) { m_origname = origname; } const char* GetOrigname() { return m_origname; } void MakeValidFilename(); @@ -213,7 +213,7 @@ class FileInfo Groups m_groups; ServerStatList m_serverStats; CString m_subject; - CString m_filename; + std::string m_filename; CString m_origname; int64 m_size = 0; int64 m_remainingSize = 0; @@ -418,6 +418,14 @@ class NzbInfo msSuccess }; + enum class PostUnpackRenamingStatus + { + None, + Failure, + Success, + Skipped + }; + enum EDeleteStatus { dsNone, @@ -554,6 +562,8 @@ class NzbInfo ECleanupStatus GetCleanupStatus() { return m_cleanupStatus; } void SetCleanupStatus(ECleanupStatus cleanupStatus) { m_cleanupStatus = cleanupStatus; } EMoveStatus GetMoveStatus() { return m_moveStatus; } + void SetPostUnpackRenamingStatus(PostUnpackRenamingStatus status) { m_postUnpackRenamingStatus = status; } + PostUnpackRenamingStatus GetPostUnpackRenamingStatus() { return m_postUnpackRenamingStatus; } void SetMoveStatus(EMoveStatus moveStatus) { m_moveStatus = moveStatus; } EDeleteStatus GetDeleteStatus() { return m_deleteStatus; } void SetDeleteStatus(EDeleteStatus deleteStatus) { m_deleteStatus = deleteStatus; } @@ -704,6 +714,7 @@ class NzbInfo EPostUnpackStatus m_unpackStatus = usNone; ECleanupStatus m_cleanupStatus = csNone; EMoveStatus m_moveStatus = msNone; + PostUnpackRenamingStatus m_postUnpackRenamingStatus = PostUnpackRenamingStatus::None; EDeleteStatus m_deleteStatus = dsNone; EMarkStatus m_markStatus = ksNone; EUrlStatus m_urlStatus = lsNone; @@ -783,6 +794,7 @@ class PostInfo ptUnpacking, ptCleaningUp, ptMoving, + ptPostUnpackRenaming, ptExecutingScript, ptFinished }; diff --git a/daemon/queue/NzbFile.cpp b/daemon/queue/NzbFile.cpp index 18ec3db90..374f943c9 100644 --- a/daemon/queue/NzbFile.cpp +++ b/daemon/queue/NzbFile.cpp @@ -28,6 +28,7 @@ #include "DiskState.h" #include "Util.h" #include "FileSystem.h" +#include "Deobfuscation.h" NzbFile::NzbFile(const char* fileName, const char* category) : m_fileName(fileName) @@ -116,128 +117,22 @@ void NzbFile::AddFileInfo(std::unique_ptr fileInfo) void NzbFile::ParseSubject(FileInfo* fileInfo, bool TryQuotes) { - // Example subject: some garbage "title" yEnc (10/99) + if (!fileInfo) return; if (!fileInfo->GetSubject()) { // Malformed file element without subject. We generate subject using internal element id. fileInfo->SetSubject(CString::FormatStr("%d", fileInfo->GetId())); + return; } - // strip the "yEnc (10/99)"-suffix - BString<1024> subject = fileInfo->GetSubject(); - char* end = subject + strlen(subject) - 1; - if (*end == ')') - { - end--; - while (strchr("0123456789", *end) && end > subject) end--; - if (*end == '/') - { - end--; - while (strchr("0123456789", *end) && end > subject) end--; - if (end - 6 > subject && !strncmp(end - 6, " yEnc (", 7)) - { - end[-6] = '\0'; - } - } - } - - if (TryQuotes) - { - // try to use the filename in quatation marks - char* p = subject; - char* start = strchr(p, '\"'); - if (start) - { - start++; - char* end = strchr(start + 1, '\"'); - if (end) - { - int len = (int)(end - start); - char* point = strchr(start + 1, '.'); - if (point && point < end) - { - BString<1024> filename; - filename.Set(start, len); - fileInfo->SetFilename(filename); - return; - } - } - } - } - - // tokenize subject, considering spaces as separators and quotation - // marks as non separatable token delimiters. - // then take the last token containing dot (".") as a filename + detail("Extracting a filename from Subject %s", fileInfo->GetSubject()); - typedef std::vector TokenList; - TokenList tokens; + std::string subject = Deobfuscation::Deobfuscate(fileInfo->GetSubject()); - // tokenizing - char* p = subject; - char* start = p; - bool quot = false; - while (true) - { - char ch = *p; - bool sep = (ch == '\"') || (!quot && ch == ' ') || (ch == '\0'); - if (sep) - { - // end of token - int len = (int)(p - start); - if (len > 0) - { - tokens.emplace_back(start, len); - } - start = p; - if (ch != '\"' || quot) - { - start++; - } - quot = *start == '\"'; - if (quot) - { - start++; - char* q = strchr(start, '\"'); - if (q) - { - p = q - 1; - } - else - { - quot = false; - } - } - } - if (ch == '\0') - { - break; - } - p++; - } + detail("Extracted Filename: %s", subject.c_str()); - if (!tokens.empty()) - { - // finding the best candidate for being a filename - char* besttoken = tokens.back(); - for (TokenList::reverse_iterator it = tokens.rbegin(); it != tokens.rend(); it++) - { - char* s = *it; - char* p = strchr(s, '.'); - if (p && (p[1] != '\0')) - { - besttoken = s; - break; - } - } - fileInfo->SetFilename(besttoken); - } - else - { - // subject is empty or contains only separators? - debug("Could not extract Filename from Subject: %s. Using Subject as Filename", fileInfo->GetSubject()); - fileInfo->SetFilename(fileInfo->GetSubject()); - } + fileInfo->SetFilename(std::move(subject)); } bool NzbFile::HasDuplicateFilenames() diff --git a/daemon/queue/QueueCoordinator.cpp b/daemon/queue/QueueCoordinator.cpp index dbe25035f..e5bf47884 100644 --- a/daemon/queue/QueueCoordinator.cpp +++ b/daemon/queue/QueueCoordinator.cpp @@ -31,6 +31,7 @@ #include "FileSystem.h" #include "Decoder.h" #include "StatMeter.h" +#include "Deobfuscation.h" bool QueueCoordinator::CoordinatorDownloadQueue::EditEntry( int ID, EEditAction action, const char* args) @@ -741,7 +742,7 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader) // if the name from article seems to be obfuscated bool useFilenameFromArticle = g_Options->GetFileNaming() == Options::nfArticle || (g_Options->GetFileNaming() == Options::nfAuto && - !Util::AlphaNum(articleDownloader->GetArticleFilename()) && + !Deobfuscation::IsExcessivelyObfuscated(articleDownloader->GetArticleFilename()) && !nzbInfo->GetManyDupeFiles()); if (useFilenameFromArticle) { diff --git a/daemon/remote/XmlRpc.cpp b/daemon/remote/XmlRpc.cpp index 9422f6e50..a28a81b1f 100644 --- a/daemon/remote/XmlRpc.cpp +++ b/daemon/remote/XmlRpc.cpp @@ -2194,7 +2194,8 @@ void ListGroupsXmlCommand::Execute() const char* ListGroupsXmlCommand::DetectStatus(NzbInfo* nzbInfo) { const char* postStageName[] = { "PP_QUEUED", "LOADING_PARS", "VERIFYING_SOURCES", "REPAIRING", - "VERIFYING_REPAIRED", "RENAMING", "RENAMING", "UNPACKING", "MOVING", "MOVING", "EXECUTING_SCRIPT", "PP_FINISHED" }; + "VERIFYING_REPAIRED", "RENAMING", "RENAMING", "UNPACKING", "MOVING", "MOVING", "POST_UNPACK_RENAMING", + "EXECUTING_SCRIPT", "PP_FINISHED" }; const char* status = nullptr; diff --git a/daemon/sources.cmake b/daemon/sources.cmake index d29c767ff..4596ad8c0 100644 --- a/daemon/sources.cmake +++ b/daemon/sources.cmake @@ -64,6 +64,7 @@ set(SRC ${CMAKE_SOURCE_DIR}/daemon/postprocess/Rename.cpp ${CMAKE_SOURCE_DIR}/daemon/postprocess/Repair.cpp ${CMAKE_SOURCE_DIR}/daemon/postprocess/Unpack.cpp + ${CMAKE_SOURCE_DIR}/daemon/postprocess/PostUnpackRenamer.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DirectRenamer.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp @@ -75,6 +76,7 @@ set(SRC ${CMAKE_SOURCE_DIR}/daemon/queue/QueueEditor.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/Scanner.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/UrlCoordinator.cpp + ${CMAKE_SOURCE_DIR}/daemon/queue/Deobfuscation.cpp ${CMAKE_SOURCE_DIR}/daemon/remote/BinRpc.cpp ${CMAKE_SOURCE_DIR}/daemon/remote/RemoteClient.cpp diff --git a/daemon/util/FileSystem.cpp b/daemon/util/FileSystem.cpp index 4f8a92ba4..f1248ea15 100644 --- a/daemon/util/FileSystem.cpp +++ b/daemon/util/FileSystem.cpp @@ -673,6 +673,15 @@ std::string FileSystem::EscapePathForShell(const std::string& path) return "\"" + path + "\""; } +std::optional FileSystem::GetFileExtension(const std::string& filename) +{ + size_t extIdx = filename.rfind("."); + if (extIdx == std::string::npos) + return std::nullopt; + + return filename.substr(extIdx); +} + /* Delete directory which is empty or contains only hidden files or directories (whose names start with dot) */ bool FileSystem::DeleteDirectory(const char* dirFilename) { diff --git a/daemon/util/FileSystem.h b/daemon/util/FileSystem.h index 8d4b783b1..1ad806042 100644 --- a/daemon/util/FileSystem.h +++ b/daemon/util/FileSystem.h @@ -56,6 +56,7 @@ class FileSystem static bool CreateDirectory(const char* dirFilename); static std::string ExtractFilePathFromCmd(const std::string& path); static std::string EscapePathForShell(const std::string& path); + static std::optional GetFileExtension(const std::string& filename); /* Delete empty directory */ static bool RemoveDirectory(const char* dirFilename); diff --git a/docs/api/LISTGROUPS.md b/docs/api/LISTGROUPS.md index 2f5c7b3e8..4f8b9c700 100644 --- a/docs/api/LISTGROUPS.md +++ b/docs/api/LISTGROUPS.md @@ -55,6 +55,7 @@ This method returns array of structures with following fields: - **RENAMING** - processed by par-renamer; - **UNPACKING** - being unpacked; - **MOVING** - moving files from intermediate directory into destination directory; + - **POST_UNPACK_RENAMING** - renaming excessively obfuscated downloaded files after unpacking; - **EXECUTING_SCRIPT** - executing post-processing script; - **PP_FINISHED** - post-processing is finished, the item is about to be moved to history. - **TotalArticles** `(int)` - Total number of articles in all files of the group. diff --git a/nzbget.conf b/nzbget.conf index 73bcf0243..a62974337 100644 --- a/nzbget.conf +++ b/nzbget.conf @@ -883,6 +883,18 @@ WriteBuffer=0 # effect on files extracted from archives. FileNaming=auto +# Rename downloaded files after unpack (yes, no). +# +# This option renames the extracted and obfuscated files using the NZB filename. +RenameAfterUnpack=yes + +# File extensions to ignore during renaming after unpack. +# +# Example: zip, .7z, .rar, .par2. +# +# NOTE: Only needed if is enabled. +RenameIgnoreExt=.zip, .7z, .rar, .par2 + # Reorder files within nzbs for optimal download order (yes, no). # # When nzb-file is added to queue the files listed within nzb can be in a random diff --git a/tests/queue/CMakeLists.txt b/tests/queue/CMakeLists.txt index 6a7683a61..68fe44567 100644 --- a/tests/queue/CMakeLists.txt +++ b/tests/queue/CMakeLists.txt @@ -1,8 +1,11 @@ set(QueueTestsSrc + main.cpp NzbFileTest.cpp + DeobfuscationTest.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/NzbFile.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DownloadInfo.cpp + ${CMAKE_SOURCE_DIR}/daemon/queue/Deobfuscation.cpp ${CMAKE_SOURCE_DIR}/daemon/feed/FeedInfo.cpp ${CMAKE_SOURCE_DIR}/daemon/main/Options.cpp ${CMAKE_SOURCE_DIR}/daemon/util/NString.cpp diff --git a/tests/queue/DeobfuscationTest.cpp b/tests/queue/DeobfuscationTest.cpp new file mode 100644 index 000000000..f8941817d --- /dev/null +++ b/tests/queue/DeobfuscationTest.cpp @@ -0,0 +1,108 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2023-2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include + +#include "Deobfuscation.h" + +using namespace Deobfuscation; + +BOOST_AUTO_TEST_CASE(IsExcessivelyObfuscatedTest) +{ + BOOST_CHECK(IsExcessivelyObfuscated("2c0837e5fa42c8cfb5d5e583168a2af4.10")); + BOOST_CHECK(IsExcessivelyObfuscated("5KzdcWdGVGUG83Q9jv8KXht4O2k57w.mkv")); + BOOST_CHECK(IsExcessivelyObfuscated("2c0837e5fa42c8cfb5d5e583168a2af4.mkv")); + BOOST_CHECK(IsExcessivelyObfuscated("a4c7d1f239b71a.a1c0a8b1790e65c9430d5a601037a4.7893")); + BOOST_CHECK(IsExcessivelyObfuscated("a1b2c3d4e5f678.901234567890abcdef01234567890123.4567")); + BOOST_CHECK(IsExcessivelyObfuscated("abc.xyz.a1b2c3d4e5f678.mkv")); + BOOST_CHECK(IsExcessivelyObfuscated("b00bs.a1b2c3d4e5f678.mkv")); + BOOST_CHECK(IsExcessivelyObfuscated("Not.obfuscated.rar") == false); + BOOST_CHECK(IsExcessivelyObfuscated("a1b2c3d4e5f678.901234567890abcdef01234567890123.rar") == false); + BOOST_CHECK(IsExcessivelyObfuscated("a1b2c3d4e5f678.901234567890abcdef01234567890123.r00") == false); + BOOST_CHECK(IsExcessivelyObfuscated("2fpJZyw12WSJz8JunjkxpZcw0XIZKKMP.7z.15") == false); + BOOST_CHECK(IsExcessivelyObfuscated("2fpJZyw12WSJz8JunjkxpZcw0XIZKKMP.7z.015") == false); + BOOST_CHECK(IsExcessivelyObfuscated("a1b2c3d4e5f678.901234567890abcdef01234567890123.zip") == false); + BOOST_CHECK(IsExcessivelyObfuscated("a1b2c3d4e5f678.901234567890abcdef01234567890123.par2") == false); +} + +BOOST_AUTO_TEST_CASE(DeobfuscationTest) +{ + BOOST_CHECK_EQUAL(Deobfuscate(""), ""); + BOOST_CHECK_EQUAL(Deobfuscate("\"A\""), "A"); + BOOST_CHECK_EQUAL(Deobfuscate("Not obfuscated"), "Not obfuscated"); + + BOOST_CHECK_EQUAL( + Deobfuscate("Any.Show.2024.S01E01.Die.verborgene.Hand.GERMAN.5.1.DL.EAC3.2160p.WEB-DL.DV.HDR.x265-TvR.vol127+128.par2 (1/0)"), + "Any.Show.2024.S01E01.Die.verborgene.Hand.GERMAN.5.1.DL.EAC3.2160p.WEB-DL.DV.HDR.x265-TvR.vol127+128.par2" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[setup_app_-_reforced_161554.339115__54385_-1.bin]-[1/10] - \"\" yEnc 4288754174 (1/8377)"), + "setup_app_-_reforced_161554.339115__54385_-1.bin" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[1/series/Any.Show.S01E01.Pilot.1080p.DSNP.WEBRip.DDP.5.1.H.265.-EDGE2020.mkv]-[1/7] - \"\" yEnc 225628476 (1/315)"), + "Any.Show.S01E01.Pilot.1080p.DSNP.WEBRip.DDP.5.1.H.265.-EDGE2020.mkv" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[Movie_(1999)_DTS-HD_MA_5.1_-RELEASE_[TBoP].mkv]-[3/15] - \"\" yEnc 9876543210 (2/12345)"), + "Movie_(1999)_DTS-HD_MA_5.1_-RELEASE_[TBoP].mkv" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[00101.mpls]-[163/591] - \"\" yEnc (2/12345)"), + "00101.mpls" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[24]-[12/filename.ext] - \"\" yEnc (2/12345)"), + "filename.ext" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[PRiVATE]-[WtFnZb]-[24]-[filename] - \"\" yEnc (2/12345)"), + "filename" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[N3wZ] \\6aZWVk237607\\::[PRiVATE]-[WtFnZb]-[The.Show.S01E02.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-playWEB.mkv]-[2/8] - \"\" yEnc 2241590477 (1/3128)"), + "The.Show.S01E02.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-playWEB.mkv" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("\"2c0837e5fa42c8cfb5d5e583168a2af4.10\" yEnc (1/111)"), + "2c0837e5fa42c8cfb5d5e583168a2af4.10" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("[02/11] - \"Some.Show.S01E18.Terminal.EAC3.2.0.1080p.WEBRip.x265-iVy.part1.rar\" yEnc(1/144)"), + "Some.Show.S01E18.Terminal.EAC3.2.0.1080p.WEBRip.x265-iVy.part1.rar" + ); + + BOOST_CHECK_EQUAL( + Deobfuscate("Re: Artist Band's The Album-Thanks much - Band, John - Artist - The Album.mp3 (2/3)"), + "Artist Band's The Album-Thanks much - Band, John - Artist - The Album.mp3" + ); + + BOOST_CHECK_EQUAL(Deobfuscate("Re: A (2/3)"), "A"); + BOOST_CHECK_EQUAL(Deobfuscate("Re: A"), "Re: A"); +} diff --git a/tests/queue/NzbFileTest.cpp b/tests/queue/NzbFileTest.cpp index 515517317..2aab8f920 100644 --- a/tests/queue/NzbFileTest.cpp +++ b/tests/queue/NzbFileTest.cpp @@ -21,38 +21,14 @@ #include "nzbget.h" -#define BOOST_TEST_MODULE "NzbFileTest" -#include +#include -#include "NzbFile.h" #include "Log.h" #include "Options.h" #include "DiskState.h" +#include "NzbFile.h" #include "FileSystem.h" -Log* g_Log; -Options* g_Options; -DiskState* g_DiskState; - -struct InitGlobals -{ - InitGlobals() - { - g_Log = new Log(); - g_Options = new Options(nullptr, nullptr); - g_DiskState = new DiskState(); - } - - ~InitGlobals() - { - delete g_Log; - delete g_Options; - delete g_DiskState; - } -}; - -BOOST_GLOBAL_FIXTURE(InitGlobals); - void TestNzb(std::string testFilename) { std::string path = FileSystem::GetCurrentDirectory().Str(); diff --git a/tests/queue/main.cpp b/tests/queue/main.cpp new file mode 100644 index 000000000..3e65f00a8 --- /dev/null +++ b/tests/queue/main.cpp @@ -0,0 +1,31 @@ +#include "nzbget.h" + +#define BOOST_TEST_MODULE "QueueTests" +#include + +#include "Log.h" +#include "Options.h" +#include "DiskState.h" + +Log* g_Log; +Options* g_Options; +DiskState* g_DiskState; + +struct InitGlobals +{ + InitGlobals() + { + g_Log = new Log(); + g_Options = new Options(nullptr, nullptr); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_Options; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/webui/downloads.js b/webui/downloads.js index 928e51417..9c20eb24d 100644 --- a/webui/downloads.js +++ b/webui/downloads.js @@ -64,6 +64,7 @@ var Downloads = (new function($) 'VERIFYING_REPAIRED': { Text: 'VERIFYING', PostProcess: true }, 'RENAMING': { Text: 'RENAMING', PostProcess: true }, 'MOVING': { Text: 'MOVING', PostProcess: true }, + 'POST_UNPACK_RENAMING': { Text: 'POST-UNPACK-RENAMING', PostProcess: true }, 'UNPACKING': { Text: 'UNPACKING', PostProcess: true }, 'EXECUTING_SCRIPT': { Text: 'PROCESSING', PostProcess: true }, 'PP_FINISHED': { Text: 'FINISHED', PostProcess: false }