From afb556bf64635eec0d536bca12dd7d1e5ced399a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 19 Feb 2022 11:34:24 +0400 Subject: [PATCH 1/3] Added field to OPDS entries --- src/opds_dumper.cpp | 4 +++- static/templates/catalog_entries.xml | 4 +++- static/templates/catalog_v2_entries.xml | 1 + static/templates/catalog_v2_entry.xml | 1 + test/server.cpp | 9 +++++++-- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index b42b682f9..361eaa204 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -72,6 +72,7 @@ IllustrationInfo getBookIllustrationInfo(const Book& book) kainjow::mustache::object getSingleBookData(const Book& book) { + const auto bookDate = book.getDate() + "T00:00:00Z"; const MustacheData bookUrl = book.getUrl().empty() ? MustacheData(false) : MustacheData(book.getUrl()); @@ -82,7 +83,8 @@ kainjow::mustache::object getSingleBookData(const Book& book) {"description", book.getDescription()}, {"language", book.getLanguage()}, {"content_id", urlEncode(book.getHumanReadableIdFromPath(), true)}, - {"updated", book.getDate() + "T00:00:00Z"}, + {"updated", bookDate}, // XXX: this should be the entry update datetime + {"book_date", bookDate}, {"category", book.getCategory()}, {"flavour", book.getFlavour()}, {"tags", book.getTags()}, diff --git a/static/templates/catalog_entries.xml b/static/templates/catalog_entries.xml index 40cbfccd7..e9691a42e 100644 --- a/static/templates/catalog_entries.xml +++ b/static/templates/catalog_entries.xml @@ -1,4 +1,6 @@ - + {{feed_id}} {{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}} {{date}} diff --git a/static/templates/catalog_v2_entries.xml b/static/templates/catalog_v2_entries.xml index 2df9523d8..223aae9c3 100644 --- a/static/templates/catalog_v2_entries.xml +++ b/static/templates/catalog_v2_entries.xml @@ -1,5 +1,6 @@ {{feed_id}} diff --git a/static/templates/catalog_v2_entry.xml b/static/templates/catalog_v2_entry.xml index 9f7df876a..a2526b29f 100644 --- a/static/templates/catalog_v2_entry.xml +++ b/static/templates/catalog_v2_entry.xml @@ -25,6 +25,7 @@ {{publisher_name}} + {{book_date}} {{#url}} {{/url}} diff --git a/test/server.cpp b/test/server.cpp index d5e05a652..3950f73c7 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -1081,8 +1081,9 @@ std::string maskVariableOPDSFeedData(std::string s) } #define OPDS_FEED_TAG \ - "\n" + "\n" #define CATALOG_LINK_TAGS \ " \n" \ @@ -1110,6 +1111,7 @@ std::string maskVariableOPDSFeedData(std::string s) " \n" \ " Kiwix\n" \ " \n" \ + " 2020-03-31T00:00:00Z\n" \ " \n" \ " \n" @@ -1136,6 +1138,7 @@ std::string maskVariableOPDSFeedData(std::string s) " \n" \ " Kiwix\n" \ " \n" \ + " 2020-03-31T00:00:00Z\n" \ " \n" \ " \n" @@ -1159,6 +1162,7 @@ std::string maskVariableOPDSFeedData(std::string s) " \n" \ " Kiwix\n" \ " \n" \ + " 2020-03-31T00:00:00Z\n" \ " \n" \ " \n" @@ -1574,6 +1578,7 @@ TEST_F(LibraryServerTest, catalog_v2_languages) #define CATALOG_V2_ENTRIES_PREAMBLE0(x) \ "\n" \ "\n" \ " 12345678-90ab-cdef-1234-567890abcdef\n" \ From 42ee14c8f50138e9390bc5296fc55a4022a18f5e Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 21 Feb 2022 22:52:48 +0400 Subject: [PATCH 2/3] New unit-test LibraryOpdsImportTest.allInOne --- test/library.cpp | 68 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/test/library.cpp b/test/library.cpp index 1f0079b49..ab6ed2683 100644 --- a/test/library.cpp +++ b/test/library.cpp @@ -26,6 +26,8 @@ const char * sampleOpdsStream = R"( 00000000-0000-0000-0000-000000000000 Encyclopédie de la Tunisie + wikipedia_fr_tunisie_novid_2018-10 + unforgettable urn:uuid:0c45160e-f917-760a-9159-dfe3c53cdcdd /meta?name=favicon&content=wikipedia_fr_tunisie_novid_2018-10 2018-10-08T00:00::00:Z @@ -36,8 +38,13 @@ const char * sampleOpdsStream = R"( Wikipedia + + Wikipedia Publishing House + + 1100 + 172 Tania Louis @@ -224,6 +231,64 @@ const char sampleLibraryXML[] = R"( namespace { +TEST(LibraryOpdsImportTest, allInOne) +{ + kiwix::Library lib; + kiwix::Manager manager(&lib); + manager.readOpds(sampleOpdsStream, "library-opds-import.unittests.dev"); + + EXPECT_EQ(10U, lib.getBookCount(true, true)); + + { + const kiwix::Book& book1 = lib.getBookById("0c45160e-f917-760a-9159-dfe3c53cdcdd"); + + EXPECT_EQ(book1.getTitle(), "Encyclopédie de la Tunisie"); + EXPECT_EQ(book1.getName(), "wikipedia_fr_tunisie_novid_2018-10"); + EXPECT_EQ(book1.getFlavour(), "unforgettable"); + EXPECT_EQ(book1.getLanguage(), "fra"); + EXPECT_EQ(book1.getDate(), "2018-10-08"); + EXPECT_EQ(book1.getDescription(), "Le meilleur de Wikipédia sur la Tunisie"); + EXPECT_EQ(book1.getCreator(), "Wikipedia"); + EXPECT_EQ(book1.getPublisher(), "Wikipedia Publishing House"); + EXPECT_EQ(book1.getTags(), "wikipedia;novid;_ftindex"); + EXPECT_EQ(book1.getCategory(), ""); + EXPECT_EQ(book1.getUrl(), "http://download.kiwix.org/zim/wikipedia/wikipedia_fr_tunisie_novid_2018-10.zim.meta4"); + EXPECT_EQ(book1.getSize(), 90030080UL); + EXPECT_EQ(book1.getMediaCount(), 1100U); // Roman MC (MediaCount) is 1100 + EXPECT_EQ(book1.getArticleCount(), 172U); // Hex AC (ArticleCount) is 172 + + const auto illustration = book1.getIllustration(48); + EXPECT_EQ(illustration->width, 48U); + EXPECT_EQ(illustration->height, 48U); + EXPECT_EQ(illustration->mimeType, "image/png"); + EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=wikipedia_fr_tunisie_novid_2018-10"); + } + + { + const kiwix::Book& book2 = lib.getBookById("0189d9be-2fd0-b4b6-7300-20fab0b5cdc8"); + EXPECT_EQ(book2.getTitle(), "TED talks - Business"); + EXPECT_EQ(book2.getName(), ""); + EXPECT_EQ(book2.getFlavour(), ""); + EXPECT_EQ(book2.getLanguage(), "eng"); + EXPECT_EQ(book2.getDate(), "2018-07-23"); + EXPECT_EQ(book2.getDescription(), "Ideas worth spreading"); + EXPECT_EQ(book2.getCreator(), "TED"); + EXPECT_EQ(book2.getPublisher(), ""); + EXPECT_EQ(book2.getTags(), ""); + EXPECT_EQ(book2.getCategory(), ""); + EXPECT_EQ(book2.getUrl(), "http://download.kiwix.org/zim/ted/ted_en_business_2018-07.zim.meta4"); + EXPECT_EQ(book2.getSize(), 8855827456UL); + EXPECT_EQ(book2.getMediaCount(), 0U); + EXPECT_EQ(book2.getArticleCount(), 0U); + + const auto illustration = book2.getIllustration(48); + EXPECT_EQ(illustration->width, 48U); + EXPECT_EQ(illustration->height, 48U); + EXPECT_EQ(illustration->mimeType, "image/png"); + EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=ted_en_business_2018-07"); + } +} + class LibraryTest : public ::testing::Test { protected: typedef kiwix::Library::BookIdCollection BookIdCollection; @@ -292,7 +357,8 @@ TEST_F(LibraryTest, sanityCheck) EXPECT_EQ(lib.getBooksPublishers(), std::vector({ "", "Kiwix", - "Kiwix & Some Enthusiasts" + "Kiwix & Some Enthusiasts", + "Wikipedia Publishing House" })); } From ae2d7d20dccf5befbb9cada75c282c8edc675ed0 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 21 Feb 2022 22:57:23 +0400 Subject: [PATCH 3/3] Handling of in OPDS import --- src/book.cpp | 4 +++- test/library.cpp | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/book.cpp b/src/book.cpp index 17201e33a..c49ab2f19 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -161,7 +161,9 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost m_language = VALUE("language"); m_creator = node.child("author").child("name").child_value(); m_publisher = node.child("publisher").child("name").child_value(); - m_date = fromOpdsDate(VALUE("updated")); + const std::string dcIssuedDate = VALUE("dc:issued"); + m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate; + m_date = fromOpdsDate(m_date); m_name = VALUE("name"); m_flavour = VALUE("flavour"); m_tags = VALUE("tags"); diff --git a/test/library.cpp b/test/library.cpp index ab6ed2683..133b284ac 100644 --- a/test/library.cpp +++ b/test/library.cpp @@ -22,7 +22,9 @@ const char * sampleOpdsStream = R"( - + 00000000-0000-0000-0000-000000000000 Encyclopédie de la Tunisie @@ -31,6 +33,7 @@ const char * sampleOpdsStream = R"( urn:uuid:0c45160e-f917-760a-9159-dfe3c53cdcdd /meta?name=favicon&content=wikipedia_fr_tunisie_novid_2018-10 2018-10-08T00:00::00:Z + 8 Oct 2018 fra Le meilleur de Wikipédia sur la Tunisie wikipedia;novid;_ftindex @@ -246,7 +249,7 @@ TEST(LibraryOpdsImportTest, allInOne) EXPECT_EQ(book1.getName(), "wikipedia_fr_tunisie_novid_2018-10"); EXPECT_EQ(book1.getFlavour(), "unforgettable"); EXPECT_EQ(book1.getLanguage(), "fra"); - EXPECT_EQ(book1.getDate(), "2018-10-08"); + EXPECT_EQ(book1.getDate(), "8 Oct 2018"); EXPECT_EQ(book1.getDescription(), "Le meilleur de Wikipédia sur la Tunisie"); EXPECT_EQ(book1.getCreator(), "Wikipedia"); EXPECT_EQ(book1.getPublisher(), "Wikipedia Publishing House");