Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added <dc:issued> field to OPDS entries #715

Merged
merged 3 commits into from
Feb 23, 2022
Merged

Added <dc:issued> field to OPDS entries #715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
afb556b
Added <dc:issued> field to OPDS entries
veloman-yunkan Feb 19, 2022
42ee14c
New unit-test LibraryOpdsImportTest.allInOne
veloman-yunkan Feb 21, 2022
ae2d7d2
Handling of <dc:issued> in OPDS import
veloman-yunkan Feb 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/book.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,9 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
m_language = VALUE("language");
m_creator = node.child("author").child("name").child_value();
m_publisher = node.child("publisher").child("name").child_value();
m_date = fromOpdsDate(VALUE("updated"));
const std::string dcIssuedDate = VALUE("dc:issued");
m_date = dcIssuedDate.empty() ? VALUE("updated") : dcIssuedDate;
m_date = fromOpdsDate(m_date);
m_name = VALUE("name");
m_flavour = VALUE("flavour");
m_tags = VALUE("tags");
Expand Down
4 changes: 3 additions & 1 deletion src/opds_dumper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ IllustrationInfo getBookIllustrationInfo(const Book& book)

kainjow::mustache::object getSingleBookData(const Book& book)
{
const auto bookDate = book.getDate() + "T00:00:00Z";
const MustacheData bookUrl = book.getUrl().empty()
? MustacheData(false)
: MustacheData(book.getUrl());
Expand All @@ -82,7 +83,8 @@ kainjow::mustache::object getSingleBookData(const Book& book)
{"description", book.getDescription()},
{"language", book.getLanguage()},
{"content_id", urlEncode(book.getHumanReadableIdFromPath(), true)},
{"updated", book.getDate() + "T00:00:00Z"},
{"updated", bookDate}, // XXX: this should be the entry update datetime
{"book_date", bookDate},
{"category", book.getCategory()},
{"flavour", book.getFlavour()},
{"tags", book.getTags()},
Expand Down
4 changes: 3 additions & 1 deletion static/templates/catalog_entries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="http://opds-spec.org/2010/catalog">
<id>{{feed_id}}</id>
<title>{{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}}</title>
<updated>{{date}}</updated>
Expand Down
1 change: 1 addition & 0 deletions static/templates/catalog_v2_entries.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="https://specs.opds.io/opds-1.2"
xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/">
<id>{{feed_id}}</id>
Expand Down
1 change: 1 addition & 0 deletions static/templates/catalog_v2_entry.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
<publisher>
<name>{{publisher_name}}</name>
</publisher>
<dc:issued>{{book_date}}</dc:issued>
{{#url}}
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="{{{url}}}" length="{{{size}}}" />
{{/url}}
Expand Down
73 changes: 71 additions & 2 deletions test/library.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,32 @@


const char * sampleOpdsStream = R"(
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog">
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:dc="http://purl.org/dc/terms/"
xmlns:opds="http://opds-spec.org/2010/catalog">
<id>00000000-0000-0000-0000-000000000000</id>
<entry>
<title>Encyclopédie de la Tunisie</title>
<name>wikipedia_fr_tunisie_novid_2018-10</name>
<flavour>unforgettable</flavour>
<id>urn:uuid:0c45160e-f917-760a-9159-dfe3c53cdcdd</id>
<icon>/meta?name=favicon&amp;content=wikipedia_fr_tunisie_novid_2018-10</icon>
<updated>2018-10-08T00:00::00:Z</updated>
<dc:issued>8 Oct 2018</dc:issued>
<language>fra</language>
<summary>Le meilleur de Wikipédia sur la Tunisie</summary>
<tags>wikipedia;novid;_ftindex</tags>
<link type="text/html" href="/wikipedia_fr_tunisie_novid_2018-10" />
<author>
<name>Wikipedia</name>
</author>
<publisher>
<name>Wikipedia Publishing House</name>
</publisher>
<link rel="http://opds-spec.org/acquisition/open-access" type="application/x-zim" href="http://download.kiwix.org/zim/wikipedia/wikipedia_fr_tunisie_novid_2018-10.zim.meta4" length="90030080" />
<link rel="http://opds-spec.org/image/thumbnail" type="image/png" href="/meta?name=favicon&amp;content=wikipedia_fr_tunisie_novid_2018-10" />
<mediaCount>1100</mediaCount>
<articleCount>172</articleCount>
</entry>
<entry>
<title>Tania Louis</title>
Expand Down Expand Up @@ -224,6 +234,64 @@ const char sampleLibraryXML[] = R"(
namespace
{

TEST(LibraryOpdsImportTest, allInOne)
{
kiwix::Library lib;
kiwix::Manager manager(&lib);
manager.readOpds(sampleOpdsStream, "library-opds-import.unittests.dev");

EXPECT_EQ(10U, lib.getBookCount(true, true));

{
const kiwix::Book& book1 = lib.getBookById("0c45160e-f917-760a-9159-dfe3c53cdcdd");

EXPECT_EQ(book1.getTitle(), "Encyclopédie de la Tunisie");
EXPECT_EQ(book1.getName(), "wikipedia_fr_tunisie_novid_2018-10");
EXPECT_EQ(book1.getFlavour(), "unforgettable");
EXPECT_EQ(book1.getLanguage(), "fra");
EXPECT_EQ(book1.getDate(), "8 Oct 2018");
EXPECT_EQ(book1.getDescription(), "Le meilleur de Wikipédia sur la Tunisie");
EXPECT_EQ(book1.getCreator(), "Wikipedia");
EXPECT_EQ(book1.getPublisher(), "Wikipedia Publishing House");
EXPECT_EQ(book1.getTags(), "wikipedia;novid;_ftindex");
EXPECT_EQ(book1.getCategory(), "");
EXPECT_EQ(book1.getUrl(), "http://download.kiwix.org/zim/wikipedia/wikipedia_fr_tunisie_novid_2018-10.zim.meta4");
EXPECT_EQ(book1.getSize(), 90030080UL);
EXPECT_EQ(book1.getMediaCount(), 1100U); // Roman MC (MediaCount) is 1100
EXPECT_EQ(book1.getArticleCount(), 172U); // Hex AC (ArticleCount) is 172

const auto illustration = book1.getIllustration(48);
EXPECT_EQ(illustration->width, 48U);
EXPECT_EQ(illustration->height, 48U);
EXPECT_EQ(illustration->mimeType, "image/png");
EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=wikipedia_fr_tunisie_novid_2018-10");
}

{
const kiwix::Book& book2 = lib.getBookById("0189d9be-2fd0-b4b6-7300-20fab0b5cdc8");
EXPECT_EQ(book2.getTitle(), "TED talks - Business");
EXPECT_EQ(book2.getName(), "");
EXPECT_EQ(book2.getFlavour(), "");
EXPECT_EQ(book2.getLanguage(), "eng");
EXPECT_EQ(book2.getDate(), "2018-07-23");
EXPECT_EQ(book2.getDescription(), "Ideas worth spreading");
EXPECT_EQ(book2.getCreator(), "TED");
EXPECT_EQ(book2.getPublisher(), "");
EXPECT_EQ(book2.getTags(), "");
EXPECT_EQ(book2.getCategory(), "");
EXPECT_EQ(book2.getUrl(), "http://download.kiwix.org/zim/ted/ted_en_business_2018-07.zim.meta4");
EXPECT_EQ(book2.getSize(), 8855827456UL);
EXPECT_EQ(book2.getMediaCount(), 0U);
EXPECT_EQ(book2.getArticleCount(), 0U);

const auto illustration = book2.getIllustration(48);
EXPECT_EQ(illustration->width, 48U);
EXPECT_EQ(illustration->height, 48U);
EXPECT_EQ(illustration->mimeType, "image/png");
EXPECT_EQ(illustration->url, "library-opds-import.unittests.dev/meta?name=favicon&content=ted_en_business_2018-07");
}
}

class LibraryTest : public ::testing::Test {
protected:
typedef kiwix::Library::BookIdCollection BookIdCollection;
Expand Down Expand Up @@ -292,7 +360,8 @@ TEST_F(LibraryTest, sanityCheck)
EXPECT_EQ(lib.getBooksPublishers(), std::vector<std::string>({
"",
"Kiwix",
"Kiwix & Some Enthusiasts"
"Kiwix & Some Enthusiasts",
"Wikipedia Publishing House"
}));
}

Expand Down
9 changes: 7 additions & 2 deletions test/server.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1081,8 +1081,9 @@ std::string maskVariableOPDSFeedData(std::string s)
}

#define OPDS_FEED_TAG \
"<feed xmlns=\"http://www.w3.org/2005/Atom\"" \
" xmlns:opds=\"http://opds-spec.org/2010/catalog\">\n"
"<feed xmlns=\"http://www.w3.org/2005/Atom\"\n" \
" xmlns:dc=\"http://purl.org/dc/terms/\"\n" \
" xmlns:opds=\"http://opds-spec.org/2010/catalog\">\n"

#define CATALOG_LINK_TAGS \
" <link rel=\"self\" href=\"\" type=\"application/atom+xml\" />\n" \
Expand Down Expand Up @@ -1110,6 +1111,7 @@ std::string maskVariableOPDSFeedData(std::string s)
" <publisher>\n" \
" <name>Kiwix</name>\n" \
" </publisher>\n" \
" <dc:issued>2020-03-31T00:00:00Z</dc:issued>\n" \
" <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile%26other.zim\" length=\"569344\" />\n" \
" </entry>\n"

Expand All @@ -1136,6 +1138,7 @@ std::string maskVariableOPDSFeedData(std::string s)
" <publisher>\n" \
" <name>Kiwix</name>\n" \
" </publisher>\n" \
" <dc:issued>2020-03-31T00:00:00Z</dc:issued>\n" \
" <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim\" length=\"569344\" />\n" \
" </entry>\n"

Expand All @@ -1159,6 +1162,7 @@ std::string maskVariableOPDSFeedData(std::string s)
" <publisher>\n" \
" <name>Kiwix</name>\n" \
" </publisher>\n" \
" <dc:issued>2020-03-31T00:00:00Z</dc:issued>\n" \
" <link rel=\"http://opds-spec.org/acquisition/open-access\" type=\"application/x-zim\" href=\"https://github.com/kiwix/libkiwix/raw/master/test/data/zimfile.zim\" length=\"125952\" />\n" \
" </entry>\n"

Expand Down Expand Up @@ -1574,6 +1578,7 @@ TEST_F(LibraryServerTest, catalog_v2_languages)
#define CATALOG_V2_ENTRIES_PREAMBLE0(x) \
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" \
"<feed xmlns=\"http://www.w3.org/2005/Atom\"\n" \
" xmlns:dc=\"http://purl.org/dc/terms/\"\n" \
" xmlns:opds=\"https://specs.opds.io/opds-1.2\"\n" \
" xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\">\n" \
" <id>12345678-90ab-cdef-1234-567890abcdef</id>\n" \
Expand Down