Merge pull request #104 from stscoundrel/feature/old-danish-dictionary

Feature/old danish dictionary
stscoundrel · Feb 24, 2024 · 880a582 · 880a582
2 parents 5020ec5 + 8d4a6fd
commit 880a582
Show file tree

Hide file tree

Showing 10 changed files with 52 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -7,10 +7,30 @@ Parses sources from following dictionary projects:
 - [Old Icelandic Dictionary](https://old-icelandic.vercel.app/)
 - [Old Norwegian Dictionary](https://old-norwegian-dictionary.vercel.app/)
 - [Old Swedish Dictionary](https://old-swedish-dictionary.vercel.app/)
+- [Old Danish Dictionary](https://old-danish-dictionary.vercel.app/)
 
 
 The parser finds over 1 000 entries that are present in all four dictionaries. There are also over 20 000 entries that appear in at least two different dictionaries, making them worth a crosslink.
 
 ### Install
 
 `yarn add scandinavian-dictionary-crosslinker`
+
+
+### Download sitemaps.
+
+Run `cargo run` in `downloader` folder. Downloads latest XML sitemaps to `resources` folder.
+
+
+### Generate crosslinks
+
+Run `go run *.go` in `crosslinks` folder. Generates crosslinks json to `resources` folder.
+
+
+### Minify outout
+
+Run `nimble build` and `./minifier` in `minifier` folder. Generates minified & gzipped json outputs.
+
+### Update data to NPM module.
+
+Run `go run main.go` in root folder to update json & readme to NPM module.
diff --git a/crosslinker/internal/sitemaps/reader.go b/crosslinker/internal/sitemaps/reader.go
@@ -7,6 +7,7 @@ var sitemapPaths = map[string]string{
 	"old-icelandic": "../resources/old-icelandic.xml",
 	"old-norwegian": "../resources/old-norwegian.xml",
 	"old-swedish":   "../resources/old-swedish.xml",
+	"old-danish":    "../resources/old-danish.xml",
 }
 
 func readXmlSitemaps() (map[string][]byte, error) {

diff --git a/downloader/src/main.rs b/downloader/src/main.rs
@@ -10,6 +10,7 @@ async fn main() {
     urls.insert("old-icelandic", "https://old-icelandic.vercel.app/sitemap.xml");
     urls.insert("old-norwegian", "https://old-norwegian-dictionary.vercel.app/sitemap.xml");
     urls.insert("old-swedish", "https://old-swedish-dictionary.vercel.app/sitemap.xml");
+    urls.insert("old-danish", "https://old-danish-dictionary.vercel.app/sitemap.xml");
 
     let client = Client::new();
 

diff --git a/minifier/src/minifier.nim b/minifier/src/minifier.nim
@@ -13,6 +13,7 @@ proc minify_link(link: string): string =
     .replace("https://old-norwegian-dictionary.vercel.app/word/", "")
     .replace("https://old-swedish-dictionary.vercel.app/word/", "")
     .replace("https://cleasby-vigfusson-dictionary.vercel.app/word/", "")
+    .replace("https://old-danish-dictionary.vercel.app/word/", "")
 
 
 proc minify_source(link: string): string =
@@ -21,6 +22,7 @@ proc minify_source(link: string): string =
     .replace("old-swedish", "os")
     .replace("old-norse", "on")
     .replace("old-icelandic", "oi")
+    .replace("old-danish", "od")
 
 
 for slug, links in jsonContent:

diff --git a/npm-module/README.md b/npm-module/README.md
@@ -7,10 +7,11 @@ Parses sources from following dictionary projects:
 - [Old Icelandic Dictionary](https://old-icelandic.vercel.app/)
 - [Old Norwegian Dictionary](https://old-norwegian-dictionary.vercel.app/)
 - [Old Swedish Dictionary](https://old-swedish-dictionary.vercel.app/)
+- [Old Danish Dictionary](https://old-danish-dictionary.vercel.app/)
 
 
 The parser finds over 1 000 entries that are present in all four dictionaries. There are also over 20 000 entries that appear in at least two different dictionaries, making them worth a crosslink.
 
 ### Install
 
-`yarn add scandinavian-dictionary-crosslinker`
+`yarn add scandinavian-dictionary-crosslinker`
diff --git a/npm-module/package.json b/npm-module/package.json
@@ -1,6 +1,6 @@
 {
   "name": "scandinavian-dictionary-crosslinker",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Finds shared entries in dictionary sitemaps, allowing crosslinking",
   "repository": "https://github.com/stscoundrel/scandinavian-dictionary-crosslinker.git",
   "author": "stscoundrel <silvennoinen.sampo@gmail.com>",

diff --git a/npm-module/resources/crosslinks.json.gz b/npm-module/resources/crosslinks.json.gz
diff --git a/npm-module/src/index.ts b/npm-module/src/index.ts
@@ -46,4 +46,9 @@ export const getOldSwedishCrosslinks = (slug: string): Crosslink[] => filterCros
   DictionarySource.OldSwedish,
 );
 
+export const getOldDanishCrosslinks = (slug: string): Crosslink[] => filterCrosslinksByLanguage(
+  slug,
+  DictionarySource.OldDanish,
+);
+
 export { Crosslink, DictionarySource } from './models';
diff --git a/npm-module/src/models.ts b/npm-module/src/models.ts
@@ -2,7 +2,8 @@ export enum DictionarySource {
   OldNorse = 'old-norse',
   OldIcelandic = 'old-icelandic',
   OldNorwegian = 'old-norwegian',
-  OldSwedish = 'old-swedish'
+  OldSwedish = 'old-swedish',
+  OldDanish = 'old-danish'
 }
 
 export interface Crosslink{

diff --git a/npm-module/tests/crosslinks.test.ts b/npm-module/tests/crosslinks.test.ts
@@ -11,7 +11,7 @@ describe('Crosslinks tests', () => {
   test('Crosslinks contain correct amount of slug entries', () => {
     const result = getCrosslinks();
 
-    expect(Object.keys(result).length).toBe(44545);
+    expect(Object.keys(result).length).toBe(48294);
   });
 
   test('Crosslinks contain correct amount of summed individual links', () => {
@@ -22,7 +22,7 @@ describe('Crosslinks tests', () => {
       sum += result[key].length;
     });
 
-    expect(sum).toEqual(120313);
+    expect(sum).toEqual(128440);
   });
 
   test('Crosslink entries are returned in correct object format', () => {
@@ -41,23 +41,19 @@ describe('Crosslinks tests', () => {
 
     expect(result.abyrgdarhlutr).toEqual(
       [
-        {
-          url: 'https://old-norwegian-dictionary.vercel.app/word/abyrgdarhlutr',
-          source: DictionarySource.OldNorwegian,
-        },
         {
           url: 'https://old-icelandic.vercel.app/word/abyrgdarhlutr',
           source: DictionarySource.OldIcelandic,
         },
+        {
+          url: 'https://old-norwegian-dictionary.vercel.app/word/abyrgdarhlutr',
+          source: DictionarySource.OldNorwegian,
+        },
       ],
     );
 
     expect(result.hneyking).toEqual(
       [
-        {
-          url: 'https://old-norwegian-dictionary.vercel.app/word/hneyking',
-          source: DictionarySource.OldNorwegian,
-        },
         {
           url: 'https://cleasby-vigfusson-dictionary.vercel.app/word/hneyking',
           source: DictionarySource.OldNorse,
@@ -66,19 +62,23 @@ describe('Crosslinks tests', () => {
           url: 'https://old-icelandic.vercel.app/word/hneyking',
           source: DictionarySource.OldIcelandic,
         },
+        {
+          url: 'https://old-norwegian-dictionary.vercel.app/word/hneyking',
+          source: DictionarySource.OldNorwegian,
+        },
       ],
     );
 
     expect(result.skurfir).toEqual(
       [
-        {
-          url: 'https://old-norwegian-dictionary.vercel.app/word/skurfir',
-          source: DictionarySource.OldNorwegian,
-        },
         {
           url: 'https://cleasby-vigfusson-dictionary.vercel.app/word/skurfir',
           source: DictionarySource.OldNorse,
         },
+        {
+          url: 'https://old-norwegian-dictionary.vercel.app/word/skurfir',
+          source: DictionarySource.OldNorwegian,
+        },
       ],
     );
 
@@ -94,10 +94,6 @@ describe('Crosslinks tests', () => {
     ]);
 
     expect(result['otta-lauss']).toEqual([
-      {
-        url: 'https://old-norwegian-dictionary.vercel.app/word/ottalauss',
-        source: DictionarySource.OldNorwegian,
-      },
       {
         source: DictionarySource.OldNorse,
         url: 'https://cleasby-vigfusson-dictionary.vercel.app/word/otta-lauss',
@@ -106,7 +102,10 @@ describe('Crosslinks tests', () => {
         source: DictionarySource.OldIcelandic,
         url: 'https://old-icelandic.vercel.app/word/ottalauss',
       },
-
+      {
+        url: 'https://old-norwegian-dictionary.vercel.app/word/ottalauss',
+        source: DictionarySource.OldNorwegian,
+      },
     ]);
   });
 });