Add raindrop integration

docker-compose.yml

@@ -1,133 +1,153 @@
 x-loki: &loki
-  logging:
-    driver: loki # Requires loki docker driver: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
-    options:
-      loki-url: "http://host.docker.internal:3100/loki/api/v1/push" # <-- MUST be this address and NOT service name, otherwise Docker crashes
+    logging:
+        driver: loki # Requires loki docker driver: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+        options:
+            loki-url: 'http://host.docker.internal:3100/loki/api/v1/push' # <-- MUST be this address and NOT service name, otherwise Docker crashes
 x-build: &build
-  x-bake:
-    platforms:
-      - linux/amd64
-      - linux/arm64
-
+    x-bake:
+        platforms:
+            - linux/amd64
+            - linux/arm64
 
 services:
-  nats-server:
-    image: docker.io/nats:2.9
-    ports:
-      - "4222:4222" # 4222 is for clients.
-      - "8222:8222" # 8222 is an HTTP management port for information reporting.
-      - "6222:6222" # 6222 is a routing port for clustering.
-    volumes:
-      - "./nats-persistence/:/data"
-    command: "--jetstream --http_port 8222 --store_dir /data"
-  fullfeedrss:
-    image: docker.io/heussd/fivefilters-full-text-rss:latest
-    deploy:
-      replicas: 2
-    environment:
-      # Leave empty to disable admin section
-      - FTR_ADMIN_PASSWORD=
-    volumes:
-      - "rss-cache:/var/www/html/cache"
-
-  loadbalancer:
-    image: docker.io/nginx
-    volumes:
-      - "./loadbalancer/nginx.conf:/etc/nginx/nginx.conf"
-    depends_on:
-        - fullfeedrss
-    ports:
-        - '80:80'
-  rss-feed-feeder:
-    image: ghcr.io/heussd/nats-news-analysis/rss-feed-feeder:latest
-    build:
-      <<: *build
-      context: rss-feed-feeder/.
-    environment:
-      - NATS_SERVER=http://nats-server:4222
-    volumes:
-      - type: bind
-        source: ./urls.txt
-        target: /urls.txt
-        consistency: cached
-        read_only: true
-  rss-article-url-feeder-go:
-    image: ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go:latest
-    deploy:
-      replicas: 6
-    build:
-      <<: *build
-      context: rss-article-url-feeder-go/.
-    environment:
-      - "NATS_SERVER=nats-server:4222"
-    restart: always
-  loki:
-    container_name: loki
-    image: grafana/loki:latest
-    ports:
-      - "3100:3100"
-    command: -config.file=/etc/loki/local-config.yaml
-    volumes:
-      - ./loki/local-config.yaml:/etc/loki/local-config.yaml
-  keyword-matcher-go:
-    deploy:
-      replicas: 8
-    <<: *loki
-    image: ghcr.io/heussd/nats-news-analysis/keyword-matcher-go:latest
-    build:
-      <<: *build
-      context: keyword-matcher-go/.
-    volumes:
-      - type: bind
-        source: ./keywords.txt
-        target: /keywords.txt
-        consistency: cached
-        read_only: true
-    environment:
-      - "KEYWORDS_FILE:/keywords.txt"
-      - "NATS_SERVER=nats-server:4222"
-      - "FULLTEXTRSS_SERVER=http://loadbalancer:80"
-  pocket-integration:
-    image: ghcr.io/heussd/nats-news-analysis/pocket-integration:latest
-    build:
-      <<: *build
-      context: pocket-integration/.
-      cache_from:
-        - ghcr.io/heussd/nats-news-analysis/pocket-integration:latest
-    environment:
-      - "NATS_SERVER=nats-server:4222"
-    secrets:
-      - POCKET_ACCESS_TOKEN
-      - POCKET_CONSUMER_KEY
-  grafana:
-    image: grafana/grafana:master
-    volumes:
-      - ./grafana/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
-      - ./grafana/dashboard-provider.yaml:/etc/grafana/provisioning/dashboards/main.yaml
-      - ./grafana/dashboards:/var/lib/grafana/dashboards
-    environment:
-      - GF_AUTH_ANONYMOUS_ENABLED=true
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-      - GF_AUTH_DISABLE_LOGIN_FORM=true
-    ports:
-      - "3000:3000"
-  prometheus:
-    image: prom/prometheus:latest
-    volumes:
-      - ./prometheus/:/etc/prometheus/
-      - ./prom-storage:/usr/local/share/prometheus
-    command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/usr/local/share/prometheus
-  natsexporter:
-    image: natsio/prometheus-nats-exporter
-    ports:
-    - "7777:7777"
-    command: ['-varz', '-subz', "-serverz", '-connz', '-channelz', '-jsz', 'all', 'http://nats-server:8222']
+    nats-server:
+        image: docker.io/nats:2.9
+        ports:
+            - '4222:4222' # 4222 is for clients.
+            - '8222:8222' # 8222 is an HTTP management port for information reporting.
+            - '6222:6222' # 6222 is a routing port for clustering.
+        volumes:
+            - './nats-persistence/:/data'
+        command: '--jetstream --http_port 8222 --store_dir /data'
+    fullfeedrss:
+        image: docker.io/heussd/fivefilters-full-text-rss:latest
+        deploy:
+            replicas: 2
+        environment:
+            # Leave empty to disable admin section
+            - FTR_ADMIN_PASSWORD=
+        volumes:
+            - 'rss-cache:/var/www/html/cache'
+
+    loadbalancer:
+        image: docker.io/nginx
+        volumes:
+            - './loadbalancer/nginx.conf:/etc/nginx/nginx.conf'
+        depends_on:
+            - fullfeedrss
+        ports:
+            - '80:80'
+    rss-feed-feeder:
+        image: ghcr.io/heussd/nats-news-analysis/rss-feed-feeder:latest
+        build:
+            <<: *build
+            context: rss-feed-feeder/.
+        environment:
+            - NATS_SERVER=http://nats-server:4222
+        volumes:
+            - type: bind
+              source: ./urls.txt
+              target: /urls.txt
+              consistency: cached
+              read_only: true
+    rss-article-url-feeder-go:
+        image: ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go:latest
+        deploy:
+            replicas: 6
+        build:
+            <<: *build
+            context: rss-article-url-feeder-go/.
+        environment:
+            - 'NATS_SERVER=nats-server:4222'
+        restart: always
+    loki:
+        container_name: loki
+        image: grafana/loki:latest
+        ports:
+            - '3100:3100'
+        command: -config.file=/etc/loki/local-config.yaml
+        volumes:
+            - ./loki/local-config.yaml:/etc/loki/local-config.yaml
+    keyword-matcher-go:
+        deploy:
+            replicas: 8
+        <<: *loki
+        image: ghcr.io/heussd/nats-news-analysis/keyword-matcher-go:latest
+        build:
+            <<: *build
+            context: keyword-matcher-go/.
+        volumes:
+            - type: bind
+              source: ./keywords.txt
+              target: /keywords.txt
+              consistency: cached
+              read_only: true
+        environment:
+            - 'KEYWORDS_FILE:/keywords.txt'
+            - 'NATS_SERVER=nats-server:4222'
+            - 'FULLTEXTRSS_SERVER=http://loadbalancer:80'
+    pocket-integration:
+        image: ghcr.io/heussd/nats-news-analysis/pocket-integration:latest
+        build:
+            <<: *build
+            context: pocket-integration/.
+            cache_from:
+                - ghcr.io/heussd/nats-news-analysis/pocket-integration:latest
+        environment:
+            - 'NATS_SERVER=nats-server:4222'
+        secrets:
+            - POCKET_ACCESS_TOKEN
+            - POCKET_CONSUMER_KEY
+    raindrop-integration:
+        image: ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest
+        build:
+            <<: *build
+            context: raindrop-integration/.
+            cache_from:
+                - ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest
+        environment:
+            - 'NATS_SERVER=nats-server:4222'
+        env_file:
+            - .env
+    grafana:
+        image: grafana/grafana:master
+        volumes:
+            - ./grafana/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml
+            - ./grafana/dashboard-provider.yaml:/etc/grafana/provisioning/dashboards/main.yaml
+            - ./grafana/dashboards:/var/lib/grafana/dashboards
+        environment:
+            - GF_AUTH_ANONYMOUS_ENABLED=true
+            - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+            - GF_AUTH_DISABLE_LOGIN_FORM=true
+        ports:
+            - '3000:3000'
+    prometheus:
+        image: prom/prometheus:latest
+        volumes:
+            - ./prometheus/:/etc/prometheus/
+            - ./prom-storage:/usr/local/share/prometheus
+        command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/usr/local/share/prometheus
+    natsexporter:
+        image: natsio/prometheus-nats-exporter
+        ports:
+            - '7777:7777'
+        command:
+            [
+                '-varz',
+                '-subz',
+                '-serverz',
+                '-connz',
+                '-channelz',
+                '-jsz',
+                'all',
+                'http://nats-server:8222',
+            ]
 
 volumes:
-  rss-cache:
+    rss-cache:
 
 secrets:
-  POCKET_ACCESS_TOKEN:
-    file: ./POCKET_ACCESS_TOKEN.txt
-  POCKET_CONSUMER_KEY:
-    file: ./POCKET_CONSUMER_KEY.txt
+    POCKET_ACCESS_TOKEN:
+        file: ./POCKET_ACCESS_TOKEN.txt
+    POCKET_CONSUMER_KEY:
+        file: ./POCKET_CONSUMER_KEY.txt

raindrop-integration/Dockerfile

@@ -0,0 +1,25 @@
+FROM golang:1.19-bullseye as builder
+
+ENV GO111MODULE=on
+
+WORKDIR /app
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY internal ./internal
+COPY cmd ./cmd
+
+RUN CGO_ENABLED=0 go build -a -ldflags '-s' -o goapp.bin ./cmd
+
+
+# https://github.com/drone/ca-certs/blob/master/Dockerfile
+FROM alpine:3.6 as alpine
+RUN apk add -U --no-cache ca-certificates
+
+
+FROM scratch
+COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=builder /app/goapp.bin /goapp.bin
+
+CMD [ "./goapp.bin" ]

raindrop-integration/Makefile

@@ -0,0 +1,5 @@
+publish:
+	nats pub match-url "{\"RegexId\": \"OMG\", \"Url\": \"https://www.tagesschau.de/ausland/europa/afrika-russland-ukraine-frieden-100.html\"}"
+
+stream:
+	nats stream add match-urls --subjects match-url --storage memory --replicas 1 --retention work --discard old --max-msgs=-1 --max-msgs-per-subject=-1 --max-bytes=-1 --max-age=-1  --allow-rollup --allow-direct

raindrop-integration/cmd/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	queue "github.com/heussd/nats-raindrop-integration.go/internal/nats"
+	"github.com/heussd/nats-raindrop-integration.go/internal/raindrop"
+	"github.com/nats-io/nats.go"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+type Match struct {
+	RegexId string
+	Url     string
+}
+
+func main() {
+
+	queue.WithMatchUrls(func(m *nats.Msg) {
+		var match Match
+
+		if err := json.Unmarshal([]byte(string(m.Data)), &match); err != nil {
+			fmt.Errorf("failed to unmarshall %w", err)
+		}
+
+		if ok, _ := raindrop.Add(match.Url); !ok {
+			fmt.Errorf("received error from raindrop\n")
+		} else {
+			fmt.Printf("Added to Raindrop: %s\n	", match.Url)
+		}
+	})
+
+	// https://callistaenterprise.se/blogg/teknik/2019/10/05/go-worker-cancellation/
+	termChan := make(chan os.Signal)
+	signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM)
+	<-termChan // Blocks here until either SIGINT or SIGTERM is received.
+}

raindrop-integration/docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+    nats-server:
+        image: nats:2.9
+        ports:
+            - '4222:4222' # 4222 is for clients.
+            - '8222:8222' # 8222 is an HTTP management port for information reporting.
+            - '6222:6222' # 6222 is a routing port for clustering.
+        command: '--jetstream'
+    nats-raindrop--integration:
+        image: ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest
+        build: .
+        environment:
+            - 'NATS_SERVER=nats-server:4222'
+        env_file:
+            - .env

raindrop-integration/go.mod

@@ -0,0 +1,21 @@
+module github.com/heussd/nats-raindrop-integration.go
+
+go 1.19
+
+require (
+	github.com/nats-io/nats.go v1.22.1
+	github.com/shomali11/util v0.0.0-20220717175126-f0771b70947f
+	github.com/stretchr/testify v1.8.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/nats-io/nats-server/v2 v2.9.10 // indirect
+	github.com/nats-io/nkeys v0.3.0 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)