diff --git a/docker-compose.yml b/docker-compose.yml index d6df146..14dd67d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,133 +1,153 @@ x-loki: &loki - logging: - driver: loki # Requires loki docker driver: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions - options: - loki-url: "http://host.docker.internal:3100/loki/api/v1/push" # <-- MUST be this address and NOT service name, otherwise Docker crashes + logging: + driver: loki # Requires loki docker driver: docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + options: + loki-url: 'http://host.docker.internal:3100/loki/api/v1/push' # <-- MUST be this address and NOT service name, otherwise Docker crashes x-build: &build - x-bake: - platforms: - - linux/amd64 - - linux/arm64 - + x-bake: + platforms: + - linux/amd64 + - linux/arm64 services: - nats-server: - image: docker.io/nats:2.9 - ports: - - "4222:4222" # 4222 is for clients. - - "8222:8222" # 8222 is an HTTP management port for information reporting. - - "6222:6222" # 6222 is a routing port for clustering. - volumes: - - "./nats-persistence/:/data" - command: "--jetstream --http_port 8222 --store_dir /data" - fullfeedrss: - image: docker.io/heussd/fivefilters-full-text-rss:latest - deploy: - replicas: 2 - environment: - # Leave empty to disable admin section - - FTR_ADMIN_PASSWORD= - volumes: - - "rss-cache:/var/www/html/cache" - - loadbalancer: - image: docker.io/nginx - volumes: - - "./loadbalancer/nginx.conf:/etc/nginx/nginx.conf" - depends_on: - - fullfeedrss - ports: - - '80:80' - rss-feed-feeder: - image: ghcr.io/heussd/nats-news-analysis/rss-feed-feeder:latest - build: - <<: *build - context: rss-feed-feeder/. - environment: - - NATS_SERVER=http://nats-server:4222 - volumes: - - type: bind - source: ./urls.txt - target: /urls.txt - consistency: cached - read_only: true - rss-article-url-feeder-go: - image: ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go:latest - deploy: - replicas: 6 - build: - <<: *build - context: rss-article-url-feeder-go/. - environment: - - "NATS_SERVER=nats-server:4222" - restart: always - loki: - container_name: loki - image: grafana/loki:latest - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - volumes: - - ./loki/local-config.yaml:/etc/loki/local-config.yaml - keyword-matcher-go: - deploy: - replicas: 8 - <<: *loki - image: ghcr.io/heussd/nats-news-analysis/keyword-matcher-go:latest - build: - <<: *build - context: keyword-matcher-go/. - volumes: - - type: bind - source: ./keywords.txt - target: /keywords.txt - consistency: cached - read_only: true - environment: - - "KEYWORDS_FILE:/keywords.txt" - - "NATS_SERVER=nats-server:4222" - - "FULLTEXTRSS_SERVER=http://loadbalancer:80" - pocket-integration: - image: ghcr.io/heussd/nats-news-analysis/pocket-integration:latest - build: - <<: *build - context: pocket-integration/. - cache_from: - - ghcr.io/heussd/nats-news-analysis/pocket-integration:latest - environment: - - "NATS_SERVER=nats-server:4222" - secrets: - - POCKET_ACCESS_TOKEN - - POCKET_CONSUMER_KEY - grafana: - image: grafana/grafana:master - volumes: - - ./grafana/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml - - ./grafana/dashboard-provider.yaml:/etc/grafana/provisioning/dashboards/main.yaml - - ./grafana/dashboards:/var/lib/grafana/dashboards - environment: - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_AUTH_DISABLE_LOGIN_FORM=true - ports: - - "3000:3000" - prometheus: - image: prom/prometheus:latest - volumes: - - ./prometheus/:/etc/prometheus/ - - ./prom-storage:/usr/local/share/prometheus - command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/usr/local/share/prometheus - natsexporter: - image: natsio/prometheus-nats-exporter - ports: - - "7777:7777" - command: ['-varz', '-subz', "-serverz", '-connz', '-channelz', '-jsz', 'all', 'http://nats-server:8222'] + nats-server: + image: docker.io/nats:2.9 + ports: + - '4222:4222' # 4222 is for clients. + - '8222:8222' # 8222 is an HTTP management port for information reporting. + - '6222:6222' # 6222 is a routing port for clustering. + volumes: + - './nats-persistence/:/data' + command: '--jetstream --http_port 8222 --store_dir /data' + fullfeedrss: + image: docker.io/heussd/fivefilters-full-text-rss:latest + deploy: + replicas: 2 + environment: + # Leave empty to disable admin section + - FTR_ADMIN_PASSWORD= + volumes: + - 'rss-cache:/var/www/html/cache' + + loadbalancer: + image: docker.io/nginx + volumes: + - './loadbalancer/nginx.conf:/etc/nginx/nginx.conf' + depends_on: + - fullfeedrss + ports: + - '80:80' + rss-feed-feeder: + image: ghcr.io/heussd/nats-news-analysis/rss-feed-feeder:latest + build: + <<: *build + context: rss-feed-feeder/. + environment: + - NATS_SERVER=http://nats-server:4222 + volumes: + - type: bind + source: ./urls.txt + target: /urls.txt + consistency: cached + read_only: true + rss-article-url-feeder-go: + image: ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go:latest + deploy: + replicas: 6 + build: + <<: *build + context: rss-article-url-feeder-go/. + environment: + - 'NATS_SERVER=nats-server:4222' + restart: always + loki: + container_name: loki + image: grafana/loki:latest + ports: + - '3100:3100' + command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./loki/local-config.yaml:/etc/loki/local-config.yaml + keyword-matcher-go: + deploy: + replicas: 8 + <<: *loki + image: ghcr.io/heussd/nats-news-analysis/keyword-matcher-go:latest + build: + <<: *build + context: keyword-matcher-go/. + volumes: + - type: bind + source: ./keywords.txt + target: /keywords.txt + consistency: cached + read_only: true + environment: + - 'KEYWORDS_FILE:/keywords.txt' + - 'NATS_SERVER=nats-server:4222' + - 'FULLTEXTRSS_SERVER=http://loadbalancer:80' + pocket-integration: + image: ghcr.io/heussd/nats-news-analysis/pocket-integration:latest + build: + <<: *build + context: pocket-integration/. + cache_from: + - ghcr.io/heussd/nats-news-analysis/pocket-integration:latest + environment: + - 'NATS_SERVER=nats-server:4222' + secrets: + - POCKET_ACCESS_TOKEN + - POCKET_CONSUMER_KEY + raindrop-integration: + image: ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest + build: + <<: *build + context: raindrop-integration/. + cache_from: + - ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest + environment: + - 'NATS_SERVER=nats-server:4222' + env_file: + - .env + grafana: + image: grafana/grafana:master + volumes: + - ./grafana/grafana-datasources.yml:/etc/grafana/provisioning/datasources/datasources.yaml + - ./grafana/dashboard-provider.yaml:/etc/grafana/provisioning/dashboards/main.yaml + - ./grafana/dashboards:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + ports: + - '3000:3000' + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus/:/etc/prometheus/ + - ./prom-storage:/usr/local/share/prometheus + command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/usr/local/share/prometheus + natsexporter: + image: natsio/prometheus-nats-exporter + ports: + - '7777:7777' + command: + [ + '-varz', + '-subz', + '-serverz', + '-connz', + '-channelz', + '-jsz', + 'all', + 'http://nats-server:8222', + ] volumes: - rss-cache: + rss-cache: secrets: - POCKET_ACCESS_TOKEN: - file: ./POCKET_ACCESS_TOKEN.txt - POCKET_CONSUMER_KEY: - file: ./POCKET_CONSUMER_KEY.txt + POCKET_ACCESS_TOKEN: + file: ./POCKET_ACCESS_TOKEN.txt + POCKET_CONSUMER_KEY: + file: ./POCKET_CONSUMER_KEY.txt diff --git a/raindrop-integration/Dockerfile b/raindrop-integration/Dockerfile new file mode 100644 index 0000000..2a8de6f --- /dev/null +++ b/raindrop-integration/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.19-bullseye as builder + +ENV GO111MODULE=on + +WORKDIR /app +COPY go.mod . +COPY go.sum . +RUN go mod download + +COPY internal ./internal +COPY cmd ./cmd + +RUN CGO_ENABLED=0 go build -a -ldflags '-s' -o goapp.bin ./cmd + + +# https://github.com/drone/ca-certs/blob/master/Dockerfile +FROM alpine:3.6 as alpine +RUN apk add -U --no-cache ca-certificates + + +FROM scratch +COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /app/goapp.bin /goapp.bin + +CMD [ "./goapp.bin" ] diff --git a/raindrop-integration/Makefile b/raindrop-integration/Makefile new file mode 100644 index 0000000..6ad1071 --- /dev/null +++ b/raindrop-integration/Makefile @@ -0,0 +1,5 @@ +publish: + nats pub match-url "{\"RegexId\": \"OMG\", \"Url\": \"https://www.tagesschau.de/ausland/europa/afrika-russland-ukraine-frieden-100.html\"}" + +stream: + nats stream add match-urls --subjects match-url --storage memory --replicas 1 --retention work --discard old --max-msgs=-1 --max-msgs-per-subject=-1 --max-bytes=-1 --max-age=-1 --allow-rollup --allow-direct diff --git a/raindrop-integration/cmd/main.go b/raindrop-integration/cmd/main.go new file mode 100644 index 0000000..e1d56af --- /dev/null +++ b/raindrop-integration/cmd/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "fmt" + queue "github.com/heussd/nats-raindrop-integration.go/internal/nats" + "github.com/heussd/nats-raindrop-integration.go/internal/raindrop" + "github.com/nats-io/nats.go" + "os" + "os/signal" + "syscall" +) + +type Match struct { + RegexId string + Url string +} + +func main() { + + queue.WithMatchUrls(func(m *nats.Msg) { + var match Match + + if err := json.Unmarshal([]byte(string(m.Data)), &match); err != nil { + fmt.Errorf("failed to unmarshall %w", err) + } + + if ok, _ := raindrop.Add(match.Url); !ok { + fmt.Errorf("received error from raindrop\n") + } else { + fmt.Printf("Added to Raindrop: %s\n ", match.Url) + } + }) + + // https://callistaenterprise.se/blogg/teknik/2019/10/05/go-worker-cancellation/ + termChan := make(chan os.Signal) + signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) + <-termChan // Blocks here until either SIGINT or SIGTERM is received. +} diff --git a/raindrop-integration/docker-compose.yml b/raindrop-integration/docker-compose.yml new file mode 100644 index 0000000..26e0449 --- /dev/null +++ b/raindrop-integration/docker-compose.yml @@ -0,0 +1,15 @@ +services: + nats-server: + image: nats:2.9 + ports: + - '4222:4222' # 4222 is for clients. + - '8222:8222' # 8222 is an HTTP management port for information reporting. + - '6222:6222' # 6222 is a routing port for clustering. + command: '--jetstream' + nats-raindrop--integration: + image: ghcr.io/heussd/nats-news-analysis/raindrop-integration:latest + build: . + environment: + - 'NATS_SERVER=nats-server:4222' + env_file: + - .env diff --git a/raindrop-integration/go.mod b/raindrop-integration/go.mod new file mode 100644 index 0000000..fe7a400 --- /dev/null +++ b/raindrop-integration/go.mod @@ -0,0 +1,21 @@ +module github.com/heussd/nats-raindrop-integration.go + +go 1.19 + +require ( + github.com/nats-io/nats.go v1.22.1 + github.com/shomali11/util v0.0.0-20220717175126-f0771b70947f + github.com/stretchr/testify v1.8.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/nats-io/nats-server/v2 v2.9.10 // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/raindrop-integration/go.sum b/raindrop-integration/go.sum new file mode 100644 index 0000000..861252e --- /dev/null +++ b/raindrop-integration/go.sum @@ -0,0 +1,114 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/kattaris/errhand v0.4.4/go.mod h1:vHHHVGHv8loyr1CP7xoZfNAvclbC/ddls4eU4rKQb9Y= +github.com/kattaris/raindrop-io-api-client v0.3.6 h1:MDlZzbyMlRu8mAa+KOEoVwBg1GNs1s1XvF4vtpPDZuI= +github.com/kattaris/raindrop-io-api-client v0.3.6/go.mod h1:/k1Twinr+W0al+sieuQIqClvxkHX1JFoo9nEjaMpqxY= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= +github.com/nats-io/nats-server/v2 v2.9.10 h1:LMC46Oi9E6BUx/xBsaCVZgofliAqKQzRPU6eKWkN8jE= +github.com/nats-io/nats-server/v2 v2.9.10/go.mod h1:AB6hAnGZDlYfqb7CTAm66ZKMZy9DpfierY1/PbpvI2g= +github.com/nats-io/nats.go v1.22.1 h1:XzfqDspY0RNufzdrB8c4hFR+R3dahkxlpWe5+IWJzbE= +github.com/nats-io/nats.go v1.22.1/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= +github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shomali11/parallelizer v0.0.0-20220717173222-a6776fbf40a9/go.mod h1:QsLM53l8gzX0sQbOjVir85bzOUucuJEF8JgE39wD7w0= +github.com/shomali11/util v0.0.0-20220717175126-f0771b70947f h1:OM0LVaVycWC+/j5Qra7USyCg2sc+shg3KwygAA+pYvA= +github.com/shomali11/util v0.0.0-20220717175126-f0771b70947f/go.mod h1:9POpw/crb6YrseaYBOwraL0lAYy0aOW79eU3bvMxgbM= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/raindrop-integration/internal/config/config.go b/raindrop-integration/internal/config/config.go new file mode 100644 index 0000000..33a1222 --- /dev/null +++ b/raindrop-integration/internal/config/config.go @@ -0,0 +1,25 @@ +package config + +import ( + "fmt" + "github.com/nats-io/nats.go" + "os" +) + +var ( + RaindropToken = GetEnv("RAINDROP_TOKEN", "") + RaindropCollection = GetEnv("RAINDROP_COLLECTION", "") + NatsServer = GetEnv("NATS_SERVER", nats.DefaultURL) + NatsInputQueueName = GetEnv("NATS_INPUT_QUEUE_NAME", "match-urls") + NatsInputQueueSubject = GetEnv("NATS_INPUT_QUEUE_NAME", "match-url") +) + +// GetEnv Taken from https://stackoverflow.com//questions/40326540/how-to-assign-default-value-if-env-var-is-empty#answer-45978733 +func GetEnv(key, fallback string) string { + value, exists := os.LookupEnv(key) + if !exists { + value = fallback + } + fmt.Println(key, "=", value) + return value +} diff --git a/raindrop-integration/internal/nats/nats.go b/raindrop-integration/internal/nats/nats.go new file mode 100644 index 0000000..7ff7e94 --- /dev/null +++ b/raindrop-integration/internal/nats/nats.go @@ -0,0 +1,51 @@ +package nats + +import ( + "fmt" + "github.com/heussd/nats-raindrop-integration.go/internal/config" + "github.com/nats-io/nats.go" + "sync" + "time" +) + +var nc *nats.Conn +var js nats.JetStreamContext +var kv nats.KeyValue + +func init() { + var err error + + for nc == nil { + if nc, err = nats.Connect(config.NatsServer); err != nil { + fmt.Printf("Could not connect to NATS server at %s, because: ", config.NatsServer) + fmt.Println(err) + time.Sleep(5 * time.Second) + } + } + + if js, err = nc.JetStream(nats.PublishAsyncMaxPending(256)); err != nil { + fmt.Println("HERE") + panic(err) + } + +} + +func WithMatchUrls(f func(m *nats.Msg)) { + wg := sync.WaitGroup{} + wg.Add(1) + + var subscribed = false + for !subscribed { + _, err := js.QueueSubscribe(config.NatsInputQueueSubject, config.NatsInputQueueName, f) + if err != nil { + fmt.Printf("Cannot subscribe queue %s subject %s, because: ", config.NatsInputQueueName, config.NatsInputQueueSubject) + fmt.Println(err) + time.Sleep(5 * time.Second) + } else { + subscribed = true + } + } + + wg.Done() + wg.Wait() +} diff --git a/raindrop-integration/internal/nats/nats_test.go b/raindrop-integration/internal/nats/nats_test.go new file mode 100644 index 0000000..aaef013 --- /dev/null +++ b/raindrop-integration/internal/nats/nats_test.go @@ -0,0 +1,16 @@ +package nats + +import ( + "fmt" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPull(t *testing.T) { + WithMatchUrls(func(m *nats.Msg) { + data := string(m.Data) + fmt.Printf("Received from %s\n", data) + assert.Equal(t, "https://www.tagesschau.de/", data) + }) +} diff --git a/raindrop-integration/internal/raindrop/raindrop.go b/raindrop-integration/internal/raindrop/raindrop.go new file mode 100644 index 0000000..f0264c3 --- /dev/null +++ b/raindrop-integration/internal/raindrop/raindrop.go @@ -0,0 +1,56 @@ +package raindrop + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/heussd/nats-raindrop-integration.go/internal/config" + "net/http" + "strconv" +) + +type pleaseParse struct{} + +type collection struct { + Id int `json:"$id"` +} +type postPayload struct { + PleaseParse pleaseParse `json:"pleaseParse"` + Collection collection `json:"collection"` + Link string `json:"link"` +} + +var collectionId, _ = strconv.Atoi(config.RaindropCollection) + +func Add(url string) (ok bool, err error) { + + payload := postPayload{ + PleaseParse: pleaseParse{}, + Collection: collection{ + Id: collectionId, + }, + Link: url, + } + + var jsonBytes []byte + if jsonBytes, err = json.Marshal(payload); err != nil { + return false, fmt.Errorf("cannot unmarshall: %w", err) + } + + req, err := http.NewRequest( + "POST", + "https://api.raindrop.io/rest/v1/raindrop?=", + bytes.NewBuffer(jsonBytes)) + + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", "Bearer "+config.RaindropToken) + + var res *http.Response + if res, err = http.DefaultClient.Do(req); err != nil { + return false, fmt.Errorf("error doing request: %w", err) + } + + defer res.Body.Close() + + return res.StatusCode == 200, nil +} diff --git a/raindrop-integration/internal/raindrop/raindrop_test.go b/raindrop-integration/internal/raindrop/raindrop_test.go new file mode 100644 index 0000000..1f8250f --- /dev/null +++ b/raindrop-integration/internal/raindrop/raindrop_test.go @@ -0,0 +1,13 @@ +package raindrop + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAdd(t *testing.T) { + res, err := Add("https://www.tagesschau.de/wirtschaft/konjunktur/inflationsrate-juli-verbraucherpreise-100.html") + assert.NoError(t, err) + assert.Equal(t, true, res) + +}