diff --git a/.github/workflows/docker-build-and-push.yml b/.github/workflows/docker-build-and-push.yml index f972a29..c3df1a5 100644 --- a/.github/workflows/docker-build-and-push.yml +++ b/.github/workflows/docker-build-and-push.yml @@ -1,41 +1,35 @@ name: Docker build & push on: - push: - branches: - - 'main' - paths-ignore: - - '**.md' - - 'docker-compose.yml' - - 'architecture.drawio' - - 'architecture.drawio.svg' - - '**.png' + push: + branches: + - 'main' + paths-ignore: + - '**.md' + - 'architecture.drawio' + - 'architecture.drawio.svg' + - '**.png' jobs: - bake: - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - # Add support for more platforms with QEMU (optional) - # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log in to the GitHub Container registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/bake-action@v2 - with: - files: | - "docker-compose.yml" - push: true \ No newline at end of file + bake: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log in to the GitHub Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/bake-action@v2 + with: + files: | + "compose.yml" + push: true diff --git a/Makefile b/Makefile index b126412..3cab911 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ start: logs: - docker-compose logs -f keyword-matcher-go + docker compose logs -f keyword-matcher-go stop: diff --git a/docker-compose.yml b/compose.yml similarity index 99% rename from docker-compose.yml rename to compose.yml index 14dd67d..80981db 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -11,7 +11,7 @@ x-build: &build services: nats-server: - image: docker.io/nats:2.9 + image: docker.io/nats:2.10 ports: - '4222:4222' # 4222 is for clients. - '8222:8222' # 8222 is an HTTP management port for information reporting. diff --git a/index.en.md b/index.en.md index bc75500..1924208 100644 --- a/index.en.md +++ b/index.en.md @@ -17,26 +17,26 @@ The system has three NATS queues: ## Involved services -All services are orchestrated and scaled with `docker-compose.yml`. +All services are orchestrated and scaled with `compose.yml`. ### Custom services -- [ghcr.io/heussd/nats-news-analysis/rss-feed-feeder](https://ghcr.io/heussd/nats-news-analysis/rss-feed-feeder) - **Bash** - Feeds rss feed urls from a text file. -- [ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go](https://ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go) - **Golang** - Feeds news articles from RSS feeds. -- [ghcr.io/heussd/nats-news-analysis/keyword-matcher-go](https://ghcr.io/heussd/nats-news-analysis/keyword-matcher-go) - **Golang** - Matches against keywords list. -- [ghcr.io/heussd/nats-news-analysis/pocket-integration](https://ghcr.io/heussd/nats-news-analysis/pocket-integration) - **Python** - Publishes matches on [getpocket.com](https://getpocket.com/). -- [ghcr.io/heussd/nats-news-analysis/raindrop-integration](https://ghcr.io/heussd/nats-news-analysis/raindrop-integration) - **Golang** - Publishes matches on [raindrop.io](https://raindrop.io/). +- [ghcr.io/heussd/nats-news-analysis/rss-feed-feeder](https://ghcr.io/heussd/nats-news-analysis/rss-feed-feeder) - **Bash** - Feeds rss feed urls from a text file. +- [ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go](https://ghcr.io/heussd/nats-news-analysis/rss-article-url-feeder-go) - **Golang** - Feeds news articles from RSS feeds. +- [ghcr.io/heussd/nats-news-analysis/keyword-matcher-go](https://ghcr.io/heussd/nats-news-analysis/keyword-matcher-go) - **Golang** - Matches against keywords list. +- [ghcr.io/heussd/nats-news-analysis/pocket-integration](https://ghcr.io/heussd/nats-news-analysis/pocket-integration) - **Python** - Publishes matches on [getpocket.com](https://getpocket.com/). +- [ghcr.io/heussd/nats-news-analysis/raindrop-integration](https://ghcr.io/heussd/nats-news-analysis/raindrop-integration) - **Golang** - Publishes matches on [raindrop.io](https://raindrop.io/). ### Third party services -- [docker.io/heussd/fivefilters-full-text-rss](https://hub.docker.com/r/heussd/fivefilters-full-text-rss) - Retrieves full text of web pages. -- [docker.io/nats](https://hub.docker.com/_/nats) - Event queue, key-value store and deduplication. -- [NGINX](https://www.nginx.com/) - Simple load balancer / reverse proxy -- [Prometheus NATS Exporter](https://github.com/nats-io/prometheus-nats-exporter) - Make NATS metrics available to Prometheus -- [Prometheus](https://prometheus.io/) - Metrics & monitoring -- [Grafana Loki](https://grafana.com/oss/loki/) - Log aggregation -- [Grafana](https://grafana.com/grafana/) - Dashboard for metrics and stats -- [getpocket.com API](https://getpocket.com/developer/) - "Read it later" online service. +- [docker.io/heussd/fivefilters-full-text-rss](https://hub.docker.com/r/heussd/fivefilters-full-text-rss) - Retrieves full text of web pages. +- [docker.io/nats](https://hub.docker.com/_/nats) - Event queue, key-value store and deduplication. +- [NGINX](https://www.nginx.com/) - Simple load balancer / reverse proxy +- [Prometheus NATS Exporter](https://github.com/nats-io/prometheus-nats-exporter) - Make NATS metrics available to Prometheus +- [Prometheus](https://prometheus.io/) - Metrics & monitoring +- [Grafana Loki](https://grafana.com/oss/loki/) - Log aggregation +- [Grafana](https://grafana.com/grafana/) - Dashboard for metrics and stats +- [getpocket.com API](https://getpocket.com/developer/) - "Read it later" online service. ## Message queue for scaling diff --git a/keyword-matcher-go/Dockerfile b/keyword-matcher-go/Dockerfile index 5e4d0e6..6732956 100644 --- a/keyword-matcher-go/Dockerfile +++ b/keyword-matcher-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-bullseye as builder +FROM golang:1.22-bullseye as builder ENV GO111MODULE=on diff --git a/keyword-matcher-go/Makefile b/keyword-matcher-go/Makefile index 6b6fd8a..86d6e9e 100644 --- a/keyword-matcher-go/Makefile +++ b/keyword-matcher-go/Makefile @@ -11,4 +11,4 @@ watch-matches: stream: - nats stream add article-urls --subjects article_url --storage memory --replicas 1 --retention work --discard old --max-msgs=-1 --max-msgs-per-subject=-1 --max-bytes=-1 --max-age=-1 --allow-rollup --allow-direct + nats stream add article-urls --subjects article-url --storage memory --replicas 1 --retention work --discard old --max-msgs=-1 --max-msgs-per-subject=-1 --max-bytes=-1 --max-age=-1 --allow-rollup --allow-direct diff --git a/keyword-matcher-go/cmd/main_test.go b/keyword-matcher-go/cmd/main_test.go index 8317d4d..131c315 100644 --- a/keyword-matcher-go/cmd/main_test.go +++ b/keyword-matcher-go/cmd/main_test.go @@ -10,6 +10,7 @@ import ( func TestNoMoreHTML(t *testing.T) { var fulltext = fulltextrss.RetrieveFullText("https://www.tagesschau.de/wissen/technologie/agri-photovoltaik-103.html") + fulltext.Content = fulltext.Content + "

" assert.Equal(t, true, strings.Contains(fulltext.Content, "

")) diff --git a/keyword-matcher-go/compose.yml b/keyword-matcher-go/compose.yml new file mode 100644 index 0000000..ea7f220 --- /dev/null +++ b/keyword-matcher-go/compose.yml @@ -0,0 +1,28 @@ +services: + nats-news-keyword-matcher-go: + image: ghcr.io/heussd/nats-news-keyword-matcher-go:main + build: . + volumes: + - type: bind + source: ./internal/keywords/keywords.txt + target: /keywords.txt + consistency: cached + read_only: true + environment: + - 'KEYWORDS_FILE:/keywords.txt' + - 'NATS_SERVER=nats-server:4222' + - 'FULLTEXTRSS_SERVER=http://fullfeedrss:80' + nats-server: + image: nats:2.10 + ports: + - '4222:4222' # 4222 is for clients. + - '8222:8222' # 8222 is an HTTP management port for information reporting. + - '6222:6222' # 6222 is a routing port for clustering. + command: '--jetstream' + fullfeedrss: + image: 'heussd/fivefilters-full-text-rss:latest' + environment: + # Leave empty to disable admin section + - FTR_ADMIN_PASSWORD= + ports: + - '80:80' diff --git a/keyword-matcher-go/docker-compose.yml b/keyword-matcher-go/docker-compose.yml deleted file mode 100644 index 49510bb..0000000 --- a/keyword-matcher-go/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: "3.7" - -services: - nats-news-keyword-matcher-go: - image: ghcr.io/heussd/nats-news-keyword-matcher-go:main - build: . - volumes: - - type: bind - source: ./internal/keywords/keywords.txt - target: /keywords.txt - consistency: cached - read_only: true - environment: - - "KEYWORDS_FILE:/keywords.txt" - - "NATS_SERVER=nats-server:4222" - - "FULLTEXTRSS_SERVER=http://fullfeedrss:80" - nats-server: - image: nats:2.8 - ports: - - "4222:4222" # 4222 is for clients. - - "8222:8222" # 8222 is an HTTP management port for information reporting. - - "6222:6222" # 6222 is a routing port for clustering. - command: "--jetstream" - fullfeedrss: - image: "heussd/fivefilters-full-text-rss:latest" - environment: - # Leave empty to disable admin section - - FTR_ADMIN_PASSWORD= - ports: - - "80:80" diff --git a/keyword-matcher-go/go.mod b/keyword-matcher-go/go.mod index d7b3a5c..863cd69 100644 --- a/keyword-matcher-go/go.mod +++ b/keyword-matcher-go/go.mod @@ -1,32 +1,31 @@ module github.com/heussd/nats-news-keyword-matcher.go -go 1.19 +go 1.22 + +toolchain go1.22.2 require ( - github.com/dlclark/regexp2 v1.7.0 - github.com/microcosm-cc/bluemonday v1.0.21 - github.com/nats-io/nats.go v1.28.0 - github.com/rs/zerolog v1.29.0 + github.com/dlclark/regexp2 v1.11.0 + github.com/microcosm-cc/bluemonday v1.0.26 + github.com/nats-io/nats.go v1.34.1 + github.com/rs/zerolog v1.32.0 github.com/stretchr/testify v1.8.1 ) require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/gorilla/css v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.18 // indirect - github.com/nats-io/nats-server/v2 v2.9.23 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/keyword-matcher-go/go.sum b/keyword-matcher-go/go.sum index 6ada852..434df10 100644 --- a/keyword-matcher-go/go.sum +++ b/keyword-matcher-go/go.sum @@ -1,50 +1,41 @@ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= -github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= -github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= -github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= -github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= -github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= -github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= +github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= -github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0= +github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -52,22 +43,15 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pocket-integration/.github/workflows/container-build-and-push.yml b/pocket-integration/.github/workflows/container-build-and-push.yml deleted file mode 100644 index 879ac06..0000000 --- a/pocket-integration/.github/workflows/container-build-and-push.yml +++ /dev/null @@ -1,51 +0,0 @@ -# https://docs.github.com/en/actions/publishing-packages/publishing-docker-images -name: Container Build and Push - -on: - push: - branches: - - main - - master - paths-ignore: - - '**.md' - - 'docker-compose.yml' - schedule: - - cron: '0 13 * * 0' - - -jobs: - build: - runs-on: ubuntu-latest - steps: - - - name: Check out the repo - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Log in to the GitHub Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: | - ghcr.io/${{ github.repository }} - - - name: Build and push Docker images - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - platforms: linux/amd64,linux/arm64,linux/arm/v7 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/rss-article-url-feeder-go/Dockerfile b/rss-article-url-feeder-go/Dockerfile index 2a8de6f..bdecdd0 100644 --- a/rss-article-url-feeder-go/Dockerfile +++ b/rss-article-url-feeder-go/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19-bullseye as builder +FROM golang:1.22-bullseye as builder ENV GO111MODULE=on diff --git a/rss-article-url-feeder-go/Makefile b/rss-article-url-feeder-go/Makefile new file mode 100644 index 0000000..b669308 --- /dev/null +++ b/rss-article-url-feeder-go/Makefile @@ -0,0 +1,18 @@ +stream: + nats stream add feed-urls \ + --subjects feed-url \ + --storage memory \ + --replicas 1 \ + --retention work \ + --discard old \ + --max-msgs=-1 \ + --max-msgs-per-subject=-1 \ + --max-bytes=-1 \ + --max-age=-1 \ + --allow-rollup \ + --allow-direct \ + --max-msg-size=-1 \ + --dupe-window=2m0s \ + --no-deny-delete \ + --no-deny-purge + diff --git a/rss-article-url-feeder-go/docker-compose.yml b/rss-article-url-feeder-go/compose.yml similarity index 86% rename from rss-article-url-feeder-go/docker-compose.yml rename to rss-article-url-feeder-go/compose.yml index ebc1d61..65dd8f9 100644 --- a/rss-article-url-feeder-go/docker-compose.yml +++ b/rss-article-url-feeder-go/compose.yml @@ -1,19 +1,17 @@ -version: "3.7" - services: rss-article-url-feeder-go: image: ghcr.io/heussd/rss-article-url-feeder-go:main build: . volumes: - type: bind - source: ./internal/urls/urls.txt + source: urls.txt target: /urls.txt consistency: cached read_only: true environment: - "NATS_SERVER=nats-server:4222" nats-server: - image: nats:2.8 + image: nats:2.10 ports: - "4222:4222" # 4222 is for clients. - "8222:8222" # 8222 is an HTTP management port for information reporting. diff --git a/rss-article-url-feeder-go/go.mod b/rss-article-url-feeder-go/go.mod index d81c6c6..45c332e 100644 --- a/rss-article-url-feeder-go/go.mod +++ b/rss-article-url-feeder-go/go.mod @@ -1,10 +1,10 @@ module github.com/heussd/rss-article-url-feeder.go -go 1.19 +go 1.22 require ( github.com/SlyMarbo/rss v1.0.5 - github.com/nats-io/nats.go v1.28.0 + github.com/nats-io/nats.go v1.34.1 github.com/shomali11/util v0.0.0-20220717175126-f0771b70947f github.com/stretchr/testify v1.8.1 ) @@ -12,16 +12,13 @@ require ( require ( github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/kr/pretty v0.1.0 // indirect - github.com/nats-io/nats-server/v2 v2.9.23 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/sys v0.19.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/rss-article-url-feeder-go/go.sum b/rss-article-url-feeder-go/go.sum index 288a201..9d3da74 100644 --- a/rss-article-url-feeder-go/go.sum +++ b/rss-article-url-feeder-go/go.sum @@ -5,25 +5,17 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIg github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak= -github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU= -github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0= -github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= -github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4= +github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -39,16 +31,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/rss-feed-feeder/Dockerfile b/rss-feed-feeder/Dockerfile index d036fa7..772a4db 100644 --- a/rss-feed-feeder/Dockerfile +++ b/rss-feed-feeder/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:latest +FROM golang:1.22 RUN go install github.com/nats-io/natscli/nats@latest diff --git a/rss-feed-feeder/compose.yml b/rss-feed-feeder/compose.yml new file mode 100644 index 0000000..20b7ded --- /dev/null +++ b/rss-feed-feeder/compose.yml @@ -0,0 +1,16 @@ +services: + nats-server: + image: nats:2.10 + ports: + - '4222:4222' # 4222 is for clients. + - '8222:8222' # 8222 is an HTTP management port for information reporting. + - '6222:6222' # 6222 is a routing port for clustering. + command: '--jetstream' + + rss-feed-feeder: + image: rss-feed-feeder + build: . + environment: + - NATS_SERVER=http://nats-server:4222 + volumes: + - ./urls.txt:/urls.txt diff --git a/rss-feed-feeder/docker-compose.yml b/rss-feed-feeder/docker-compose.yml deleted file mode 100644 index f6cbb29..0000000 --- a/rss-feed-feeder/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.7" - -services: - nats-server: - image: nats:2.8 - ports: - - "4222:4222" # 4222 is for clients. - - "8222:8222" # 8222 is an HTTP management port for information reporting. - - "6222:6222" # 6222 is a routing port for clustering. - command: "--jetstream" - - rss-feed-feeder: - image: rss-feed-feeder - build: . - environment: - - NATS_SERVER=http://nats-server:4222 - volumes: - - ./urls.txt:/urls.txt diff --git a/rss-feed-feeder/feed.sh b/rss-feed-feeder/feed.sh index 461b21f..ac64566 100755 --- a/rss-feed-feeder/feed.sh +++ b/rss-feed-feeder/feed.sh @@ -6,7 +6,7 @@ set -o nounset sleep 5 nats stream add feed-urls \ - --server=$NATS_SERVER \ + --server="$NATS_SERVER" \ --subjects feed-url \ --storage memory \ --replicas 1 \ @@ -25,7 +25,7 @@ nats stream add feed-urls \ nats stream info feed-urls \ - --server=$NATS_SERVER + --server="$NATS_SERVER" while true; do while read feed; do @@ -34,7 +34,7 @@ while true; do ;; *) nats pub feed-url "$feed" \ - --server=$NATS_SERVER \ + --server="$NATS_SERVER" \ --header="Nats-Msg-Id:$feed" # De-duplication works with Msg Id esac done