-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #36 from terratensor/rssfeed
Rssfeed
- Loading branch information
Showing
13 changed files
with
649 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| # syntax = docker/dockerfile:1.2 | ||
| FROM golang:latest AS builder | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY go.mod . | ||
| COPY go.sum . | ||
| RUN go mod download | ||
|
|
||
| COPY . . | ||
|
|
||
| RUN --mount=type=cache,target=/root/.cache/go-build \ | ||
| CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o ./feed-server ./cmd/server | ||
|
|
||
| # 2 | ||
|
|
||
| FROM scratch | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY --from=builder /app/feed-server /app/feed-server | ||
| COPY --from=builder /app/static /app/static | ||
| COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ | ||
| COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo | ||
| ENV TZ=Europe/Moscow | ||
|
|
||
| EXPOSE 8000 | ||
|
|
||
| CMD ["./feed-server"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| # syntax = docker/dockerfile:1.2 | ||
| FROM golang:latest AS builder | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY go.mod . | ||
| COPY go.sum . | ||
| RUN go mod download | ||
|
|
||
| COPY . . | ||
|
|
||
| ARG INDEX_NOW_KEY | ||
| ENV INDEX_NOW_KEY=${INDEX_NOW_KEY} | ||
|
|
||
| RUN --mount=type=cache,target=/root/.cache/go-build \ | ||
| CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o ./feed-static-generator ./cmd/rssfeed | ||
|
|
||
| # 2 | ||
|
|
||
| FROM scratch | ||
|
|
||
| WORKDIR /app | ||
|
|
||
| COPY --from=builder /app/feed-static-generator /app/feed-static-generator | ||
| COPY --from=builder /app/static /app/static | ||
| COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ | ||
| # Корневой сертификат удостоверяющего центра (УЦ) Минцифры | ||
| COPY --from=builder /app/certs/rootca_ssl_rsa2022.cer /etc/ssl/certs/ | ||
| COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo | ||
| ENV TZ=Europe/Moscow | ||
|
|
||
| ARG MANTICORE_INDEX | ||
| ENV MANTICORE_INDEX=${MANTICORE_INDEX} | ||
| ARG GENERATOR_DELAY | ||
| ENV GENERATOR_DELAY=${GENERATOR_DELAY} | ||
|
|
||
| CMD ["./feed-static-generator"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/terratensor/feed-parser/internal/entities/feed" | ||
| "github.com/terratensor/feed-parser/internal/rssfeed" | ||
| "github.com/terratensor/feed-parser/internal/storage/manticore" | ||
| "log" | ||
| "os" | ||
| "os/signal" | ||
| "sync" | ||
| "time" | ||
| ) | ||
|
|
||
| var authorMap = map[int]string{ | ||
| 1: "Президент Российской Федерации", | ||
| 2: "Министерство иностранных дел Российской Федерации", | ||
| 3: "Министерство обороны Российской Федерации", | ||
| } | ||
|
|
||
| func main() { | ||
|
|
||
| ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt) | ||
|
|
||
| delayStr := os.Getenv("GENERATOR_DELAY") | ||
| delay, err := time.ParseDuration(delayStr) | ||
| if err != nil { | ||
| delay = 15 * time.Minute | ||
| } | ||
|
|
||
| index := os.Getenv("MANTICORE_INDEX") | ||
| if index == "" { | ||
| index = "feed" | ||
| } | ||
|
|
||
| manticoreClient, err := manticore.New(index) | ||
| if err != nil { | ||
| log.Printf("failed to initialize manticore client: %v", err) | ||
| os.Exit(1) | ||
| } | ||
| entries := feed.NewFeedStorage(manticoreClient) | ||
|
|
||
| duration := 24 * 8 * time.Hour | ||
|
|
||
| wg := &sync.WaitGroup{} | ||
|
|
||
| for { | ||
| wg.Add(1) | ||
| go generateFeed(ctx, entries, duration, wg) | ||
| wg.Wait() | ||
| time.Sleep(delay) | ||
| } | ||
| } | ||
|
|
||
| func generateFeed(ctx context.Context, entries *feed.Entries, duration time.Duration, wg *sync.WaitGroup) { | ||
|
|
||
| defer wg.Done() | ||
|
|
||
| limit := entries.Storage.CalculateLimitCount(duration) | ||
| ch, err := entries.Find(ctx, duration) | ||
| if err != nil { | ||
| log.Fatalf("failed to find all entries: %v", err) | ||
| } | ||
|
|
||
| svoddFeed := &rssfeed.RssFeed{ | ||
| Title: "Поиск по сайтам Кремля, МИД и Минобороны", | ||
| Link: "https://rss.feed.svodd.ru", | ||
| Description: "Поиск по сайтам Президента России, Министерства иностранных дел Российской Федерации, Министерство обороны Российской Федерации", | ||
| } | ||
| count := 0 | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| return | ||
| case e, ok := <-ch: | ||
| if !ok { | ||
| return | ||
| } | ||
|
|
||
| link := makeEntryUrl(e.Url, e.Language) | ||
|
|
||
| item := &rssfeed.RssItem{ | ||
| Title: e.Title, | ||
| Link: link, | ||
| PubDate: rssfeed.AnyTimeFormat(time.RFC1123Z, *e.Published), | ||
| Author: populateAuthorField(e.Author, e.ResourceID), | ||
| Content: e.Content, | ||
| Description: e.Summary, | ||
| } | ||
|
|
||
| svoddFeed.Add(item) | ||
| count++ | ||
| } | ||
| if count >= limit { | ||
| break | ||
| } | ||
| } | ||
|
|
||
| file, err := os.Create("./static/rss.xml") | ||
| if err != nil { | ||
| log.Printf("failed to create file: %v", err) | ||
| } | ||
| defer file.Close() | ||
|
|
||
| err = rssfeed.WriteXML(svoddFeed, file) | ||
| if err != nil { | ||
| log.Printf("failed to write xml: %v", err) | ||
| } | ||
|
|
||
| log.Printf("🚩 Создан rss.xml. Всего записей: %d\n", count) | ||
| } | ||
|
|
||
| func populateAuthorField(author string, resourceID int) string { | ||
| if val, ok := authorMap[resourceID]; ok { | ||
| if resourceID == 3 && author != "" { | ||
| return author | ||
| } | ||
| return val | ||
| } | ||
| return author | ||
| } | ||
|
|
||
| func makeEntryUrl(url string, language string) string { | ||
| base := "https://feed.svodd.ru" | ||
| if language != "ru" && language != "" { | ||
| base += "/" + language | ||
| } | ||
| return fmt.Sprintf("%s/entry?url=%v", base, url) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "fmt" | ||
| "log" | ||
| "log/slog" | ||
| "net/http" | ||
| "os" | ||
| "time" | ||
| ) | ||
|
|
||
| func handlerRssFeed(w http.ResponseWriter, r *http.Request) { | ||
|
|
||
| streamXmlBytes, err := os.ReadFile("./static/rss.xml") | ||
|
|
||
| if err != nil { | ||
| log.Printf("error reading file: %v\n", err) | ||
| } | ||
|
|
||
| b := bytes.NewBuffer(streamXmlBytes) | ||
|
|
||
| w.Header().Set("Content-type", "application/xml") | ||
|
|
||
| if _, err := b.WriteTo(w); err != nil { | ||
| log.Printf("error writing response: %v\n", err) | ||
| fmt.Fprintf(w, "%s", err) | ||
| } | ||
| } | ||
|
|
||
| func main() { | ||
| if tz := os.Getenv("TZ"); tz != "" { | ||
| var err error | ||
| time.Local, err = time.LoadLocation(tz) | ||
| if err != nil { | ||
| log.Printf("error loading location '%s': %v\n", tz, err) | ||
| } | ||
| } | ||
|
|
||
| // output current time zone | ||
| tnow := time.Now() | ||
| tz, _ := tnow.Zone() | ||
| log.Printf("Local time zone %s. Server started at %s", tz, | ||
| tnow.Format("2006-01-02T15:04:05.000 MST")) | ||
| log.Println("Listening on port 8000") | ||
|
|
||
| logger := slog.New( | ||
| slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}), | ||
| ) | ||
|
|
||
| mux := http.NewServeMux() | ||
| handler := http.HandlerFunc(handlerRssFeed) | ||
| mux.Handle("/rss.xml", logMiddleware(handler, logger)) | ||
| log.Fatal(http.ListenAndServe(":8000", mux)) | ||
| } | ||
|
|
||
| func logMiddleware(next http.Handler, logger *slog.Logger) http.Handler { | ||
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| logger.Info( | ||
| "request", | ||
| slog.String("method", r.Method), | ||
| slog.String("URL", r.URL.String()), | ||
| slog.String("proto", r.Proto), | ||
| slog.String("host", r.Host), | ||
| slog.String("remote", r.RemoteAddr), | ||
| slog.String("requestURI", r.RequestURI), | ||
| slog.String("userAgent", r.UserAgent()), | ||
| ) | ||
| next.ServeHTTP(w, r) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.