Add opensearch-csv-exporter

Initial commit

.gitignore

@@ -0,0 +1 @@
+ca.crt

Dockerfile

@@ -0,0 +1,4 @@
+FROM gcr.io/distroless/static-debian11:nonroot
+COPY opensearch-csv-exporter /
+USER nonroot
+ENTRYPOINT ["/opensearch-csv-exporter"]

Makefile

@@ -0,0 +1,48 @@
+.PHONY:	build push run
+
+IMAGE = quay.io/fortnox/opensearch-csv-exporter
+# supply when running make: make all VERSION=1.0.0
+#VERSION = 0.0.1 
+
+build:
+	CGO_ENABLED=0 GOOS=linux go build ./cmd/opensearch-csv-exporter/
+
+docker: build
+	docker build --pull --rm -t $(IMAGE):$(VERSION) .
+	rm opensearch-csv-exporter
+
+push:
+	docker push $(IMAGE):$(VERSION)
+
+all: build docker push
+
+run:
+	docker run -i --env-file=.env --rm -p 8080:8080 -t $(IMAGE):$(VERSION)
+
+test: imports
+	go test ./...
+
+cover:
+	@echo Running coverage
+	go install github.com/wadey/gocovmerge
+	$(eval PKGS := $(shell go list ./... | grep -v /vendor/))
+	$(eval PKGS_DELIM := $(shell echo $(PKGS) | sed -e 's/ /,/g'))
+	go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test -test.v -test.timeout=120s -covermode=count -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile -coverpkg $(PKGS_DELIM) {{.ImportPath}}{{end}}' $(PKGS) | xargs -I {} bash -c {}
+	gocovmerge `ls *.coverprofile` > cover.out
+	rm *.coverprofile
+
+cover-html: cover
+	go tool cover -html cover.out
+cover-test: cover
+	go install github.com/jonaz/gototcov
+	gototcov -f cover.out -limit 80 -ignore-zero
+
+localrun:
+	bash -c "env `grep -Ev '^#' .env | xargs` go run cmd/opensearch-csv-exporter/*.go"
+
+# To format your files according to goimports you can run: `goimports -w .`
+# or setup your ide to do it for you
+imports: SHELL:=/bin/bash
+imports:
+	go install golang.org/x/tools/cmd/goimports@latest
+	ASD=$$(goimports -l . 2>&1); test -z "$$ASD" || (echo "Code is not formatted correctly according to goimports!  $$ASD" && exit 1)

README.md

@@ -0,0 +1,102 @@
+# opensearch-csv-exporter
+
+## Overview
+
+The OpenSearch CSV Export API allows you to export data from an OpenSearch index to CSV format. By making a POST request to the API endpoint, you can specify the date range, query, and columns to export. The exported data will be compressed and saved to a file.
+
+
+## Configuration Options
+
+The `opensearch-csv-exporter` utility provides various configuration options that can be used to customize its behavior. The following command-line flags can be used to set these options:
+
+| Flag                          | Description                                           |
+|-------------------------------|-------------------------------------------------------|
+| -log-format                   | Change the log format. (default: text)                |
+| -log-formatter                | Change the log formatter. (default: <nil>)            |
+| -log-level                    | Change the log level.                                 |
+| -opensearch-addresses         | Change the OpenSearch addresses. (default: [])        |
+| -opensearch-cacertfilepath    | Change the OpenSearch CA certificate file path.       |
+| -opensearch-indices           | Change the OpenSearch indices. (default: [])          |
+| -port                         | Change the port. (default: 8080)                      |
+
+These options can also be set using environment variables. The generated environment variables for each option are:
+
+| Environment Variable           | Description                                     |
+|--------------------------------|-------------------------------------------------|
+| CONFIG_LOG_FORMAT              | Change the log format.                          |
+| CONFIG_LOG_FORMATTER           | Change the log formatter.                       |
+| CONFIG_LOG_LEVEL               | Change the log level.                           |
+| CONFIG_OPENSEARCH_ADDRESSES    | Change the OpenSearch addresses.                |
+| CONFIG_OPENSEARCH_CACERTFILEPATH | Change the OpenSearch CA certificate file path. |
+| CONFIG_OPENSEARCH_INDICES       | Change the OpenSearch indices.                  |
+| CONFIG_PORT                     | Change the port.                                |
+
+## Endpoint
+
+```
+POST /api/opensearch/csv-export-v1
+```
+
+## Request Parameters
+
+The request should include the following parameters:
+
+| Parameter   | Type    | Description                            |
+|-------------|---------|----------------------------------------|
+| fromDate    | string  | The start date for the export (format: "YYYY-MM-DD"). |
+| toDate      | string  | The end date for the export (format: "YYYY-MM-DD").   |
+| query       | string  | The query to filter the documents.                     |
+| columns     | array   | The list of columns to include in the CSV.              |
+
+## Request Headers
+
+The request should include the following header for authentication:
+
+```
+Authorization: Basic base64(username:password)
+```
+
+Replace `username` and `password` with your actual credentials, encoded in Base64.
+
+## Response
+
+The response will be a compressed CSV file containing the exported data. The file will be downloaded with the filename `test.csv.gz` in this example.
+
+## Example
+
+### cURL Command
+
+```bash
+curl -XPOST localhost:8080/api/opensearch/csv-export-v1 -u username:password -d '{"fromDate":"2023-06-13","toDate":"2023-06-14","query":"MY_QUERY","columns":["MY_CUSTOM_CSV_COLUMN"]}' -o test.csv.gz
+```
+
+### Example Request
+
+```http
+POST /api/opensearch/csv-export-v1 HTTP/1.1
+Host: localhost:8080
+Authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ=
+Content-Type: application/json
+Content-Length: 101
+
+{
+  "fromDate": "2023-06-13",
+  "toDate": "2023-06-14",
+  "query": "MY_QUERY",
+  "columns": ["MY_CUSTOM_CSV_COLUMN"]
+}
+```
+
+### Example Response
+
+The response will be a file named `test.csv.gz`, containing the exported data.
+
+## Error Handling
+
+If an error occurs during the export process, the API will return an appropriate HTTP status code along with an error message in the response body.
+
+## Authentication
+
+The API uses Basic Authentication for authentication purposes. The `Authorization` header should contain the Base64 encoded username and password.
+
+Please note that it is highly recommended to use secure connections (e.g., HTTPS) when using this API in a production environment to protect sensitive information.

cmd/opensearch-csv-exporter/csv.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"compress/gzip"
+	"encoding/csv"
+	"fmt"
+	"io"
+
+	"github.com/tidwall/gjson"
+)
+
+type CSV struct {
+	writer *csv.Writer
+
+	gzip    *gzip.Writer
+	columns []string
+}
+
+func NewCSV(columns []string, writer io.Writer) (*CSV, error) {
+
+	g := gzip.NewWriter(writer)
+
+	c := &CSV{
+		gzip:    g,
+		writer:  csv.NewWriter(g),
+		columns: append([]string{"@timestamp", "message"}, columns...),
+	}
+	c.writer.Comma = ';'
+	err := c.writer.Write(c.columns)
+	if err != nil {
+		return c, fmt.Errorf("failed to write header to csv writer: %w", err)
+	}
+	return c, nil
+}
+
+func (csv *CSV) Close() error {
+	csv.writer.Flush()
+	err := csv.writer.Error()
+	if err != nil {
+		return fmt.Errorf("failed to close csv writer: %w", err)
+	}
+
+	err = csv.gzip.Flush()
+	if err != nil {
+		return fmt.Errorf("failed to flush gzip writer: %w", err)
+	}
+	err = csv.gzip.Close()
+	if err != nil {
+		return fmt.Errorf("failed to close gzip writer: %w", err)
+	}
+
+	return nil
+}
+
+func (csv *CSV) write(doc []byte) error {
+	var record []string
+	for _, column := range csv.columns {
+		data := gjson.GetBytes(doc, column).Value()
+		if data == nil {
+			data = ""
+		}
+		record = append(record, fmt.Sprintf("%v", data))
+	}
+	err := csv.writer.Write(record)
+	if err != nil {
+		return fmt.Errorf("failed to write record to csv: %s", err)
+	}
+	return nil
+}