docker: wire in log level. use local dockerfile in test

README.md

@@ -1,12 +1,11 @@
 # Project Description
 Open Source Feature Flag system.
-Dorkly is a git-based open-source feature flag backend for [LaunchDarkly](https://launchdarkly.com/features/feature-flags/) SDKs.
-This project implements a small but valuable subset of LaunchDarkly's featureset. It is not a drop-in replacement for LaunchDarkly. 
-It is intended to be a simple, open-source alternative for small projects that don't need all of LaunchDarkly's features or are unable to use LaunchDarkly's backend for compliance reasons.
+Dorkly is a git-based open source feature flag backend for [LaunchDarkly](https://launchdarkly.com/features/feature-flags/)'s open source SDKs.
 It consists of:
-1. A GitHub Action) that reads in human-friendly yaml files, and converts them to an archive format consumed by:
-2. A Docker image that runs a backend service that serves the flags to your application. This backend service is a [very thin wrapper](docker/Dockerfile) around the [ld-relay](https://docs.launchdarkly.com/sdk/relay-proxy) appliance running in [offline mode](https://docs.launchdarkly.com/sdk/relay-proxy/offline)
-3. A [Terraform module](https://registry.terraform.io/modules/dorklyorg/dorkly-flags/aws/latest) that provisions the backend service on AWS and sets up the necessary permissions, and a GitHub repository to store the flags.
+1. Feature flag definitions stored as yaml in a GitHub repository
+2. A GitHub Action that reads in human-friendly yaml files, and converts them to an archive format consumed by:
+3. A Docker image that runs a backend service that serves the flags to your application. This backend service is a [very thin wrapper](docker/Dockerfile) around the [ld-relay](https://docs.launchdarkly.com/sdk/relay-proxy) appliance running in [offline mode](https://docs.launchdarkly.com/sdk/relay-proxy/offline)
+4. A [Terraform module](https://registry.terraform.io/modules/dorklyorg/dorkly-flags/aws/latest) that provisions the backend service on AWS and sets up the necessary permissions, and a GitHub repository to store the flags.
 
 # Getting Started: One time setup
 ## First steps
@@ -67,6 +66,8 @@ In your newly created GitHub repo you'll notice some example yaml files under th
 ## Relevant Links
 [Feature Flags](https://launchdarkly.com/features/feature-flags/)
 [Relay Proxy Configuration](https://docs.launchdarkly.com/sdk/features/relay-proxy-configuration/proxy-mode)
+https://docs.launchdarkly.com/sdk/relay-proxy/sdk-config
+
 
 
 ### Go code

docker/Dockerfile

@@ -9,7 +9,7 @@ RUN apk add aws-cli jq --no-cache
 ENV FILE_DATA_SOURCE=/dorkly/flags.tar.gz
 
 # Sets ld-relay log level
-ENV LOG_LEVEL=debug
+ENV LOG_LEVEL=info
 
 COPY flags.tar.gz /dorkly/flags.tar.gz
 COPY pull.sh /pull.sh

internal/dorkly/integration_test.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/launchdarkly/go-sdk-common/v3/ldtime"
 	"github.com/launchdarkly/go-server-sdk-evaluation/v3/ldmodel"
-	"github.com/launchdarkly/go-server-sdk/v7/interfaces"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/testcontainers/testcontainers-go"
@@ -19,18 +17,24 @@ import (
 )
 
 // This test ensures the ld-relay can load a dorkly-generated archive and serve flags from it.
-func TestIntegration_LdRelayCanLoadArchive(t *testing.T) {
+// It serves as a partial integration test for the dorkly project.
+func TestDocker_LdRelayCanLoadArchive(t *testing.T) {
 	ctx := context.Background()
-	containerFlagsArchivePath := "/dorkly/flags.tar.gz"
 	containerReq := testcontainers.ContainerRequest{
-		Image:        "launchdarkly/ld-relay:8.4.2", // tag should be kept in sync with docker/Dockerfile
+		FromDockerfile: testcontainers.FromDockerfile{
+			Context:    "../../Docker/",
+			Dockerfile: "Dockerfile",
+			Repo:       "dorkly-testcontainers",
+			Tag:        "LdRelayCanLoadArchive",
+		},
 		ExposedPorts: []string{"8030/tcp"},
 		Env: map[string]string{
-			"FILE_DATA_SOURCE": containerFlagsArchivePath,
+			"LOG_LEVEL": "debug",
+			"S3_URL":    "required to be non-empty but ok to be this bogus value so we can ensure resiliency if S3 connection fails",
 		},
 		Files: []testcontainers.ContainerFile{{
 			HostFilePath:      "testdata/flags.tar.gz",
-			ContainerFilePath: containerFlagsArchivePath,
+			ContainerFilePath: "/dorkly/flags.tar.gz",
 			FileMode:          0755,
 		}},
 		WaitingFor: wait.ForLog("Starting server listening on port 8030"),
@@ -115,70 +119,10 @@ func testFlagsForEnv(t *testing.T, ctx context.Context, url, env string) {
 	})
 }
 
-// all structs below are copied from https://github.com/launchdarkly/ld-relay/blob/7d67cb6e8f4edd2e0ea5a20b7640a3d35a8e165d/internal/api/status_reps.go
-
 // StatusRep is the JSON representation returned by the status endpoint.
-//
-// This is exported for use in integration test code.
 type StatusRep struct {
-	Environments  map[string]EnvironmentStatusRep `json:"environments"`
-	Status        string                          `json:"status"`
-	Version       string                          `json:"version"`
-	ClientVersion string                          `json:"clientVersion"`
-}
-
-// EnvironmentStatusRep is the per-environment JSON representation returned by the status endpoint.
-//
-// This is exported for use in integration test code.
-type EnvironmentStatusRep struct {
-	SDKKey           string               `json:"sdkKey"`
-	EnvID            string               `json:"envId,omitempty"`
-	EnvKey           string               `json:"envKey,omitempty"`
-	EnvName          string               `json:"envName,omitempty"`
-	ProjKey          string               `json:"projKey,omitempty"`
-	ProjName         string               `json:"projName,omitempty"`
-	MobileKey        string               `json:"mobileKey,omitempty"`
-	ExpiringSDKKey   string               `json:"expiringSdkKey,omitempty"`
-	Status           string               `json:"status"`
-	ConnectionStatus ConnectionStatusRep  `json:"connectionStatus"`
-	DataStoreStatus  DataStoreStatusRep   `json:"dataStoreStatus"`
-	BigSegmentStatus *BigSegmentStatusRep `json:"bigSegmentStatus,omitempty"`
-}
-
-// BigSegmentStatusRep is the big segment status representation returned by the status endpoint.
-//
-// This is exported for use in integration test code.
-type BigSegmentStatusRep struct {
-	Available          bool                       `json:"available"`
-	PotentiallyStale   bool                       `json:"potentiallyStale"`
-	LastSynchronizedOn ldtime.UnixMillisecondTime `json:"lastSynchronizedOn"`
-}
-
-// ConnectionStatusRep is the data source status representation returned by the status endpoint.
-//
-// This is exported for use in integration test code.
-type ConnectionStatusRep struct {
-	State      interfaces.DataSourceState `json:"state"`
-	StateSince ldtime.UnixMillisecondTime `json:"stateSince"`
-	LastError  *ConnectionErrorRep        `json:"lastError,omitempty"`
-}
-
-// ConnectionErrorRep is the optional error information in ConnectionStatusRep.
-//
-// This is exported for use in integration test code.
-type ConnectionErrorRep struct {
-	Kind interfaces.DataSourceErrorKind `json:"kind"`
-	Time ldtime.UnixMillisecondTime     `json:"time"`
-}
-
-// DataStoreStatusRep is the data store status representation returned by the status endpoint.
-//
-// This is exported for use in integration test code.
-type DataStoreStatusRep struct {
-	State      string                     `json:"state"`
-	StateSince ldtime.UnixMillisecondTime `json:"stateSince"`
-	Database   string                     `json:"database,omitempty"`
-	DBServer   string                     `json:"dbServer,omitempty"`
-	DBPrefix   string                     `json:"dbPrefix,omitempty"`
-	DBTable    string                     `json:"dbTable,omitempty"`
+	Environments  map[string]interface{} `json:"environments"`
+	Status        string                 `json:"status"`
+	Version       string                 `json:"version"`
+	ClientVersion string                 `json:"clientVersion"`
 }

tasks.md

@@ -4,7 +4,7 @@
 - [x] Create terraform module in separate repo with example
 - [x] Do all S3 stuff in go code. (with option for local mode testing)
 - [x] Logging: switch to zap or similar
-- [ ] Logging: more descriptive log messages + clear indication of steps.
+- [x] Logging: more descriptive log messages + clear indication of steps.
 - [x] Secrets: terraform: generate and store in aws secrets manager
 - [x] Secrets: dorkly go code: read from aws secrets manager and inject into each environment's config
 - [x] Use localstack to test s3
@@ -18,7 +18,8 @@
 - [ ] yaml files validation: warn if flag is defined in the project but not an environment
 - [ ] yaml files validation: error if flag is defined in an environment but not in the project
 - [ ] yaml files validation: error if env flag type does not match project flag type (ie 'true' for a rollout flag)
-  MVP ==============================================
+  MVP ? ==============================================
+- [ ] Consider enabling configuring ld-relay client context (aka goals endpoint): https://github.com/launchdarkly/ld-relay/blob/1adf0dde5b11343d3bdf011c86e3f7116c4960fc/internal/relayenv/js_context.go#L7
 - [ ] Terraform: validate variables (see TODOs in https://github.com/dorklyorg/terraform-aws-dorkly-flags/blob/main/variables.tf)
 - [ ] Terraform: Consider a command line tool to quickly create new flags and maybe turn them off in all envs
 - [ ] Terraform: Use freeform workflow with human input to create flags