diff --git a/.env b/.env index ba540bac..95e9739b 100644 --- a/.env +++ b/.env @@ -192,12 +192,3 @@ MINIO_EXTERNAL_PORT=19000 MINIO_CONSOLE_PORT=9001 MINIO_CONSOLE_EXTERNAL_PORT=19001 MINIO_DATA_DIR=~/.local/instill/minio/data - -# Component global secrets -# Provide your API key for the AI vendors so that you can use -# ${secret.INSTILL_SECRET} in the pipeline to access these shared keys. -OPENAI_SECRET_KEY= -STABILITYAI_SECRET_KEY= -ANTHROPIC_SECRET_KEY= -COHERE_SECRET_KEY= -MISTRALAI_SECRET_KEY= diff --git a/.env.component b/.env.component new file mode 100644 index 00000000..c7931ed4 --- /dev/null +++ b/.env.component @@ -0,0 +1,17 @@ +# Provide your API key for the AI vendors so that you can set the components up +# with default credentials. +CFG_CONNECTOR_SECRETS_OPENAI_APIKEY= +CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY= +CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY= +CFG_CONNECTOR_SECRETS_COHERE_APIKEY= +CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY= +CFG_CONNECTOR_SECRETS_GROQ_APIKEY= +CFG_CONNECTOR_SECRETS_FIREWORKSAI_APIKEY= + +# Numbers Protocol API key. +CFG_CONNECTOR_SECRETS_NUMBERS_XAPIKEY= + +# OAuth secrets. When these are filled, the specified component will support +# OAuth integrations. +CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTID= +CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTSECRET=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 diff --git a/.env.console b/.env.console new file mode 100644 index 00000000..8a2a84cf --- /dev/null +++ b/.env.console @@ -0,0 +1,4 @@ +# Client variables for OAuth integrations. +# These values depend on .env.component. +INTEGRATION_GOOGLE_DRIVE_CLIENT_ID=${CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTID} +INTEGRATION_GOOGLE_DRIVE_CLIENT_SECRET=${CFG_CONNECTOR_SECRETS_GOOGLEDRIVE_CLIENTSECRET} diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1c64d8ef..24ed22bb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -35,6 +35,22 @@ The env variable `PROFILE` is intended to specify which service component you wa When you set `PROFILE=exclude-{service}`, in which `{service}` can be `pipeline`, it means you want to develop on that particular service. The `make` command will launch the corresponding stack **WITHOUT** that service component and **WITH** all other services. Given that, you can later on spin up and down the `{service}` in your dev container. Please take the [pipeline-backend](https://github.com/instill-ai/pipeline-backend#local-dev) as an example. +#### Component environment variables + +Some components can be configured with global secrets. This has several +applications: + +- By accepting a global API key, some components have a default setup. When + the `setup` block is omitted in the recipe, this API key will be used. +- In order to connect to 3rd party vendors via OAuth, the application + client ID and secret must be injected. + +You can set the values of these global secrets in +[`.env.component`](./.env.component) before running the Docker container in +order to add a global configuration to your components. These values will +be injected into `pipeline-backend`. Additionally, `console` will also +receive the OAuth configuration values. + ### Tear down the local dev system Simply run: diff --git a/docker-compose.yml b/docker-compose.yml index 855b4a34..1aab5843 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,6 +137,7 @@ services: container_name: ${PIPELINE_BACKEND_HOST} image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION} restart: unless-stopped + env_file: .env.component environment: CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT} CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT} @@ -157,11 +158,6 @@ services: CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT} CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED} CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT} - CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY} - CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY} - CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY} command: - /bin/sh - -c @@ -193,6 +189,7 @@ services: container_name: ${PIPELINE_BACKEND_HOST}-worker image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION} restart: unless-stopped + env_file: .env.component environment: CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT} CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT} @@ -212,11 +209,6 @@ services: CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT} CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED} CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT} - CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY} - CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY} - CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY} entrypoint: ./pipeline-backend-worker depends_on: pipeline_backend: @@ -364,6 +356,9 @@ services: container_name: ${CONSOLE_HOST} image: ${CONSOLE_IMAGE}:${CONSOLE_VERSION} restart: unless-stopped + env_file: + - path: .env.component + - path: .env.console environment: NEXT_PUBLIC_GENERAL_API_VERSION: v1beta NEXT_PUBLIC_MODEL_API_VERSION: v1alpha