diff --git a/.github/workflows/update-stage.yml b/.github/workflows/update-stage.yml index 839b524..be2c8c3 100644 --- a/.github/workflows/update-stage.yml +++ b/.github/workflows/update-stage.yml @@ -8,46 +8,55 @@ jobs: deploy: runs-on: self-hosted steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Load dotenv from secret for Grafana Configuration - run: | - cat < .env - # Sensitive configurations from GitHub Secrets - GF_SMTP_HOST = ${{ secrets.SMTP_HOST }} - GF_SMTP_USER = ${{ secrets.SMTP_USER }} - GF_SMTP_PASSWORD = ${{ secrets.SMTP_PASSWORD }} - GF_SMTP_FROM_ADDRESS = ${{ secrets.FROM_ADDRESS }} - EOF - - - name: Check if Caddy containers are running - id: check-caddy-container - run: | - { - echo 'stdout<> "$GITHUB_OUTPUT" - - - name: when caddy container down, caddy up - if: steps.check-caddy-container.outputs.stdout == '' - run: | - docker compose --profile caddy up -d --no-recreate - - - name: Set Caddyfile Environment Variables - run: | - sed -i 's/{$AWS_REQ_IP}/${{secrets.AWS_REQ_IP}}/' ./Caddyfile - - - name: Copy Caddyfile into Caddy Container - run: | - docker cp ./Caddyfile caddy:/etc/caddy/Caddyfile - - - name: Gracefully reload Caddy - run: | - docker exec -w /etc/caddy caddy caddy reload - - - name: Run Docker Compose Of Log - run: | - docker compose --profile log up -d - \ No newline at end of file + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Load dotenv from secret for Grafana Configuration + run: | + cat < .env + # Sensitive configurations from GitHub Secrets + GF_SMTP_HOST = ${{ secrets.SMTP_HOST }} + GF_SMTP_USER = ${{ secrets.SMTP_USER }} + GF_SMTP_PASSWORD = ${{ secrets.SMTP_PASSWORD }} + GF_SMTP_FROM_ADDRESS = ${{ secrets.FROM_ADDRESS }} + MINIO_ROOT_USER = ${{ secrets.MINIO_ROOT_USER }} + MINIO_ROOT_PASSWORD = ${{ secrets.MINIO_ROOT_PASSWORD }} + EOF + + - name: Check if Caddy containers are running + id: check-caddy-container + run: | + { + echo 'stdout<> "$GITHUB_OUTPUT" + + - name: Run Docker Compose Of Loki + run: | + docker compose --profile loki up -d --no-recreate + + - name: Run Docker Compose Of Minio + run: | + docker compose --profile minio up -d --no-recreate + + - name: Initialize grafana container + run: | + docker compose --profile grafana-console up -d --no-recreate + + - name: when caddy container down, caddy up + if: steps.check-caddy-container.outputs.stdout == '' + run: | + docker compose --profile caddy up -d --no-recreate + + - name: Set Caddyfile Environment Variables + run: | + sed -i 's/{$AWS_REQ_IP}/${{secrets.AWS_REQ_IP}}/' ./Caddyfile + + - name: Copy Caddyfile into Caddy Container + run: | + docker cp ./Caddyfile caddy:/etc/caddy/Caddyfile + + - name: Gracefully reload Caddy + run: | + docker exec -w /etc/caddy caddy caddy reload diff --git a/docker-compose.yml b/docker-compose.yml index b965f0d..ffad930 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ networks: services: caddy: - profiles: ['caddy'] + profiles: ["caddy"] image: caddy:2.7.6-alpine container_name: caddy restart: always @@ -13,62 +13,107 @@ services: - "80:80" - "443:443" network_mode: host - + + minio: + profiles: ["minio"] + container_name: minio + image: minio/minio:latest + volumes: + - minio_data_volume:/data + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + networks: + - loki + ports: + - "9000:9000" + - "9001:9001" + command: server data --address ':9000' --console-address ':9001' + shm_size: "1gb" + + createbuckets: + profiles: ["minio"] + image: minio/mc + volumes: + - minio_data_volume:/data + networks: + - loki + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add minio http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}; + /usr/bin/mc mb data/loki-aws; + /usr/bin/mc mb data/loki-stage; + /usr/bin/mc mb data/loki-test; + /usr/bin/mc anonymous set public data/loki-aws; + /usr/bin/mc anonymous set public data/loki-stage; + /usr/bin/mc anonymous set public data/loki-test; + exit 0; + " + loki-test: - profiles: ['log'] + profiles: ["loki"] image: grafana/loki:2.9.0 container_name: loki-test + environment: + - MINIO_ROOT_PASSWORD + - MINIO_ROOT_USER ports: - "3199:3100" volumes: - - '$PWD/grafana/loki-config-test.yml:/etc/loki/loki-config.yml' - command: -config.file=/etc/loki/loki-config.yml + - "$PWD/grafana/loki-config-test.yml:/etc/loki/loki-config.yml" + command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki loki-stage: - profiles: ['log'] + profiles: ["loki"] image: grafana/loki:2.9.0 container_name: loki-stage + environment: + - MINIO_ROOT_PASSWORD + - MINIO_ROOT_USER ports: - "3101:3100" volumes: - - '$PWD/grafana/loki-config-stage.yml:/etc/loki/loki-config.yml' - command: -config.file=/etc/loki/loki-config.yml + - "$PWD/grafana/loki-config-stage.yml:/etc/loki/loki-config.yml" + command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki loki-aws: - profiles: ['log'] + profiles: ["loki"] image: grafana/loki:2.9.0 container_name: loki-aws + ports: - "3100:3100" volumes: - - '$PWD/grafana/loki-config-aws.yml:/etc/loki/loki-config.yml' - command: -config.file=/etc/loki/loki-config.yml + - "$PWD/grafana/loki-config-aws.yml:/etc/loki/loki-config.yml" + command: -config.file=/etc/loki/loki-config.yml -config.expand-env=true networks: - loki - grafana: - profiles: ['log'] + profiles: ["grafana-console"] image: grafana/grafana:latest - env_file: + env_file: - .env container_name: grafana - restart: unless-stopped + restart: + unless-stopped # environment: # - TERM=linux # - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-polystat-panel ports: - - '3000:3000' + - "3000:3000" volumes: - - 'grafana_storage:/var/lib/grafana' - - '$PWD/grafana/grafana.ini:/etc/grafana/grafana.ini' + - "grafana_storage:/var/lib/grafana" + - "$PWD/grafana/grafana.ini:/etc/grafana/grafana.ini" networks: - loki - volumes: grafana_storage: {} + minio_data_volume: diff --git a/grafana/loki-config-aws.yml b/grafana/loki-config-aws.yml index 68c3c9f..d79e948 100644 --- a/grafana/loki-config-aws.yml +++ b/grafana/loki-config-aws.yml @@ -23,11 +23,28 @@ query_range: enabled: true max_size_mb: 100 +ingester: + chunk_idle_period: 1m # Flush chunks that are idle for 1 minute. + max_chunk_age: 1h # Flush chunks older than 1 hour. + chunk_target_size: 1048576 # Target size of 1MB before flushing. + chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing. + max_transfer_retries: 0 + +storage_config: + aws: + s3: http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio.:9000/loki-aws + s3forcepathstyle: true + boltdb_shipper: + active_index_directory: /loki/boltdb-shipper-active + cache_location: /loki/boltdb-shipper-cache + resync_interval: 5s + shared_store: s3 + schema_config: configs: - from: 2020-10-24 store: boltdb-shipper - object_store: filesystem + object_store: aws schema: v11 index: prefix: index_ @@ -35,7 +52,6 @@ schema_config: ruler: alertmanager_url: http://localhost:9093 - # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ # diff --git a/grafana/loki-config-stage.yml b/grafana/loki-config-stage.yml index 68c3c9f..bdd27fb 100644 --- a/grafana/loki-config-stage.yml +++ b/grafana/loki-config-stage.yml @@ -27,7 +27,34 @@ schema_config: configs: - from: 2020-10-24 store: boltdb-shipper - object_store: filesystem + object_store: s3 # MinIO로 로그 데이터 저장 + schema: v11 + index: + prefix: index_ + period: 24h + +ingester: + chunk_idle_period: 1m # Flush chunks that are idle for 1 minute. + max_chunk_age: 1h # Flush chunks older than 1 hour. + chunk_target_size: 1048576 # Target size of 1MB before flushing. + chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing. + max_transfer_retries: 0 + +storage_config: + aws: + s3: http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio.:9000/loki-stage + s3forcepathstyle: true + boltdb_shipper: + active_index_directory: /loki/boltdb-shipper-active + cache_location: /loki/boltdb-shipper-cache + resync_interval: 5s + shared_store: s3 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: aws schema: v11 index: prefix: index_ @@ -35,7 +62,6 @@ schema_config: ruler: alertmanager_url: http://localhost:9093 - # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ # diff --git a/grafana/loki-config-test.yml b/grafana/loki-config-test.yml index 68c3c9f..cef7973 100644 --- a/grafana/loki-config-test.yml +++ b/grafana/loki-config-test.yml @@ -23,11 +23,28 @@ query_range: enabled: true max_size_mb: 100 +ingester: + chunk_idle_period: 1m # Flush chunks that are idle for 1 minute. + max_chunk_age: 1h # Flush chunks older than 1 hour. + chunk_target_size: 1048576 # Target size of 1MB before flushing. + chunk_retain_period: 30s # Retain chunks for 30 seconds in memory after flushing. + max_transfer_retries: 0 + +storage_config: + aws: + s3: http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio.:9000/loki-test + s3forcepathstyle: true + boltdb_shipper: + active_index_directory: /loki/boltdb-shipper-active + cache_location: /loki/boltdb-shipper-cache + resync_interval: 5s + shared_store: s3 + schema_config: configs: - from: 2020-10-24 store: boltdb-shipper - object_store: filesystem + object_store: aws schema: v11 index: prefix: index_ @@ -35,7 +52,6 @@ schema_config: ruler: alertmanager_url: http://localhost:9093 - # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ #