Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[COST-5059] - Change source status beat cadence #5135

Merged
merged 4 commits into from
May 29, 2024
Merged

[COST-5059] - Change source status beat cadence #5135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5d4b4b8
[COST-5059] - Change source status beat cadence
lcouzens May 28, 2024
a59aeb9
fix all my mistakes
lcouzens May 28, 2024
1497e3c
requested changes
lcouzens May 29, 2024
a83ff63
Merge branch 'main' into COST-5059
lcouzens May 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions deploy/clowdapp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,8 +649,8 @@ objects:
value: ${REPORT_DOWNLOAD_SCHEDULE}
- name: SCHEDULE_REPORT_CHECKS
value: ${SCHEDULE_REPORT_CHECKS}
- name: SOURCE_STATUS_FREQUENCY_MINUTES
value: ${SOURCE_STATUS_FREQUENCY_MINUTES}
- name: SOURCE_STATUS_SCHEDULE
value: ${SOURCE_STATUS_SCHEDULE}
- name: REMOVE_EXPIRED_REPORT_DATA_ON_DAY
value: ${REMOVE_EXPIRED_REPORT_DATA_ON_DAY}
- name: REMOVE_EXPIRED_REPORT_UTC_TIME
Expand Down Expand Up @@ -4659,8 +4659,8 @@ parameters:
name: SCHEDULE_REPORT_CHECKS
value: "True"
- displayName: Frequency to check sources availability (minutes)
name: SOURCE_STATUS_FREQUENCY_MINUTES
value: "60"
name: SOURCE_STATUS_SCHEDULE
value: 0 3 * * *
- displayName: Minimum replicas
name: SOURCES_CLIENT_REPLICAS
required: true
Expand Down
8 changes: 4 additions & 4 deletions deploy/kustomize/patches/scheduler.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@
value: ${REPORT_DOWNLOAD_SCHEDULE}
- name: SCHEDULE_REPORT_CHECKS
value: ${SCHEDULE_REPORT_CHECKS}
- name: SOURCE_STATUS_FREQUENCY_MINUTES
value: ${SOURCE_STATUS_FREQUENCY_MINUTES}
- name: SOURCE_STATUS_SCHEDULE
value: ${SOURCE_STATUS_SCHEDULE}
- name: REMOVE_EXPIRED_REPORT_DATA_ON_DAY
value: ${REMOVE_EXPIRED_REPORT_DATA_ON_DAY}
- name: REMOVE_EXPIRED_REPORT_UTC_TIME
Expand Down Expand Up @@ -212,5 +212,5 @@
path: /parameters/-
value:
displayName: Frequency to check sources availability (minutes)
name: SOURCE_STATUS_FREQUENCY_MINUTES
value: '60'
name: SOURCE_STATUS_SCHEDULE
value: "0 3 * * *"
2 changes: 1 addition & 1 deletion docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ services:
- DJANGO_SETTINGS_MODULE=koku.settings
- PROMETHEUS_MULTIPROC_DIR=/tmp
- SCHEDULE_REPORT_CHECKS=True
- SOURCE_STATUS_FREQUENCY_MINUTES
- SOURCE_STATUS_SCHEDULE
- UPLOAD_NORMALIZED_DATA_INTERVAL=10000000
- UNLEASH_HOST=${UNLEASH_HOST-unleash}
- UNLEASH_PREFIX=${UNLEASH_PREFIX-http}
Expand Down
18 changes: 10 additions & 8 deletions koku/koku/celery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ def readiness_check(self):
self._write_response(ProbeResponse(status, msg))


def validate_cron_expression(expresssion):
def validate_cron_expression(expresssion, default="0 * * * *"):
if not croniter.is_valid(expresssion):
print(f"Invalid report-download-schedule {expresssion}. Falling back to default `0 4,16 * * *`")
expresssion = "0 4,16 * * *"
print(f"Invalid report-download-schedule {expresssion}. Falling back to default {default}")
expresssion = default
return expresssion


Expand All @@ -114,11 +114,11 @@ def validate_cron_expression(expresssion):

# Toggle to enable/disable scheduled checks for new reports.
if ENVIRONMENT.bool("SCHEDULE_REPORT_CHECKS", default=False):
download_expression = "0 * * * *"
download_fallback = validate_cron_expression("0 * * * *")
download_task = "masu.celery.tasks.check_report_updates"
# The schedule to scan for new reports.
REPORT_DOWNLOAD_SCHEDULE = ENVIRONMENT.get_value("REPORT_DOWNLOAD_SCHEDULE", default=download_expression)
REPORT_DOWNLOAD_SCHEDULE = validate_cron_expression(REPORT_DOWNLOAD_SCHEDULE)
download_expression = ENVIRONMENT.get_value("REPORT_DOWNLOAD_SCHEDULE", default=download_fallback)
REPORT_DOWNLOAD_SCHEDULE = validate_cron_expression(download_expression, download_fallback)
report_schedule = crontab(*REPORT_DOWNLOAD_SCHEDULE.split(" ", 5))
CHECK_REPORT_UPDATES_DEF = {
"task": download_task,
Expand Down Expand Up @@ -175,8 +175,10 @@ def validate_cron_expression(expresssion):
}

# Specify the frequency for pushing source status.
SOURCE_STATUS_FREQUENCY_MINUTES = ENVIRONMENT.get_value("SOURCE_STATUS_FREQUENCY_MINUTES", default="30")
source_status_schedule = crontab(minute=f"*/{SOURCE_STATUS_FREQUENCY_MINUTES}")
status_fallback = validate_cron_expression("0 3 * * *")
status_expression = ENVIRONMENT.get_value("SOURCE_STATUS_SCHEDULE", default=status_fallback)
SOURCE_STATUS_SCHEDULE = validate_cron_expression(status_expression, status_fallback)
source_status_schedule = crontab(*SOURCE_STATUS_SCHEDULE.split(" ", 5))

# task to push source status`
app.conf.beat_schedule["source_status_beat"] = {
Expand Down
Loading