Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Splitting syspath preparation into stages #41672

Merged
merged 4 commits into from
Aug 23, 2024
Merged

Splitting syspath preparation into stages #41672

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e37d8e9
Splitting syspath preparation into stages
amoghrajesh Aug 22, 2024
9515bfe
fixing tests
amoghrajesh Aug 22, 2024
f8fb71e
fixing tests
amoghrajesh Aug 23, 2024
3c4fd3b
fixing tests
amoghrajesh Aug 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions airflow/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -653,11 +653,8 @@ def configure_action_logging() -> None:
"""Any additional configuration (register callback) for airflow.utils.action_loggers module."""


def prepare_syspath():
"""Ensure certain subfolders of AIRFLOW_HOME are on the classpath."""
if DAGS_FOLDER not in sys.path:
sys.path.append(DAGS_FOLDER)

def prepare_syspath_for_config_and_plugins():
"""Update sys.path for the config and plugins directories."""
# Add ./config/ for loading custom log parsers etc, or
# airflow_local_settings etc.
config_path = os.path.join(AIRFLOW_HOME, "config")
Expand All @@ -668,6 +665,12 @@ def prepare_syspath():
sys.path.append(PLUGINS_FOLDER)


def prepare_syspath_for_dags_folder():
"""Update sys.path to include the DAGs folder."""
if DAGS_FOLDER not in sys.path:
sys.path.append(DAGS_FOLDER)


def get_session_lifetime_config():
"""Get session timeout configs and handle outdated configs gracefully."""
session_lifetime_minutes = conf.get("webserver", "session_lifetime_minutes", fallback=None)
Expand Down Expand Up @@ -721,12 +724,13 @@ def import_local_settings():
def initialize():
"""Initialize Airflow with all the settings from this file."""
configure_vars()
prepare_syspath()
prepare_syspath_for_config_and_plugins()
configure_policy_plugin_manager()
# Load policy plugins _before_ importing airflow_local_settings, as Pluggy uses LIFO and we want anything
# in airflow_local_settings to take precendec
load_policy_plugins(POLICY_PLUGIN_MANAGER)
import_local_settings()
prepare_syspath_for_dags_folder()
global LOGGING_CLASS_PATH
LOGGING_CLASS_PATH = configure_logging()
State.state_color.update(STATE_COLORS)
Expand Down Expand Up @@ -756,7 +760,6 @@ def is_usage_data_collection_enabled() -> bool:
MEGABYTE = KILOBYTE * KILOBYTE
WEB_COLORS = {"LIGHTBLUE": "#4d9de0", "LIGHTORANGE": "#FF9933"}


# Updating serialized DAG can not be faster than a minimum interval to reduce database
# write rate.
MIN_SERIALIZED_DAG_UPDATE_INTERVAL = conf.getint("core", "min_serialized_dag_update_interval", fallback=30)
Expand Down
Loading