Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add read-only Config endpoint #9497

Merged
merged 14 commits into from
Jun 26, 2020
63 changes: 59 additions & 4 deletions airflow/api_connexion/endpoints/config_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,67 @@
# specific language governing permissions and limitations
# under the License.

# TODO(mik-laj): We have to implement it.
# Do you want to help? Please look at: https://github.com/apache/airflow/issues/8136
from flask import Response, request

from airflow.api_connexion.schemas.config_schema import Config, ConfigOption, ConfigSection, config_schema
from airflow.configuration import conf
from airflow.settings import json

def get_config():
LINE_SEP = '\n' # `\n` cannot appear in f-strings


def _conf_dict_to_config(conf_dict: dict) -> Config:
"""Convert config dict to a Config object"""
config = Config(
sections=[
ConfigSection(
name=section,
options=[
ConfigOption(key=key, value=value)
for key, value in options.items()
]
)
for section, options in conf_dict.items()
]
)
return config


def _option_to_text(config_option: ConfigOption) -> str:
"""Convert a single config option to text"""
return f'{config_option.key} = {config_option.value}'


def _section_to_text(config_section: ConfigSection) -> str:
"""Convert a single config section to text"""
return (f'[{config_section.name}]{LINE_SEP}'
f'{LINE_SEP.join(_option_to_text(option) for option in config_section.options)}{LINE_SEP}')


def _config_to_text(config: Config) -> str:
"""Convert the entire config to text"""
return LINE_SEP.join(_section_to_text(s) for s in config.sections)


def _config_to_json(config: Config) -> str:
"""Convert a Config object to a JSON formatted string"""
return json.dumps(config_schema.dump(config).data, indent=4)


def get_config() -> Response:
"""
Get current configuration.
"""
raise NotImplementedError("Not implemented yet.")
serializer = {
'text/plain': _config_to_text,
'application/json': _config_to_json,
}
response_types = serializer.keys()
return_type = request.accept_mimetypes.best_match(response_types)
conf_dict = conf.as_dict(display_source=False, display_sensitive=True)
config = _conf_dict_to_config(conf_dict)
if return_type not in serializer:
return Response(status=406)
else:
config_text = serializer[return_type](config)
return Response(config_text, headers={'Content-Type': return_type})
3 changes: 0 additions & 3 deletions airflow/api_connexion/openapi/v1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1133,9 +1133,6 @@ paths:
x-openapi-router-controller: airflow.api_connexion.endpoints.config_endpoint
operationId: get_config
tags: [Config]
parameters:
- $ref: '#/components/parameters/PageLimit'
- $ref: '#/components/parameters/PageOffset'
responses:
'200':
description: Return current configuration.
Expand Down
57 changes: 57 additions & 0 deletions airflow/api_connexion/schemas/config_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from typing import List, NamedTuple

from marshmallow import Schema, fields


class ConfigOptionSchema(Schema):
""" Config Option Schema """
key = fields.String(required=True)
value = fields.String(required=True)


class ConfigOption(NamedTuple):
""" Config option """
key: str
value: str


class ConfigSectionSchema(Schema):
""" Config Section Schema"""
name = fields.String(required=True)
options = fields.List(fields.Nested(ConfigOptionSchema))


class ConfigSection(NamedTuple):
""" List of config options within a section """
name: str
options: List[ConfigOption]


class ConfigSchema(Schema):
""" Config Schema"""
sections = fields.List(fields.Nested(ConfigSectionSchema))


class Config(NamedTuple):
""" List of config sections with their options """
sections: List[ConfigSection]


config_schema = ConfigSchema(strict=True)
67 changes: 58 additions & 9 deletions tests/api_connexion/endpoints/test_config_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,72 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import unittest

import pytest
import textwrap

from mock import patch

from airflow.www import app

MOCK_CONF = {
'core': {
'parallelism': '1024',
},
'smtp': {
'smtp_host': 'localhost',
'smtp_mail_from': 'airflow@example.com',
},
}


class TestGetConfig(unittest.TestCase):
@patch(
"airflow.api_connexion.endpoints.config_endpoint.conf.as_dict",
return_value=MOCK_CONF
)
class TestGetConfig:
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
def setup_class(cls) -> None:
cls.app = app.create_app(testing=True) # type:ignore
cls.client = None

def setUp(self) -> None:
def setup_method(self) -> None:
self.client = self.app.test_client() # type:ignore

@pytest.mark.skip(reason="Not implemented yet")
def test_should_response_200(self):
response = self.client.get("/api/v1/config")
def test_should_response_200_text_plain(self, mock_as_dict):
response = self.client.get("/api/v1/config", headers={'Accept': 'text/plain'})
assert response.status_code == 200
expected = textwrap.dedent("""\
[core]
parallelism = 1024

[smtp]
smtp_host = localhost
smtp_mail_from = airflow@example.com
""")
assert expected == response.data.decode()

def test_should_response_200_application_json(self, mock_as_dict):
response = self.client.get("/api/v1/config", headers={'Accept': 'application/json'})
assert response.status_code == 200
expected = {
'sections': [
{
'name': 'core',
'options': [
{'key': 'parallelism', 'value': '1024'},
]
},
{
'name': 'smtp',
'options': [
{'key': 'smtp_host', 'value': 'localhost'},
{'key': 'smtp_mail_from', 'value': 'airflow@example.com'},
]
},
]
}
assert expected == response.json

def test_should_response_406(self, mock_as_dict):
response = self.client.get("/api/v1/config", headers={'Accept': 'application/octet-stream'})
assert response.status_code == 406
58 changes: 58 additions & 0 deletions tests/api_connexion/schemas/test_config_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

from airflow.api_connexion.schemas.config_schema import Config, ConfigOption, ConfigSection, config_schema


class TestConfigSchema:
def test_serialize(self):
config = Config(
sections=[
ConfigSection(
name='sec1',
options=[
ConfigOption(key='apache', value='airflow'),
ConfigOption(key='hello', value='world'),
]
),
ConfigSection(
name='sec2',
options=[
ConfigOption(key='foo', value='bar'),
]
),
]
)
result = config_schema.dump(config)
expected = {
'sections': [
{
'name': 'sec1',
'options': [
{'key': 'apache', 'value': 'airflow'},
{'key': 'hello', 'value': 'world'},
]
},
{
'name': 'sec2',
'options': [
{'key': 'foo', 'value': 'bar'},
]
},
]
}
assert result.data == expected