Skip to content

Commit

Permalink
UI Add Cluster Activity Page (#31123)
Browse files Browse the repository at this point in the history
* Dashboard init view

* Live metrics and first page layout

* Rebase and UI improvements

* Add DagRun to dashboard

* Update wording.

* Add filter bar

* Add historical data metrics

* Drop colors for now fix type

* Update following code review

* Use State colors

* Handle data fetch errors

* Add simple top component test

* Add cluster_activity tests

* Add labels to filter bar and period duration for visualization

* FilterBar style

* Sort legend in PieChart

* Add triggerer to health card

* Update UI and wording following Jed review
  • Loading branch information
pierrejeambrun committed Jun 1, 2023
1 parent 7db42fe commit d67b383
Show file tree
Hide file tree
Showing 34 changed files with 2,549 additions and 489 deletions.
20 changes: 10 additions & 10 deletions airflow/security/permissions.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,27 @@
RESOURCE_AIRFLOW = "Airflow"
RESOURCE_AUDIT_LOG = "Audit Logs"
RESOURCE_BROWSE_MENU = "Browse"
RESOURCE_DAG = "DAGs"
RESOURCE_DAG_PREFIX = "DAG:"
RESOURCE_LOGIN = "Logins"
RESOURCE_DOCS_MENU = "Docs"
RESOURCE_DOCS = "Documentation"
RESOURCE_CONFIG = "Configurations"
RESOURCE_CONNECTION = "Connections"
RESOURCE_DAG_DEPENDENCIES = "DAG Dependencies"
RESOURCE_DAG = "DAGs"
RESOURCE_DAG_CODE = "DAG Code"
RESOURCE_DAG_DEPENDENCIES = "DAG Dependencies"
RESOURCE_DAG_PREFIX = "DAG:"
RESOURCE_DAG_RUN = "DAG Runs"
RESOURCE_IMPORT_ERROR = "ImportError"
RESOURCE_DAG_WARNING = "DAG Warnings"
RESOURCE_CLUSTER_ACTIVITY = "Cluster Activity"
RESOURCE_DATASET = "Datasets"
RESOURCE_DOCS = "Documentation"
RESOURCE_DOCS_MENU = "Docs"
RESOURCE_IMPORT_ERROR = "ImportError"
RESOURCE_JOB = "Jobs"
RESOURCE_LOGIN = "Logins"
RESOURCE_MY_PASSWORD = "My Password"
RESOURCE_MY_PROFILE = "My Profile"
RESOURCE_PASSWORD = "Passwords"
RESOURCE_PERMISSION = "Permission Views" # Refers to a Perm <-> View mapping, not an MVC View.
RESOURCE_POOL = "Pools"
RESOURCE_PLUGIN = "Plugins"
RESOURCE_POOL = "Pools"
RESOURCE_PROVIDER = "Providers"
RESOURCE_RESOURCE = "View Menus"
RESOURCE_ROLE = "Roles"
Expand All @@ -54,8 +56,6 @@
RESOURCE_VARIABLE = "Variables"
RESOURCE_WEBSITE = "Website"
RESOURCE_XCOM = "XComs"
RESOURCE_DATASET = "Datasets"


# Action Constants
ACTION_CAN_CREATE = "can_create"
Expand Down
4 changes: 3 additions & 1 deletion airflow/www/extensions/init_appbuilder_links.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@ def init_appbuilder_links(app):

appbuilder.add_link(name="DAGs", href="Airflow.index")
appbuilder.menu.menu.insert(0, appbuilder.menu.menu.pop()) # Place in the first menu slot
appbuilder.add_link(name="Datasets", href="Airflow.datasets")
appbuilder.add_link(name="Cluster Activity", href="Airflow.cluster_activity")
appbuilder.menu.menu.insert(1, appbuilder.menu.menu.pop()) # Place in the second menu slot
appbuilder.add_link(name="Datasets", href="Airflow.datasets")
appbuilder.menu.menu.insert(2, appbuilder.menu.menu.pop()) # Place in the third menu slot

# Docs links
appbuilder.add_link(name="Documentation", label="Documentation", href=get_docs_url(), category="Docs")
Expand Down
1 change: 1 addition & 0 deletions airflow/www/jest-setup.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import "@testing-library/jest-dom";
import axios from "axios";
import { setLogger } from "react-query";
import "jest-canvas-mock";

// eslint-disable-next-line import/no-extraneous-dependencies
import moment from "moment-timezone";
Expand Down
4 changes: 3 additions & 1 deletion airflow/www/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
"file-loader": "^6.0.0",
"imports-loader": "^1.1.0",
"jest": "^27.3.1",
"jest-canvas-mock": "^2.5.1",
"mini-css-extract-plugin": "^1.6.2",
"moment": "^2.29.4",
"moment-locales-webpack-plugin": "^1.2.0",
Expand All @@ -86,7 +87,7 @@
"webpack-manifest-plugin": "^4.0.0"
},
"dependencies": {
"@chakra-ui/react": "^2.2.0",
"@chakra-ui/react": "2.4.2",
"@emotion/cache": "^11.9.3",
"@emotion/react": "^11.9.3",
"@emotion/styled": "^11",
Expand All @@ -107,6 +108,7 @@
"dagre-d3": "^0.6.4",
"datatables.net": "^1.11.4",
"datatables.net-bs": "^1.11.4",
"echarts": "^5.4.2",
"elkjs": "^0.7.1",
"eonasdan-bootstrap-datetimepicker": "^4.17.47",
"framer-motion": "^6.0.0",
Expand Down
2 changes: 2 additions & 0 deletions airflow/www/security.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin):
(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_CODE),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_RUN),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_DATASET),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_CLUSTER_ACTIVITY),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_IMPORT_ERROR),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_DAG_WARNING),
(permissions.ACTION_CAN_READ, permissions.RESOURCE_JOB),
Expand All @@ -90,6 +91,7 @@ class AirflowSecurityManager(SecurityManager, LoggingMixin):
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DAG_DEPENDENCIES),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DAG_RUN),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DATASET),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_CLUSTER_ACTIVITY),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DOCS),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_DOCS_MENU),
(permissions.ACTION_CAN_ACCESS_MENU, permissions.RESOURCE_JOB),
Expand Down
16 changes: 13 additions & 3 deletions airflow/www/static/js/api/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ import useUpstreamDatasetEvents from "./useUpstreamDatasetEvents";
import useTaskInstance from "./useTaskInstance";
import useDag from "./useDag";
import useDagCode from "./useDagCode";
import useHealth from "./useHealth";
import usePools from "./usePools";
import useDags from "./useDags";
import useDagRuns from "./useDagRuns";
import useHistoricalMetricsData from "./useHistoricalMetricsData";

axios.interceptors.response.use((res: AxiosResponse) =>
res.data ? camelcaseKeys(res.data, { deep: true }) : res
Expand All @@ -52,24 +57,29 @@ axios.defaults.headers.common.Accept = "application/json";
export {
useClearRun,
useClearTask,
useMarkTaskDryRun,
useDag,
useDagCode,
useDagRuns,
useDags,
useDataset,
useDatasetDependencies,
useDatasetEvents,
useDatasets,
useExtraLinks,
useGraphData,
useGridData,
useHealth,
useMappedInstances,
useMarkFailedRun,
useMarkFailedTask,
useMarkSuccessRun,
useMarkSuccessTask,
useMarkTaskDryRun,
usePools,
useQueueRun,
useSetDagRunNote,
useSetTaskInstanceNote,
useTaskInstance,
useUpstreamDatasetEvents,
useDag,
useDagCode,
useHistoricalMetricsData,
};
50 changes: 50 additions & 0 deletions airflow/www/static/js/api/useDagRuns.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";
import type { API } from "src/types";

import { getMetaValue } from "src/utils";

const useDagRuns = ({
dagId,
state,
limit,
orderBy,
}: API.GetDagRunsVariables) => {
const dagRunsUrl = getMetaValue("dag_runs_url").replace("__DAG_ID__", dagId);

return useQuery(
["dagRuns", state, dagId, limit],
async () =>
axios.get<AxiosResponse, API.DAGRunCollection>(dagRunsUrl, {
params: {
state: state ? state.join(",") : state,
limit,
order_by: orderBy,
},
}),
{
refetchInterval: (autoRefreshInterval || 1) * 1000,
}
);
};

export default useDagRuns;
40 changes: 40 additions & 0 deletions airflow/www/static/js/api/useDags.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";
import type { API } from "src/types";

import { getMetaValue } from "src/utils";

const dagsUrl = getMetaValue("dags_url");

const useDags = ({ paused }: API.GetDagsVariables) =>
useQuery(
["dags", paused],
async () =>
axios.get<AxiosResponse, API.DAGCollection>(dagsUrl, {
params: { paused },
}),
{
refetchInterval: (autoRefreshInterval || 1) * 1000,
}
);

export default useDags;
37 changes: 37 additions & 0 deletions airflow/www/static/js/api/useHealth.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";
import type { API } from "src/types";

import { getMetaValue } from "src/utils";

const healthUrl = getMetaValue("health_url");

const useHealth = () =>
useQuery(
["health"],
async () => axios.get<AxiosResponse, API.HealthInfo>(healthUrl),
{
refetchInterval: (autoRefreshInterval || 1) * 1000,
}
);

export default useHealth;
40 changes: 40 additions & 0 deletions airflow/www/static/js/api/useHistoricalMetricsData.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";
import type { HistoricalMetricsData } from "src/types";

import { getMetaValue } from "src/utils";

const url = getMetaValue("historical_metrics_data_url");

const useHistoricalMetricsData = (startDate: string, endDate: string) =>
useQuery(
["historical_metrics_data", startDate, endDate],
async () =>
axios.get<AxiosResponse, HistoricalMetricsData>(url, {
params: { start_date: startDate, end_date: endDate },
}),
{
refetchInterval: (autoRefreshInterval || 1) * 1000,
}
);

export default useHistoricalMetricsData;
41 changes: 41 additions & 0 deletions airflow/www/static/js/api/usePools.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import axios, { AxiosResponse } from "axios";
import { useQuery } from "react-query";
import { useAutoRefresh } from "src/context/autorefresh";
import type { API } from "src/types";

import { getMetaValue } from "src/utils";

const poolsUrl = getMetaValue("pools_url");

const usePools = () => {
const { isRefreshOn } = useAutoRefresh();

return useQuery(
["pools"],
async () => axios.get<AxiosResponse, API.PoolCollection>(poolsUrl),
{
refetchInterval: isRefreshOn && (autoRefreshInterval || 1) * 1000,
}
);
};

export default usePools;
Loading

0 comments on commit d67b383

Please sign in to comment.