-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Upgrade ClickHouse server image to v23.4
- Upgrade ClickHouse server to v23.4 - Upgrade ClickHouse operator and metrics exporter to v0.21.0 Notable changes: - The inner table name of Materialized View has been changed to `.inner_id.<uuid>`. We are no longer able to find the table by self-defined Materialized View name. Thus, in `create_table.sh`, we save the data of Materialized Views to pre-defined underlying tables. - ClickHouse upgrade test used to always apply the latest clickhouse-operator-install-bundle.yaml. In this upgrade, latest operator yaml does not work with version N-1 ClickHouse data schema. Therefore, we change the test to use the correct version of operator yaml corresponding to the tested ClickHouse data schema. - We removed the theia- prefix in clickhouse operator images in - We fix several e2e test flakiness brought by ClickHouse upgrade: increase theia-manager ping timeout when connecting to ClickHouse Service; changing from pinging http port to pinging tcp port when checking for ClickHouse Service readiness. Signed-off-by: heanlan <hanlan@vmware.com>
- Loading branch information
Showing
18 changed files
with
1,738 additions
and
985 deletions.
There are no files selected for viewing
1,677 changes: 796 additions & 881 deletions
1,677
build/charts/theia/crds/clickhouse-operator-install-bundle.yaml
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
192 changes: 192 additions & 0 deletions
192
build/charts/theia/provisioning/datasources/migrators/000005_0-6-0.down.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,192 @@ | ||
| DROP VIEW flows_pod_view_local; | ||
| DROP VIEW flows_node_view_local; | ||
| DROP VIEW flows_policy_view_local; | ||
|
|
||
| --Create a Materialized View to aggregate data for pods | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS flows_pod_view_local | ||
| ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') | ||
| ORDER BY ( | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourcePodName, | ||
| destinationPodName, | ||
| destinationIP, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| flowType, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| sourceTransportPort, | ||
| destinationTransportPort, | ||
| clusterUUID) | ||
| POPULATE | ||
| AS SELECT | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourcePodName, | ||
| destinationPodName, | ||
| destinationIP, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| flowType, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| sourceTransportPort, | ||
| destinationTransportPort, | ||
| sum(octetDeltaCount) AS octetDeltaCount, | ||
| sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, | ||
| sum(throughput) AS throughput, | ||
| sum(reverseThroughput) AS reverseThroughput, | ||
| sum(throughputFromSourceNode) AS throughputFromSourceNode, | ||
| sum(throughputFromDestinationNode) AS throughputFromDestinationNode, | ||
| clusterUUID | ||
| FROM flows_local | ||
| GROUP BY | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourcePodName, | ||
| destinationPodName, | ||
| destinationIP, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| flowType, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| sourceTransportPort, | ||
| destinationTransportPort, | ||
| clusterUUID; | ||
|
|
||
| --Create a Materialized View to aggregate data for nodes | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS flows_node_view_local | ||
| ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') | ||
| ORDER BY ( | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourceNodeName, | ||
| destinationNodeName, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| clusterUUID) | ||
| POPULATE | ||
| AS SELECT | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourceNodeName, | ||
| destinationNodeName, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| sum(octetDeltaCount) AS octetDeltaCount, | ||
| sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, | ||
| sum(throughput) AS throughput, | ||
| sum(reverseThroughput) AS reverseThroughput, | ||
| sum(throughputFromSourceNode) AS throughputFromSourceNode, | ||
| sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, | ||
| sum(throughputFromDestinationNode) AS throughputFromDestinationNode, | ||
| sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, | ||
| clusterUUID | ||
| FROM flows_local | ||
| GROUP BY | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| sourceNodeName, | ||
| destinationNodeName, | ||
| sourcePodNamespace, | ||
| destinationPodNamespace, | ||
| clusterUUID; | ||
|
|
||
| --Create a Materialized View to aggregate data for network policies | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS flows_policy_view_local | ||
| ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') | ||
| ORDER BY ( | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| egressNetworkPolicyName, | ||
| egressNetworkPolicyNamespace, | ||
| egressNetworkPolicyRuleAction, | ||
| ingressNetworkPolicyName, | ||
| ingressNetworkPolicyNamespace, | ||
| ingressNetworkPolicyRuleAction, | ||
| sourcePodName, | ||
| sourceTransportPort, | ||
| sourcePodNamespace, | ||
| destinationPodName, | ||
| destinationTransportPort, | ||
| destinationPodNamespace, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| destinationIP, | ||
| clusterUUID) | ||
| POPULATE | ||
| AS SELECT | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| egressNetworkPolicyName, | ||
| egressNetworkPolicyNamespace, | ||
| egressNetworkPolicyRuleAction, | ||
| ingressNetworkPolicyName, | ||
| ingressNetworkPolicyNamespace, | ||
| ingressNetworkPolicyRuleAction, | ||
| sourcePodName, | ||
| sourceTransportPort, | ||
| sourcePodNamespace, | ||
| destinationPodName, | ||
| destinationTransportPort, | ||
| destinationPodNamespace, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| destinationIP, | ||
| sum(octetDeltaCount) AS octetDeltaCount, | ||
| sum(reverseOctetDeltaCount) AS reverseOctetDeltaCount, | ||
| sum(throughput) AS throughput, | ||
| sum(reverseThroughput) AS reverseThroughput, | ||
| sum(throughputFromSourceNode) AS throughputFromSourceNode, | ||
| sum(reverseThroughputFromSourceNode) AS reverseThroughputFromSourceNode, | ||
| sum(throughputFromDestinationNode) AS throughputFromDestinationNode, | ||
| sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode, | ||
| clusterUUID | ||
| FROM flows_local | ||
| GROUP BY | ||
| timeInserted, | ||
| flowEndSeconds, | ||
| flowEndSecondsFromSourceNode, | ||
| flowEndSecondsFromDestinationNode, | ||
| egressNetworkPolicyName, | ||
| egressNetworkPolicyNamespace, | ||
| egressNetworkPolicyRuleAction, | ||
| ingressNetworkPolicyName, | ||
| ingressNetworkPolicyNamespace, | ||
| ingressNetworkPolicyRuleAction, | ||
| sourcePodName, | ||
| sourceTransportPort, | ||
| sourcePodNamespace, | ||
| destinationPodName, | ||
| destinationTransportPort, | ||
| destinationPodNamespace, | ||
| destinationServicePort, | ||
| destinationServicePortName, | ||
| destinationIP, | ||
| clusterUUID; | ||
|
|
||
| INSERT INTO ".inner.flows_pod_view_local" SELECT * FROM pod_view_table_local; | ||
| INSERT INTO ".inner.flows_node_view_local" SELECT * FROM node_view_table_local; | ||
| INSERT INTO ".inner.flows_policy_view_local" SELECT * FROM policy_view_table_local; | ||
|
|
||
| DROP TABLE pod_view_table_local; | ||
| DROP TABLE node_view_table_local; | ||
| DROP TABLE policy_view_table_local; |
Oops, something went wrong.