From 6e4ce5348f24ef301a7307b05885fb5f00c2d2e7 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 2 Jan 2023 13:35:19 -0500 Subject: [PATCH 1/2] Use log4j 2 for all modules and write logs to file for unit tests --- common/pom.xml | 4 ++ common/src/test/resources/log4j2.properties | 27 +++++++++++ core/pom.xml | 18 ++++++++ core/src/test/resources/log4j2.properties | 31 +++++++++++++ flink/pom.xml | 6 +-- flink/src/test/resources/log4j2.properties | 27 +++++++++++ pom.xml | 46 ++++++++++++++++++- python-adapter/pom.xml | 6 ++- .../src/test/resources/log4j2.properties | 31 +++++++++++++ sql/pom.xml | 4 ++ sql/src/test/resources/log4j2.properties | 31 +++++++++++++ viz/pom.xml | 6 ++- viz/src/test/resources/log4j2.properties | 31 +++++++++++++ 13 files changed, 261 insertions(+), 7 deletions(-) create mode 100644 common/src/test/resources/log4j2.properties create mode 100644 core/src/test/resources/log4j2.properties create mode 100644 flink/src/test/resources/log4j2.properties create mode 100644 python-adapter/src/test/resources/log4j2.properties create mode 100644 sql/src/test/resources/log4j2.properties create mode 100644 viz/src/test/resources/log4j2.properties diff --git a/common/pom.xml b/common/pom.xml index 3a796d1afd..15b4c1bacc 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -33,6 +33,10 @@ jar + + org.apache.logging.log4j + log4j-1.2-api + org.geotools gt-main diff --git a/common/src/test/resources/log4j2.properties b/common/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..3f66cfce2c --- /dev/null +++ b/common/src/test/resources/log4j2.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex diff --git a/core/pom.xml b/core/pom.xml index 715780191b..7e56f959e6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -77,6 +77,10 @@ org.apache.hadoop hadoop-client + + org.apache.logging.log4j + log4j-1.2-api + edu.ucar cdm-core @@ -130,6 +134,20 @@ hadoop-minicluster ${hadoop.version} test + + + log4j + log4j + + + ch.qos.reload4j + reload4j + + + org.slf4j + * + + org.mockito diff --git a/core/src/test/resources/log4j2.properties b/core/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..5f89859463 --- /dev/null +++ b/core/src/test/resources/log4j2.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn diff --git a/flink/pom.xml b/flink/pom.xml index 8814f658ec..01d215a62b 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -135,10 +135,8 @@ scalatest_${scala.compat.version} - ch.qos.reload4j - reload4j - 1.2.21 - test + org.apache.logging.log4j + log4j-1.2-api commons-codec diff --git a/flink/src/test/resources/log4j2.properties b/flink/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..3f66cfce2c --- /dev/null +++ b/flink/src/test/resources/log4j2.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex diff --git a/pom.xml b/pom.xml index 5b360b35a2..35d8dee840 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ 2.13.4 1.19.0 0.16.1 + 2.17.2 3.3.0 3.0 @@ -90,6 +91,15 @@ junit junit + + + org.apache.logging.log4j + log4j-core + + + org.apache.logging.log4j + log4j-slf4j-impl + @@ -193,6 +203,17 @@ spark-core_${scala.compat.version} ${spark.version} provided + + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + org.apache.spark @@ -210,12 +231,35 @@ log4j log4j + + ch.qos.reload4j + reload4j + org.slf4j - slf4j-log4j12 + * + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + provided + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + commons-lang commons-lang diff --git a/python-adapter/pom.xml b/python-adapter/pom.xml index 23622e56e0..c6ae65db1b 100644 --- a/python-adapter/pom.xml +++ b/python-adapter/pom.xml @@ -77,7 +77,11 @@ org.apache.hadoop hadoop-client - + + org.apache.logging.log4j + log4j-1.2-api + + org.locationtech.jts jts-core diff --git a/python-adapter/src/test/resources/log4j2.properties b/python-adapter/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..5f89859463 --- /dev/null +++ b/python-adapter/src/test/resources/log4j2.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn diff --git a/sql/pom.xml b/sql/pom.xml index f77d3a1494..5c4c6d5451 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -73,6 +73,10 @@ org.apache.hadoop hadoop-client + + org.apache.logging.log4j + log4j-1.2-api + org.geotools gt-main diff --git a/sql/src/test/resources/log4j2.properties b/sql/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..5f89859463 --- /dev/null +++ b/sql/src/test/resources/log4j2.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn diff --git a/viz/pom.xml b/viz/pom.xml index fe9c06306f..37f1be4115 100644 --- a/viz/pom.xml +++ b/viz/pom.xml @@ -77,7 +77,11 @@ org.apache.hadoop hadoop-client - + + org.apache.logging.log4j + log4j-1.2-api + + org.locationtech.jts jts-core diff --git a/viz/src/test/resources/log4j2.properties b/viz/src/test/resources/log4j2.properties new file mode 100644 index 0000000000..5f89859463 --- /dev/null +++ b/viz/src/test/resources/log4j2.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.file.ref = File + +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex + +# Ignore messages below warning level from Jetty, because it's a bit verbose +logger.jetty.name = org.sparkproject.jetty +logger.jetty.level = warn From 3ba33b5f80cefbdd6497183a3c75c5fe45568823 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 2 Jan 2023 19:38:55 -0500 Subject: [PATCH 2/2] Remove unnecessary log4j code in flink tests and add slf4j-api dependency so modules could start using it --- flink/pom.xml | 1 + flink/src/test/java/org/apache/sedona/flink/TestBase.java | 3 --- pom.xml | 7 +++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/flink/pom.xml b/flink/pom.xml index 01d215a62b..dc7389c3ab 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -137,6 +137,7 @@ org.apache.logging.log4j log4j-1.2-api + test commons-codec diff --git a/flink/src/test/java/org/apache/sedona/flink/TestBase.java b/flink/src/test/java/org/apache/sedona/flink/TestBase.java index db24dc882b..30650f94b1 100644 --- a/flink/src/test/java/org/apache/sedona/flink/TestBase.java +++ b/flink/src/test/java/org/apache/sedona/flink/TestBase.java @@ -25,8 +25,6 @@ import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; import org.apache.sedona.flink.expressions.Constructors; import org.locationtech.jts.geom.*; import org.wololo.jts2geojson.GeoJSONWriter; @@ -62,7 +60,6 @@ static void initialize() { } static void initialize(boolean enableWebUI) { - Logger.getLogger("org").setLevel(Level.WARN); env = enableWebUI? StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration()): StreamExecutionEnvironment.getExecutionEnvironment(); EnvironmentSettings settings = EnvironmentSettings.newInstance().inStreamingMode().build(); diff --git a/pom.xml b/pom.xml index 35d8dee840..eb9d33b9ea 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,7 @@ 1.19.0 0.16.1 2.17.2 + 1.7.36 3.3.0 3.0 @@ -241,6 +242,12 @@ + + org.slf4j + slf4j-api + ${slf4j.version} + provided + org.apache.logging.log4j log4j-1.2-api