diff --git a/tests/popmon/spark/test_spark.py b/tests/popmon/spark/test_spark.py index a84af036..94bac442 100644 --- a/tests/popmon/spark/test_spark.py +++ b/tests/popmon/spark/test_spark.py @@ -14,7 +14,8 @@ spark_found = False -def get_spark(): +@pytest.fixture +def spark(): if not spark_found: return None @@ -34,23 +35,12 @@ def get_spark(): return spark -@pytest.fixture -def spark_co(): - """ - :return: Spark configuration - """ - spark = get_spark() - return spark - - @pytest.mark.spark @pytest.mark.skipif(not spark_found, reason="spark not found") @pytest.mark.filterwarnings( "ignore:createDataFrame attempted Arrow optimization because" ) -def test_spark_stability_metrics(): - spark = get_spark() - +def test_spark_stability_metrics(spark): spark_df = spark.createDataFrame(pytest.test_df) # generate metrics directly from spark dataframe @@ -80,7 +70,7 @@ def test_spark_stability_metrics(): @pytest.mark.filterwarnings( "ignore:createDataFrame attempted Arrow optimization because" ) -def test_spark_make_histograms(spark_co): +def test_spark_make_histograms(spark): pytest.age["data"]["name"] = "b'age'" pytest.company["data"]["name"] = "b'company'" pytest.eyesColor["data"]["name"] = "b'eyeColor'" @@ -93,8 +83,6 @@ def test_spark_make_histograms(spark_co): pytest.latitude_longitude["data"]["name"] = "b'latitude:longitude'" pytest.latitude_longitude["data"]["bins:name"] = "unit_func" - spark = get_spark() - spark_df = spark.createDataFrame(pytest.test_df) # test make_histograms() function call with spark df