Skip to content

Commit

Permalink
test(spark): simplify use of spark context fixture
Browse files Browse the repository at this point in the history
  • Loading branch information
sbrugman committed Mar 27, 2021
1 parent 796ab96 commit 8482ba7
Showing 1 changed file with 4 additions and 16 deletions.
20 changes: 4 additions & 16 deletions tests/popmon/spark/test_spark.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
spark_found = False


def get_spark():
@pytest.fixture
def spark():
if not spark_found:
return None

Expand All @@ -34,23 +35,12 @@ def get_spark():
return spark


@pytest.fixture
def spark_co():
"""
:return: Spark configuration
"""
spark = get_spark()
return spark


@pytest.mark.spark
@pytest.mark.skipif(not spark_found, reason="spark not found")
@pytest.mark.filterwarnings(
"ignore:createDataFrame attempted Arrow optimization because"
)
def test_spark_stability_metrics():
spark = get_spark()

def test_spark_stability_metrics(spark):
spark_df = spark.createDataFrame(pytest.test_df)

# generate metrics directly from spark dataframe
Expand Down Expand Up @@ -80,7 +70,7 @@ def test_spark_stability_metrics():
@pytest.mark.filterwarnings(
"ignore:createDataFrame attempted Arrow optimization because"
)
def test_spark_make_histograms(spark_co):
def test_spark_make_histograms(spark):
pytest.age["data"]["name"] = "b'age'"
pytest.company["data"]["name"] = "b'company'"
pytest.eyesColor["data"]["name"] = "b'eyeColor'"
Expand All @@ -93,8 +83,6 @@ def test_spark_make_histograms(spark_co):
pytest.latitude_longitude["data"]["name"] = "b'latitude:longitude'"
pytest.latitude_longitude["data"]["bins:name"] = "unit_func"

spark = get_spark()

spark_df = spark.createDataFrame(pytest.test_df)

# test make_histograms() function call with spark df
Expand Down

0 comments on commit 8482ba7

Please sign in to comment.