diff --git a/projects/vdk-plugins/vdk-ingest-http/requirements.txt b/projects/vdk-plugins/vdk-ingest-http/requirements.txt index df19837582..32530a0291 100644 --- a/projects/vdk-plugins/vdk-ingest-http/requirements.txt +++ b/projects/vdk-plugins/vdk-ingest-http/requirements.txt @@ -3,5 +3,6 @@ pytest pytest-httpserver requests +simplejson vdk-core vdk-test-utils diff --git a/projects/vdk-plugins/vdk-ingest-http/setup.py b/projects/vdk-plugins/vdk-ingest-http/setup.py index f237ce1a02..8e3ae7699e 100644 --- a/projects/vdk-plugins/vdk-ingest-http/setup.py +++ b/projects/vdk-plugins/vdk-ingest-http/setup.py @@ -14,7 +14,7 @@ description="Versatile Data Kit SDK ingestion plugin to ingest data via http requests.", long_description=pathlib.Path("README.md").read_text(), long_description_content_type="text/markdown", - install_requires=["vdk-core"], + install_requires=["vdk-core", "simplejson"], package_dir={"": "src"}, packages=setuptools.find_namespace_packages(where="src"), entry_points={ diff --git a/projects/vdk-plugins/vdk-ingest-http/src/vdk/plugin/ingest_http/ingest_over_http.py b/projects/vdk-plugins/vdk-ingest-http/src/vdk/plugin/ingest_http/ingest_over_http.py index 406dc03f99..17b534df65 100644 --- a/projects/vdk-plugins/vdk-ingest-http/src/vdk/plugin/ingest_http/ingest_over_http.py +++ b/projects/vdk-plugins/vdk-ingest-http/src/vdk/plugin/ingest_http/ingest_over_http.py @@ -1,7 +1,6 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 import gzip -import json import logging import sys from typing import Dict @@ -10,6 +9,7 @@ from typing import Optional import requests +import simplejson as json from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from vdk.internal.builtin_plugins.ingestion.ingester_base import IIngesterPlugin diff --git a/projects/vdk-plugins/vdk-ingest-http/tests/test_ingest_over_http.py b/projects/vdk-plugins/vdk-ingest-http/tests/test_ingest_over_http.py index 41bdf416d8..3dce928afe 100644 --- a/projects/vdk-plugins/vdk-ingest-http/tests/test_ingest_over_http.py +++ b/projects/vdk-plugins/vdk-ingest-http/tests/test_ingest_over_http.py @@ -1,12 +1,13 @@ # Copyright 2021 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 import gzip -import json import sys +from decimal import Decimal from unittest import mock from unittest.mock import MagicMock import pytest +import simplejson as json from vdk.internal.core.errors import PlatformServiceError from vdk.internal.core.errors import UserCodeError from vdk.internal.core.errors import VdkConfigurationError @@ -231,7 +232,31 @@ def arg(a): ) -@mock.patch("json.dumps") +@mock.patch("requests.Session.post", side_effect=mocked_requests_post) +def test_ingest_over_http_special_type(mock_post): + http_ingester: IngestOverHttp = IngestOverHttp(job_context) + payload["decimal"] = Decimal(15.5) + http_ingester.ingest_payload( + payload=[payload], + destination_table="test_table", + target="http://example.com/data-source", + ) + + assert mock_post.call_count == 1 + + payload["@table"] = "test_table" + + mock_post.assert_called_with( + url="http://example.com/data-source", + data=json.dumps([payload]), + headers={"Content-Type": "application/octet-stream"}, + timeout=(None, None), + cert=None, + verify=None, + ) + + +@mock.patch("simplejson.dumps") def test_ingest_over_http_json_dumps_parameters_propagation(mock_jsondumps): allow_nan = True