From f6907aa71e82c852252bd5296786836a37c681f7 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Thu, 26 Nov 2020 10:19:44 +0100 Subject: [PATCH 1/2] feat: CSV parser is able to parse export from UI --- influxdb_client/client/flux_csv_parser.py | 18 +++++++++++++----- tests/test_FluxCSVParser.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/influxdb_client/client/flux_csv_parser.py b/influxdb_client/client/flux_csv_parser.py index 979fb70e..4fbb5693 100644 --- a/influxdb_client/client/flux_csv_parser.py +++ b/influxdb_client/client/flux_csv_parser.py @@ -13,6 +13,12 @@ from influxdb_client.client.flux_table import FluxTable, FluxColumn, FluxRecord +ANNOTATION_DEFAULT = "#default" +ANNOTATION_GROUP = "#group" +ANNOTATION_DATATYPE = "#datatype" +ANNOTATIONS = [ANNOTATION_DEFAULT, ANNOTATION_GROUP, ANNOTATION_DATATYPE] + + class FluxQueryException(Exception): """The exception from InfluxDB.""" @@ -68,6 +74,7 @@ def _parse_flux_response(self): table_id = -1 start_new_table = False table = None + groups = [] parsing_state_error = False for csv in self._reader: @@ -90,7 +97,7 @@ def _parse_flux_response(self): token = csv[0] # start new table - if "#datatype" == token: + if token in ANNOTATIONS and not start_new_table: # Return already parsed DataFrame if (self._serialization_mode is FluxSerializationMode.dataFrame) & hasattr(self, '_data_frame'): @@ -105,18 +112,19 @@ def _parse_flux_response(self): raise FluxCsvParserException("Unable to parse CSV response. FluxTable definition was not found.") # # datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string,string,string - if "#datatype" == token: + if ANNOTATION_DATATYPE == token: self.add_data_types(table, csv) - elif "#group" == token: - self.add_groups(table, csv) + elif ANNOTATION_GROUP == token: + groups = csv - elif "#default" == token: + elif ANNOTATION_DEFAULT == token: self.add_default_empty_values(table, csv) else: # parse column names if start_new_table: + self.add_groups(table, groups) self.add_column_names_and_tags(table, csv) start_new_table = False # Create DataFrame with default values diff --git a/tests/test_FluxCSVParser.py b/tests/test_FluxCSVParser.py index 2019616b..30b993ae 100644 --- a/tests/test_FluxCSVParser.py +++ b/tests/test_FluxCSVParser.py @@ -141,6 +141,23 @@ def test_response_with_error(self): self.assertEqual('engine: unknown field type for value: xyz', exception.message) self.assertEqual('', exception.reference) + def test_ParseExportFromUserInterface(self): + + data = "#group,false,false,true,true,true,true,true,true,false,false\n" \ + + "#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,double,dateTime:RFC3339\n" \ + + "#default,mean,,,,,,,,,\n" \ + + ",result,table,_start,_stop,_field,_measurement,city,location,_value,_time\n" \ + + ",,0,1754-06-26T11:30:27.613654848Z,2040-10-27T12:13:46.485Z,temperatureC,weather,London,us-midwest,30,1975-09-01T16:59:54.5Z\n" \ + + ",,1,1754-06-26T11:30:27.613654848Z,2040-10-27T12:13:46.485Z,temperatureF,weather,London,us-midwest,86,1975-09-01T16:59:54.5Z\n"; + + tables = self._parse_to_tables(data=data) + self.assertEqual(2, tables.__len__()) + self.assertEqual(1, tables[0].records.__len__()) + self.assertEqual(1, tables[1].records.__len__()) + self.assertFalse(tables[1].columns[0].group) + self.assertFalse(tables[1].columns[1].group) + self.assertTrue(tables[1].columns[2].group) + @staticmethod def _parse_to_tables(data: str): fp = BytesIO(str.encode(data)) From b4d9af71a0cd36166eefbe31754926d102fca9e9 Mon Sep 17 00:00:00 2001 From: Jakub Bednar Date: Thu, 26 Nov 2020 10:23:18 +0100 Subject: [PATCH 2/2] docs: updated CHANGELOG.md --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bdbfef6..1066cce0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ ## 1.13.0 [unreleased] -### Bug Fixes -1. [#164](https://github.com/influxdata/influxdb-client-python/pull/170): Skip DataFrame rows without data - all fields are nan. +### Features +1. [#171](https://github.com/influxdata/influxdb-client-python/pull/171): CSV parser is able to parse export from UI +### Bug Fixes +1. [#170](https://github.com/influxdata/influxdb-client-python/pull/170): Skip DataFrame rows without data - all fields are nan. ## 1.12.0 [2020-10-30]