perf: rewrite data access in gooddata-pandas

data_access.py had poor performance because it loaded all attributes. The new implementation fetches only the necessary ones which should perform better on bigger workspaces. JIRA: STL-1036 risk: low
gooddata · Jan 17, 2025 · d484e7a · d484e7a
1 parent ff4c2de
commit d484e7a
Show file tree

Hide file tree

Showing 43 changed files with 892 additions and 30,807 deletions.
diff --git a/gooddata-pandas/gooddata_pandas/data_access.py b/gooddata-pandas/gooddata_pandas/data_access.py
@@ -16,7 +16,7 @@
     ObjId,
     TableDimension,
 )
-from gooddata_sdk.utils import IdObjType
+from gooddata_sdk.utils import IdObjType, filter_for_attributes_labels
 
 from gooddata_pandas.utils import (
     ColumnsDef,
@@ -446,7 +446,12 @@ def compute_and_extract(
     if not exec_def.has_attributes():
         return _extract_for_metrics_only(response, cols, col_to_metric_idx), dict()
     else:
-        attributes = sdk.catalog_workspace_content.get_attributes_catalog(workspace_id, include=["labels", "datasets"])
+        filter_query = filter_for_attributes_labels(exec_def.attributes)
+        # if there is to many labels then all attributes are fetched and no rsql filter is used
+        # it prevention again 414 Request-URI Too Long
+        attributes = sdk.catalog_workspace_content.get_attributes_catalog(
+            workspace_id, include=["labels", "datasets"], rsql_filter=filter_query
+        )
         return _extract_from_attributes_and_maybe_metrics(
             response,
             attributes,

diff --git a/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml b/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_bytes_limits_failure.yaml
@@ -1,4 +1,4 @@
-# (C) 2024 GoodData Corporation
+# (C) 2025 GoodData Corporation
 version: 1
 interactions:
   - request:
@@ -121,7 +121,7 @@ interactions:
         content-length:
           - '1219'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -182,10 +182,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
+              executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b/metadata
       body: null
       headers:
         Accept:
@@ -252,7 +252,7 @@ interactions:
         content-length:
           - '2168'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -355,7 +355,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
+              executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
           resultSize: 4237
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b?offset=0%2C0&limit=100%2C100
       body: null
       headers:
         Accept:
@@ -439,7 +439,7 @@ interactions:
         content-length:
           - '11295'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:

diff --git a/...ata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml b/...ata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_dimensions_limits_failure.yaml
@@ -1,4 +1,4 @@
-# (C) 2024 GoodData Corporation
+# (C) 2025 GoodData Corporation
 version: 1
 interactions:
   - request:
@@ -121,7 +121,7 @@ interactions:
         content-length:
           - '1219'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -182,10 +182,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
+              executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b/metadata
       body: null
       headers:
         Accept:
@@ -252,7 +252,7 @@ interactions:
         content-length:
           - '2168'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -355,7 +355,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_1
             links:
-              executionResult: 7ee8ce1d028718cc038ddca8930ed19b19f8aac4:5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3
+              executionResult: 5a031a05a194b8c615d6ca8e5c09fae3da44b4a3:9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -372,7 +372,7 @@ interactions:
           resultSize: 4237
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/7ee8ce1d028718cc038ddca8930ed19b19f8aac4%3A5fd6ab8e4c7d54aae4276edb69e6ffd56d3a906b81a84bf3bb12e1ac75d3e4e3?offset=0%2C0&limit=100%2C100
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/5a031a05a194b8c615d6ca8e5c09fae3da44b4a3%3A9ac5d29e15fc8f507cf76124ecc28f23748c22bd65d839a3907ae0a4adc38c2b?offset=0%2C0&limit=100%2C100
       body: null
       headers:
         Accept:
@@ -439,7 +439,7 @@ interactions:
         content-length:
           - '11295'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:18 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:53 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:

diff --git a/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml b/gooddata-pandas/tests/dataframe/fixtures/dataframe_for_exec_def_one_dim1.yaml
@@ -1,4 +1,4 @@
-# (C) 2024 GoodData Corporation
+# (C) 2025 GoodData Corporation
 version: 1
 interactions:
   - request:
@@ -119,7 +119,7 @@ interactions:
         content-length:
           - '1179'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:55 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -178,10 +178,10 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
+              executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d/metadata
       body: null
       headers:
         Accept:
@@ -248,7 +248,7 @@ interactions:
         content-length:
           - '2067'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:55 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -349,7 +349,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
+              executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -363,7 +363,7 @@ interactions:
           resultSize: 2913
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d?offset=0&limit=500
       body: null
       headers:
         Accept:
@@ -430,7 +430,7 @@ interactions:
         content-length:
           - '98918'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -4819,7 +4819,7 @@ interactions:
               - 364
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995/metadata
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d/metadata
       body: null
       headers:
         Accept:
@@ -4886,7 +4886,7 @@ interactions:
         content-length:
           - '2067'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string:
@@ -4987,7 +4987,7 @@ interactions:
                         name: Order Amount
                 localIdentifier: dim_0
             links:
-              executionResult: c06b00baf3852d8d1dac66f3231bb224be66a113:b65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995
+              executionResult: 528df9e31711404e621a8003f018633ae97a3d06:5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d
           resultSpec:
             dimensions:
               - localIdentifier: dim_0
@@ -5001,7 +5001,7 @@ interactions:
           resultSize: 2913
   - request:
       method: GET
-      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/c06b00baf3852d8d1dac66f3231bb224be66a113%3Ab65c60a7f92636c16bbfded9b076ffad6173f12cde8ded544841547a3601e995?offset=0&limit=500
+      uri: http://localhost:3000/api/v1/actions/workspaces/demo/execution/afm/execute/result/528df9e31711404e621a8003f018633ae97a3d06%3A5952200aa148f8ef7216a496c7c5ab3c89b429a937ddefdc6722ff86de6ccb4d?offset=0&limit=500
       body: null
       headers:
         Accept:
@@ -5068,7 +5068,7 @@ interactions:
         content-length:
           - '98918'
         set-cookie:
-          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Mon, 07 Oct 2024 14:26:22 GMT;
+          - SPRING_REDIRECT_URI=; Max-Age=0; Expires=Fri, 17 Jan 2025 13:32:56 GMT;
             Path=/; HTTPOnly; SameSite=Lax
       body:
         string: