Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove excess data fetching on responses page #779

Open
wants to merge 54 commits into
base: v2.0.2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
7a89122
WIP endpoint
IgorKrupenja Jan 2, 2025
23ce93b
WIP structure
IgorKrupenja Jan 2, 2025
5c9c592
PoC BE
IgorKrupenja Jan 2, 2025
98ee671
Clean up
IgorKrupenja Jan 3, 2025
5afea00
Clean up
IgorKrupenja Jan 3, 2025
1349b11
PoC Entities
IgorKrupenja Jan 3, 2025
703b8ad
WIP page size
IgorKrupenja Jan 3, 2025
1e8b9b1
Clean up
IgorKrupenja Jan 3, 2025
c613b2e
WIP filter
IgorKrupenja Jan 3, 2025
0b81042
Clean up
IgorKrupenja Jan 3, 2025
8eb1cca
PoC working filter
IgorKrupenja Jan 3, 2025
cc892c5
Revert "PoC working filter"
IgorKrupenja Jan 3, 2025
a851176
Simplify
IgorKrupenja Jan 3, 2025
8f9d34a
Fix search
IgorKrupenja Jan 3, 2025
2b62af7
Add debounce
IgorKrupenja Jan 3, 2025
d0dd346
Clean up
IgorKrupenja Jan 3, 2025
8c169de
WIP regex template
IgorKrupenja Jan 5, 2025
d8e018c
Move fetch to service
IgorKrupenja Jan 5, 2025
e4b3a2e
Extract hook
IgorKrupenja Jan 5, 2025
03d53ff
Rename stuff
IgorKrupenja Jan 5, 2025
f647867
Add util
IgorKrupenja Jan 5, 2025
f2c5375
AUse util
IgorKrupenja Jan 5, 2025
3f34fd7
Clean up
IgorKrupenja Jan 5, 2025
ce9d7bb
Add fetcher
IgorKrupenja Jan 5, 2025
4000e8c
Fix names
IgorKrupenja Jan 5, 2025
d69f92a
Fix data mapping
IgorKrupenja Jan 5, 2025
014cfa6
Fix type
IgorKrupenja Jan 5, 2025
17f5b3d
Revert unrelated changes
IgorKrupenja Jan 5, 2025
4ea0d49
Revert "Revert unrelated changes"
IgorKrupenja Jan 5, 2025
39f80a4
Add hook
IgorKrupenja Jan 5, 2025
b921b0a
Revert unrelated changes
IgorKrupenja Jan 5, 2025
1540cff
Merge base
IgorKrupenja Jan 5, 2025
4d5652b
Fix
IgorKrupenja Jan 5, 2025
ba39852
WIP responses
IgorKrupenja Jan 6, 2025
c177bcf
Fix response
IgorKrupenja Jan 6, 2025
2879802
Fix response
IgorKrupenja Jan 6, 2025
55afdca
Clean up
IgorKrupenja Jan 6, 2025
e39df79
Clean up
IgorKrupenja Jan 6, 2025
6379e0e
Clean up
IgorKrupenja Jan 6, 2025
64995c5
Add stories template
IgorKrupenja Jan 6, 2025
cc01ad7
Clean up
IgorKrupenja Jan 6, 2025
7d97650
Clean up
IgorKrupenja Jan 6, 2025
d8ae2c0
Clean up
IgorKrupenja Jan 7, 2025
8fd1d08
WIP stories
IgorKrupenja Jan 7, 2025
2eae53f
Add dependency queries
IgorKrupenja Jan 7, 2025
df584a7
WIP data mapping
IgorKrupenja Jan 7, 2025
4a656ed
Rename field
IgorKrupenja Jan 7, 2025
4e44b75
Add field
IgorKrupenja Jan 7, 2025
02f90aa
Clean up
IgorKrupenja Jan 7, 2025
b31136a
Clean up
IgorKrupenja Jan 7, 2025
007dde4
Clean up
IgorKrupenja Jan 7, 2025
cd4e5ad
Fix deps
IgorKrupenja Jan 7, 2025
0723413
Revert unrelated changes
IgorKrupenja Jan 7, 2025
992139d
Merge base
IgorKrupenja Jan 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions DSL/DMapper/hbs/get_responses_dependencies.handlebars
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
[
{
"response": [
{{#each responses}}
{{assign 'response' name}}
{"name": "{{name}}",
{{assign 'response' @key}}
{"response": "{{@key}}",
"text": {{#if this.[0].condition}}"{{{this.[1].text}}}"{{else}}"{{{this.[0].text}}}"{{/if}},
"rules": [
{{assign 'multiple' false}}
{{#each ../rules}}
Expand All @@ -20,7 +22,7 @@
{{#each _source.steps as |step|}}
{{#if (eq @root.response step.action)}}
{{#if @root.multiple}},{{/if}}
"{{../_source.story}}"
"{{../_source.story}}"
{{assign 'multiple' true}}
{{/if}}
{{/each}}
Expand All @@ -29,3 +31,4 @@
}{{#unless @last}},{{/unless}}
{{/each}}
]
}
38 changes: 0 additions & 38 deletions DSL/DMapper/hbs/get_responses_list.handlebars

This file was deleted.

6 changes: 6 additions & 0 deletions DSL/DMapper/hbs/get_responses_names.handlebars
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[
{{~#each data~}}
{{~#if @index}},{{/if~}}
"{{@key}}"
{{~/each~}}
]
3 changes: 3 additions & 0 deletions DSL/OpenSearch/deploy-opensearch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ curl -H "Content-Type: application/x-ndjson" -X PUT "$URL/responses" -ku "$AUTH"
if $MOCK_ALLOWED; then curl -H "Content-Type: application/x-ndjson" -X PUT "$URL/responses/_bulk" -ku "$AUTH" --data-binary "@mock/responses.json"; fi
curl -L -X POST "$URL/_scripts/response-with-name-and-text" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/response-with-name-and-text.json"
curl -L -X POST "$URL/_scripts/response" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/response.json"
curl -L -X POST "$URL/_scripts/responses-with-pagination" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/responses-with-pagination.json"

# intents
curl -XDELETE "$URL/intents?ignore_unavailable=true" -u "$AUTH" --insecure
Expand All @@ -31,6 +32,7 @@ curl -L -X POST "$URL/_scripts/rule-with-name" -H 'Content-Type: application/jso
curl -L -X POST "$URL/_scripts/rule-with-forms" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/rule-with-forms.json"
curl -L -X POST "$URL/_scripts/rule-with-responses" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/rule-with-responses.json"
curl -L -X POST "$URL/_scripts/rule-with-slots" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/rule-with-slots.json"
curl -L -X POST "$URL/_scripts/rules-by-responses" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/rules-by-responses.json"

# stories
curl -XDELETE "$URL/stories?ignore_unavailable=true" -u "$AUTH" --insecure
Expand All @@ -40,6 +42,7 @@ curl -L -X POST "$URL/_scripts/story-with-name" -H 'Content-Type: application/js
curl -L -X POST "$URL/_scripts/story-with-forms" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/story-with-forms.json"
curl -L -X POST "$URL/_scripts/story-with-responses" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/story-with-responses.json"
curl -L -X POST "$URL/_scripts/story-with-slots" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/story-with-slots.json"
curl -L -X POST "$URL/_scripts/stories-by-responses" -H 'Content-Type: application/json' -H 'Cookie: customJwtCookie=test' --data-binary "@templates/stories-by-responses.json"

# test-stories
curl -XDELETE "$URL/test-stories?ignore_unavailable=true" -u "$AUTH" --insecure
Expand Down
6 changes: 6 additions & 0 deletions DSL/OpenSearch/templates/responses-with-pagination.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"script": {
"lang": "mustache",
"source": "{\"query\":{\"term\":{\"_id\":\"responses\"}},\"script_fields\":{\"filtered_responses\":{\"script\":{\"lang\":\"painless\",\"source\":\"def responses = params._source.responses; def searchTerm = '{{filter}}'; int from = {{from}}; int size = {{size}}; def filteredResponses = new HashMap(); for (def entry : responses.entrySet()) { def responseName = entry.getKey(); if (searchTerm == '' || responseName.toLowerCase().contains(searchTerm.toLowerCase())) { filteredResponses.put(responseName, entry.getValue()); } } int start = (int)Math.min(from, filteredResponses.size()); int end = (int)Math.min(from + size, filteredResponses.size()); def result = new HashMap(); int counter = 0; for (def entry : filteredResponses.entrySet()) { if (counter >= start && counter < end) { result.put(entry.getKey(), entry.getValue()); } counter++; } return result;\"}}}}"
}
}
10 changes: 10 additions & 0 deletions DSL/OpenSearch/templates/rules-by-responses.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"script": {
"lang": "mustache",
"source": "{\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"terms\": {\n \"steps.action\": {{#toJson}}responses{{/toJson}}\n }\n }\n ],\n \"minimum_should_match\": 1\n }\n },\n \"_source\": [\"rule\", \"steps.action\"],\n \"size\": {{size}}\n}",
"params": {
"responses": [],
"size": 1000
}
}
}
10 changes: 10 additions & 0 deletions DSL/OpenSearch/templates/stories-by-responses.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"script": {
"lang": "mustache",
"source": "{\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"terms\": {\n \"steps.action\": {{#toJson}}responses{{/toJson}}\n }\n }\n ],\n \"minimum_should_match\": 1\n }\n },\n \"_source\": [\"story\", \"steps.action\"],\n \"size\": {{size}}\n}",
"params": {
"responses": [],
"size": 1000
}
}
}
64 changes: 53 additions & 11 deletions DSL/Ruuter.private/GET/rasa/responses-list.yml
Original file line number Diff line number Diff line change
@@ -1,34 +1,76 @@
declaration:
call: declare
version: 0.1
description: "Decription placeholder for 'RESPONSES-LIST'"
description: "Description placeholder for 'RESPONSES'"
method: get
accepts: json
returns: json
namespace: training
params:
- field: filter
type: string
description: "Query string"
- field: from
type: string
description: "Item number"
- field: size
type: string
description: "Page size"

getResponses:
call: http.get
call: http.post
args:
url: "[#TRAINING_OPENSEARCH]/domain/_search"
query:
size: 10000
q: _id:responses
_source: responses
url: "[#TRAINING_OPENSEARCH]/domain/_search/template"
body:
id: "responses-with-pagination"
params: ${incoming.params}
result: getResponsesResult

mapResponsesNames:
call: http.post
args:
url: "[#TRAINING_DMAPPER]/hbs/training/get_responses_names"
headers:
type: "json"
body:
data: ${getResponsesResult.response.body.hits.hits[0].fields.filtered_responses[0]}
result: responsesNames

getStories:
call: http.post
args:
url: "[#TRAINING_OPENSEARCH]/stories/_search/template"
body:
id: "stories-by-responses"
params:
responses: "${responsesNames.response.body}"
size: 1000
result: getStoriesResult

getRules:
call: http.post
args:
url: "[#TRAINING_OPENSEARCH]/rules/_search/template"
body:
id: "rules-by-responses"
params:
responses: "${responsesNames.response.body}"
size: 1000
result: getRulesResult

mapResponsesData:
call: http.post
args:
url: "[#TRAINING_DMAPPER]/hbs/training/get_responses_list"
url: "[#TRAINING_DMAPPER]/hbs/training/get_responses_dependencies"
headers:
type: "json"
body:
response: ${getResponsesResult.response.body.hits.hits}
rules: ${getRulesResult.response.body.hits.hits}
stories: ${getStoriesResult.response.body.hits.hits}
responses: ${getResponsesResult.response.body.hits.hits[0].fields.filtered_responses[0]}
result: responsesData
next: returnSuccess

returnSuccess:
return: ${responsesData.response.body.responses}
return: ${responsesData.response.body}
wrapper: false
next: end
52 changes: 0 additions & 52 deletions DSL/Ruuter.private/GET/rasa/responses/dependencies.yml

This file was deleted.

Loading