Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check before delete #3209

Merged
merged 71 commits into from
Jan 24, 2025
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
71 commits
Select commit Hold shift + click to select a range
432d81f
add logic to detect agent before deleting
xinyual Nov 8, 2024
a273f30
add logic to detect agent before deleting
xinyual Nov 8, 2024
a839db4
add logic to detect pipelines before delete model
xinyual Nov 8, 2024
9691e96
check pipeline before deleting
xinyual Nov 11, 2024
0c100fe
apply spotless
xinyual Nov 11, 2024
aafe91f
remove useless file
xinyual Nov 11, 2024
d93928d
rename functions
xinyual Nov 12, 2024
9f7947e
fix failure test
xinyual Nov 12, 2024
8fea510
add UT
xinyual Nov 12, 2024
ca41ec4
apply spotless
xinyual Nov 12, 2024
56ec29f
renam
xinyual Nov 18, 2024
88d5fd5
refactor to parallel check
xinyual Nov 18, 2024
e6c6bfc
concate error message
xinyual Nov 19, 2024
ba587fd
move logic after user access check
xinyual Nov 19, 2024
de464e9
change agent model searcher map to set
xinyual Nov 19, 2024
ef68097
rename and remove useless method
xinyual Nov 19, 2024
525926f
fix bug to fetch all pipelines
xinyual Nov 19, 2024
5c1b3da
apply spotless
xinyual Nov 19, 2024
7cc0a08
apply spotless
xinyual Nov 19, 2024
b4adbc7
remove and add comment
xinyual Nov 20, 2024
b963587
rename and add more UTs
xinyual Nov 21, 2024
9c831ba
use correct key
xinyual Nov 21, 2024
f53a656
simplify function
xinyual Nov 21, 2024
542de96
change to a better class
xinyual Nov 21, 2024
266d3fa
apply spotless
xinyual Nov 21, 2024
1df7947
change compareAndSet to set
xinyual Nov 25, 2024
cde3934
apply comment
xinyual Nov 25, 2024
6bb7c63
change name and reformat logic
xinyual Dec 4, 2024
b709839
change name
xinyual Dec 4, 2024
642b022
remove useless line
xinyual Dec 4, 2024
eb33bd2
rebase
xinyual Dec 4, 2024
6a38c2b
change to a better method
xinyual Dec 4, 2024
e04d9d6
change name
xinyual Dec 9, 2024
411dc73
apply spotless
xinyual Dec 9, 2024
e29c011
add java doc for function
xinyual Dec 9, 2024
b777398
add another interface
xinyual Dec 13, 2024
273af49
apply java spotless
xinyual Dec 13, 2024
3b142c3
change interface to with model
xinyual Dec 16, 2024
42d2d6f
apply spot less
xinyual Dec 16, 2024
b472292
add settings
xinyual Dec 17, 2024
8ef451e
apply spot less
xinyual Dec 17, 2024
d9e0925
add test for cluster setting
xinyual Dec 18, 2024
c83bfac
apply spotless
xinyual Dec 19, 2024
2a53248
Merge remote-tracking branch 'mainBranch/main' into checkBeforeDelete
xinyual Jan 9, 2025
68773e5
recover useless change
xinyual Jan 10, 2025
99e11ee
change default value of cluster setting
xinyual Jan 16, 2025
7794944
rebase and solve conflict
xinyual Jan 17, 2025
6d5ae9c
rename setting and add comment
xinyual Jan 17, 2025
5607524
apply spot
xinyual Jan 17, 2025
bddacf6
remove logic for hidden model
xinyual Jan 21, 2025
e087473
reorder code
xinyual Jan 22, 2025
a6512f3
reorder code
xinyual Jan 22, 2025
e3aed91
reorder code
xinyual Jan 22, 2025
9fddc49
merge from mainBranch
xinyual Jan 22, 2025
fe342c7
apply spot
xinyual Jan 22, 2025
83340f5
add UT
xinyual Jan 22, 2025
c675707
add more UT
xinyual Jan 22, 2025
8026bd2
remove search for hidden agent
xinyual Jan 23, 2025
5bbc566
fix logic and apply spot
xinyual Jan 23, 2025
acf05e4
add exist for UT
xinyual Jan 23, 2025
96e6023
change dsl to query index
xinyual Jan 23, 2025
5751635
change query logic
xinyual Jan 23, 2025
8ada9a1
remove useless ut
xinyual Jan 23, 2025
e14dbad
rebert
xinyual Jan 23, 2025
07e1bbc
apply spot
xinyual Jan 23, 2025
462437e
rechange code
xinyual Jan 23, 2025
5202cd0
apply spot
xinyual Jan 23, 2025
dae14bd
remove useless should
xinyual Jan 23, 2025
2a7a8f4
Merge remote-tracking branch 'mainBranch/main' into checkBeforeDelete
xinyual Jan 23, 2025
6a3dbcb
apply spot
xinyual Jan 23, 2025
608da39
fix final dsl logic and ut
xinyual Jan 23, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public class CommonValue {
public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words";
public static final Set<String> stopWordsIndices = ImmutableSet.of(".plugins-ml-stop-words");
public static final Integer ML_MEMORY_MESSAGE_INDEX_SCHEMA_VERSION = 1;
public static final String TOOL_MODEL_RELATED_FIELD_PREFIX = "tools.parameters.";
public static final String USER_FIELD_MAPPING = " \""
+ CommonValue.USER
+ "\": {\n"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.opensearch.ml.engine.tools;

import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX;
import static org.opensearch.ml.common.CommonValue.TOOL_MODEL_RELATED_FIELD_PREFIX;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.opensearch.action.search.SearchRequest;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.ml.common.spi.tools.Tool;
import org.opensearch.search.builder.SearchSourceBuilder;

public class AgentModelsSearcher {
private final Set<String> relatedModelIdSet;

public AgentModelsSearcher(Map<String, Tool.Factory> toolFactories) {
relatedModelIdSet = new HashSet<>();
for (Map.Entry<String, Tool.Factory> entry : toolFactories.entrySet()) {
Tool.Factory toolFactory = entry.getValue();
relatedModelIdSet.addAll(toolFactory.getAllModelKeys());
}
}

public SearchRequest constructQueryRequest(String candidateModelId) {
SearchRequest searchRequest = new SearchRequest(ML_AGENT_INDEX);
BoolQueryBuilder shouldQuery = QueryBuilders.boolQuery();
for (String keyField : relatedModelIdSet) {
shouldQuery.should(QueryBuilders.termsQuery(TOOL_MODEL_RELATED_FIELD_PREFIX + keyField, candidateModelId));
}
searchRequest.source(new SearchSourceBuilder().query(shouldQuery));
return searchRequest;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import static org.opensearch.ml.common.utils.StringUtils.gson;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.opensearch.action.ActionRequest;
Expand Down Expand Up @@ -138,6 +139,11 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}

private Map<String, String> extractInputParameters(Map<String, String> parameters) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,11 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}

private Table getTableWithHeader() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,5 +144,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -206,5 +206,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -172,5 +172,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of(MODEL_ID_FIELD);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand Down Expand Up @@ -195,5 +196,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.ml.engine.tools;

import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -174,5 +175,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Loading
Loading