Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable caching when queries are profiled #48195

Merged
merged 5 commits into from
Nov 20, 2019
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,11 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) {
return false;
}

// Profiled queries should not use the cache
if (request.source() != null && request.source().profile()) {
return false;
}

IndexSettings settings = context.indexShard().indexSettings();
// if not explicitly set in the request, use the index setting, if not, use the request
if (request.requestCache() == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,19 @@ public class ContextIndexSearcher extends IndexSearcher {
*/
private static int CHECK_CANCELLED_SCORER_INTERVAL = 1 << 11;

/**
* A policy to bypass the cache entirely
*/
private static final QueryCachingPolicy NEVER_CACHE_POLICY = new QueryCachingPolicy() {
@Override
public void onUse(Query query) {}

@Override
public boolean shouldCache(Query query) {
return false;
}
};

private AggregatedDfs aggregatedDfs;
private QueryProfiler profiler;
private Runnable checkCancelled;
Expand All @@ -79,6 +92,10 @@ public ContextIndexSearcher(IndexReader reader, Similarity similarity, QueryCach
}

public void setProfiler(QueryProfiler profiler) {
if (profiler != null) {
// disable query caching on profiled query
setQueryCachingPolicy(NEVER_CACHE_POLICY);
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

out of curiosity, why isn't returning false on ProfileWeight enough?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because the sub-query might be cacheable so returning false only ensures that we don't cache the query at the profile weight level. Sub queries are still eligible for caching so we have to disable caching entirely.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought we wrapped sub weights too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do and that works for intermediate level but leaves are still eligible for caching so a simple range query could get cached ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't the range weight be wrapped as well?

this.profiler = profiler;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public void extractTerms(Set<Term> set) {

@Override
public boolean isCacheable(LeafReaderContext ctx) {
return subQueryWeight.isCacheable(ctx);
return false;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.index.cache.request.RequestCacheStats;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket;
Expand Down Expand Up @@ -414,9 +415,49 @@ public void testCacheWithFilteredAlias() {
assertCacheState(client, "index", 2, 2);
}

public void testProfileDisableCache() throws Exception {
Client client = client();
assertAcked(
client.admin().indices().prepareCreate("index")
.addMapping("_doc", "k", "type=keyword")
.setSettings(
Settings.builder()
.put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
)
.get()
);
indexRandom(true, client.prepareIndex("index", "_doc").setSource("k", "hello"));
ensureSearchable("index");

int expectedHits = 0;
int expectedMisses = 0;
for (int i = 0; i < 5; i++) {
boolean profile = i % 2 == 0;
SearchResponse resp = client.prepareSearch("index")
.setRequestCache(true)
.setProfile(profile)
.setQuery(QueryBuilders.termQuery("k", "hello"))
.get();
assertSearchResponse(resp);
ElasticsearchAssertions.assertAllSuccessful(resp);
assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
if (profile == false) {
if (i == 1) {
expectedMisses ++;
} else {
expectedHits ++;
}
}
assertCacheState(client, "index", expectedHits, expectedMisses);
}
}

private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
RequestCacheStats requestCacheStats = client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal()
.getRequestCache();
RequestCacheStats requestCacheStats = client.admin().indices().prepareStats(index)
.setRequestCache(true)
.get().getTotal().getRequestCache();
// Check the hit count and miss count together so if they are not
// correct we can see both values
assertEquals(Arrays.asList(expectedHits, expectedMisses, 0L),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LRUQueryCache;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryCachingPolicy;
Expand All @@ -47,6 +48,7 @@
import org.elasticsearch.search.internal.ContextIndexSearcher;
import org.elasticsearch.search.profile.ProfileResult;
import org.elasticsearch.test.ESTestCase;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;

Expand Down Expand Up @@ -81,7 +83,16 @@ public static void setup() throws IOException {
reader = w.getReader();
w.close();
searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
IndexSearcher.getDefaultQueryCache(), MAYBE_CACHE_POLICY);
IndexSearcher.getDefaultQueryCache(), ALWAYS_CACHE_POLICY);
}

@After
public void checkNoCache() {
LRUQueryCache cache = (LRUQueryCache) searcher.getQueryCache();
assertThat(cache.getHitCount(), equalTo(0L));
assertThat(cache.getCacheCount(), equalTo(0L));
assertThat(cache.getTotalCount(), equalTo(cache.getMissCount()));
assertThat(cache.getCacheSize(), equalTo(0L));
}

@AfterClass
Expand Down Expand Up @@ -158,10 +169,6 @@ public void testUseIndexStats() throws IOException {

public void testApproximations() throws IOException {
QueryProfiler profiler = new QueryProfiler();
// disable query caching since we want to test approximations, which won't
// be exposed on a cached entry
ContextIndexSearcher searcher = new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(),
null, MAYBE_CACHE_POLICY);
searcher.setProfiler(profiler);
Query query = new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random());
searcher.count(query);
Expand All @@ -184,7 +191,6 @@ public void testApproximations() throws IOException {

long rewriteTime = profiler.getRewriteTime();
assertThat(rewriteTime, greaterThan(0L));

}

public void testCollector() throws IOException {
Expand Down Expand Up @@ -288,17 +294,4 @@ public boolean shouldCache(Query query) throws IOException {
}

};

private static final QueryCachingPolicy NEVER_CACHE_POLICY = new QueryCachingPolicy() {

@Override
public void onUse(Query query) {}

@Override
public boolean shouldCache(Query query) throws IOException {
return false;
}

};

}