From 8013500153ff8bbd2bd190ce245dab4fcda4c107 Mon Sep 17 00:00:00 2001 From: Shinsuke Sugaya Date: Mon, 4 Nov 2024 16:44:18 +0900 Subject: [PATCH] add weight --- .../org/codelibs/fess/crawler/entity/EsUrlQueue.java | 3 +++ .../crawler/service/impl/AbstractCrawlerService.java | 8 +++++--- .../fess/crawler/service/impl/EsUrlQueueService.java | 3 ++- .../src/main/resources/mapping/queue.json | 3 +++ .../org/codelibs/fess/crawler/entity/UrlQueue.java | 3 +++ .../codelibs/fess/crawler/entity/UrlQueueImpl.java | 12 ++++++++++++ 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java index 40b6b595..6b64cbe2 100644 --- a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java +++ b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java @@ -38,6 +38,8 @@ public class EsUrlQueue extends UrlQueueImpl implements ToXContent { public static final String CREATE_TIME = "createTime"; + public static final String WEIGHT = "weight"; + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -62,6 +64,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (createTime != null) { builder.field(CREATE_TIME, createTime); } + builder.field(WEIGHT, weight); builder.endObject(); return builder; } diff --git a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java index 290b9c99..de330bc8 100644 --- a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java +++ b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java @@ -353,7 +353,7 @@ protected T get(final Class clazz, final String sessionId, final String u } protected List getList(final Class clazz, final String sessionId, final QueryBuilder queryBuilder, final Integer from, - final Integer size, final SortBuilder sortBuilder) { + final Integer size, final SortBuilder... sortBuilders) { return getList(clazz, builder -> { if (StringUtil.isNotBlank(sessionId)) { if (queryBuilder instanceof BoolQueryBuilder) { @@ -370,8 +370,10 @@ protected List getList(final Class clazz, final String sessionId, fina } else { builder.setQuery(QueryBuilders.matchAllQuery()); } - if (sortBuilder != null) { - builder.addSort(sortBuilder); + if (sortBuilders != null) { + for (SortBuilder sortBuilder : sortBuilders) { + builder.addSort(sortBuilder); + } } if (from != null) { builder.setFrom(from); diff --git a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java index 23dc14f9..9fac8d3b 100644 --- a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java +++ b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java @@ -253,7 +253,8 @@ public EsUrlQueue poll(final String sessionId) { } protected List fetchUrlQueueList(final String sessionId) { - return getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize, SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC)); + return getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize, + SortBuilders.fieldSort(EsUrlQueue.WEIGHT).order(SortOrder.DESC), SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC)); } @Override diff --git a/fess-crawler-es/src/main/resources/mapping/queue.json b/fess-crawler-es/src/main/resources/mapping/queue.json index b61c9aa6..1526208f 100644 --- a/fess-crawler-es/src/main/resources/mapping/queue.json +++ b/fess-crawler-es/src/main/resources/mapping/queue.json @@ -9,6 +9,9 @@ "method": { "type": "keyword" }, + "weight": { + "type": "float" + }, "createTime": { "type": "long" }, diff --git a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java index ad93f7fe..053c8263 100644 --- a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java +++ b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java @@ -61,4 +61,7 @@ public interface UrlQueue { void setCreateTime(Long createTime); + float getWeight(); + + void setWeight(float weight); } diff --git a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java index 9885f261..1333ede8 100644 --- a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java +++ b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java @@ -40,6 +40,8 @@ public class UrlQueueImpl implements UrlQueue { protected Long createTime; + protected float weight = 1.0f; + /* * (non-Javadoc) * @@ -220,6 +222,16 @@ public void setLastModified(final Long lastModified) { this.lastModified = lastModified; } + @Override + public float getWeight() { + return weight; + } + + @Override + public void setWeight(float weight) { + this.weight = weight; + } + @Override public String toString() { return "UrlQueueImpl [id=" + id + ", sessionId=" + sessionId + ", method=" + method + ", url=" + url + ", encoding=" + encoding