add weight

codelibs · Nov 4, 2024 · 8013500 · 8013500
1 parent 5a20703
commit 8013500
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 4 deletions.
diff --git a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/entity/EsUrlQueue.java
@@ -38,6 +38,8 @@ public class EsUrlQueue extends UrlQueueImpl<String> implements ToXContent {
 
     public static final String CREATE_TIME = "createTime";
 
+    public static final String WEIGHT = "weight";
+
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startObject();
@@ -62,6 +64,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (createTime != null) {
             builder.field(CREATE_TIME, createTime);
         }
+        builder.field(WEIGHT, weight);
         builder.endObject();
         return builder;
     }

diff --git a/...awler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java b/...awler-es/src/main/java/org/codelibs/fess/crawler/service/impl/AbstractCrawlerService.java
@@ -353,7 +353,7 @@ protected <T> T get(final Class<T> clazz, final String sessionId, final String u
     }
 
     protected <T> List<T> getList(final Class<T> clazz, final String sessionId, final QueryBuilder queryBuilder, final Integer from,
-            final Integer size, final SortBuilder<?> sortBuilder) {
+            final Integer size, final SortBuilder<?>... sortBuilders) {
         return getList(clazz, builder -> {
             if (StringUtil.isNotBlank(sessionId)) {
                 if (queryBuilder instanceof BoolQueryBuilder) {
@@ -370,8 +370,10 @@ protected <T> List<T> getList(final Class<T> clazz, final String sessionId, fina
             } else {
                 builder.setQuery(QueryBuilders.matchAllQuery());
             }
-            if (sortBuilder != null) {
-                builder.addSort(sortBuilder);
+            if (sortBuilders != null) {
+                for (SortBuilder<?> sortBuilder : sortBuilders) {
+                    builder.addSort(sortBuilder);
+                }
             }
             if (from != null) {
                 builder.setFrom(from);

diff --git a/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java b/fess-crawler-es/src/main/java/org/codelibs/fess/crawler/service/impl/EsUrlQueueService.java
@@ -253,7 +253,8 @@ public EsUrlQueue poll(final String sessionId) {
     }
 
     protected List<EsUrlQueue> fetchUrlQueueList(final String sessionId) {
-        return getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize, SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC));
+        return getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize,
+                SortBuilders.fieldSort(EsUrlQueue.WEIGHT).order(SortOrder.DESC), SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC));
     }
 
     @Override

diff --git a/fess-crawler-es/src/main/resources/mapping/queue.json b/fess-crawler-es/src/main/resources/mapping/queue.json
@@ -9,6 +9,9 @@
       "method": {
         "type": "keyword"
       },
+      "weight": {
+        "type": "float"
+      },
       "createTime": {
         "type": "long"
       },

diff --git a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueue.java
@@ -61,4 +61,7 @@ public interface UrlQueue<IDTYPE> {
 
     void setCreateTime(Long createTime);
 
+    float getWeight();
+
+    void setWeight(float weight);
 }
diff --git a/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java b/fess-crawler/src/main/java/org/codelibs/fess/crawler/entity/UrlQueueImpl.java
@@ -40,6 +40,8 @@ public class UrlQueueImpl<IDTYPE> implements UrlQueue<IDTYPE> {
 
     protected Long createTime;
 
+    protected float weight = 1.0f;
+
     /*
      * (non-Javadoc)
      *
@@ -220,6 +222,16 @@ public void setLastModified(final Long lastModified) {
         this.lastModified = lastModified;
     }
 
+    @Override
+    public float getWeight() {
+        return weight;
+    }
+
+    @Override
+    public void setWeight(float weight) {
+        this.weight = weight;
+    }
+
     @Override
     public String toString() {
         return "UrlQueueImpl [id=" + id + ", sessionId=" + sessionId + ", method=" + method + ", url=" + url + ", encoding=" + encoding