change text_general to text_en to improve English search #3859

IQSS · Apr 29, 2014 · 9b5c634 · 9b5c634
1 parent c7dd849
commit 9b5c634
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 121 deletions.
diff --git a/conf/solr/4.6.0/schema.xml b/conf/solr/4.6.0/schema.xml
diff --git a/scripts/search/tests/expected/highlighting-nick-trees b/scripts/search/tests/expected/highlighting-nick-trees
@@ -21,7 +21,7 @@
         },
         {
             "id":"dataset_17_draft",
-            "matched_fields":"[title, citation_t, description, notesText, authorAffiliation, keyword, contributorName]",
+            "matched_fields":"[title, citation_t, description, notesText, authorAffiliation, authorName, keyword, contributorName]",
             "detailsArray":[
                 {
                     "title":[
@@ -30,7 +30,7 @@
                 },
                 {
                     "citation_t":[
-                        "Tree, Tony, 2014, \"Rings of <span class=\"search-term-match\">Trees</span> and Other Observations\", http://dx.doi.org/10.5072/FK2/17,  Root"
+                        "<span class=\"search-term-match\">Tree</span>, Tony, 2014, \"Rings of Trees and Other Observations\", http://dx.doi.org/10.5072/FK2/17,  Root"
                     ]
                 },
                 {
@@ -48,6 +48,11 @@
                         "<span class=\"search-term-match\">Trees</span> Inc."
                     ]
                 },
+                {
+                    "authorName":[
+                        "<span class=\"search-term-match\">Tree</span>, Tony"
+                    ]
+                },
                 {
                     "keyword":[
                         "<span class=\"search-term-match\">trees</span>"
@@ -73,8 +78,13 @@
         },
         {
             "id":"dataverse_11",
-            "matched_fields":"[name]",
+            "matched_fields":"[description, name]",
             "detailsArray":[
+                {
+                    "description":[
+                        "A <span class=\"search-term-match\">tree</span> dataverse with some birds"
+                    ]
+                },
                 {
                     "name":[
                         "<span class=\"search-term-match\">Trees</span>"

diff --git a/scripts/search/tests/expected/highlighting-pete-trees b/scripts/search/tests/expected/highlighting-pete-trees
@@ -21,7 +21,7 @@
         },
         {
             "id":"dataset_17_draft",
-            "matched_fields":"[title, citation_t, description, notesText, authorAffiliation, keyword, contributorName]",
+            "matched_fields":"[title, citation_t, description, notesText, authorAffiliation, authorName, keyword, contributorName]",
             "detailsArray":[
                 {
                     "title":[
@@ -30,7 +30,7 @@
                 },
                 {
                     "citation_t":[
-                        "Tree, Tony, 2014, \"Rings of <span class=\"search-term-match\">Trees</span> and Other Observations\", http://dx.doi.org/10.5072/FK2/17,  Root"
+                        "<span class=\"search-term-match\">Tree</span>, Tony, 2014, \"Rings of Trees and Other Observations\", http://dx.doi.org/10.5072/FK2/17,  Root"
                     ]
                 },
                 {
@@ -48,6 +48,11 @@
                         "<span class=\"search-term-match\">Trees</span> Inc."
                     ]
                 },
+                {
+                    "authorName":[
+                        "<span class=\"search-term-match\">Tree</span>, Tony"
+                    ]
+                },
                 {
                     "keyword":[
                         "<span class=\"search-term-match\">trees</span>"
@@ -73,8 +78,13 @@
         },
         {
             "id":"dataverse_11",
-            "matched_fields":"[name]",
+            "matched_fields":"[description, name]",
             "detailsArray":[
+                {
+                    "description":[
+                        "A <span class=\"search-term-match\">tree</span> dataverse with some birds"
+                    ]
+                },
                 {
                     "name":[
                         "<span class=\"search-term-match\">Trees</span>"

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java
@@ -341,9 +341,9 @@ public String getDisplayName() {
     }
 
     public SolrField getSolrField() {
-        SolrField.SolrType solrType2 = SolrField.SolrType.TEXT_GENERAL;
+        SolrField.SolrType solrType = SolrField.SolrType.TEXT_EN;
         if (fieldType != null) {
-            solrType2 = fieldType.equals("date") ? SolrField.SolrType.INTEGER : SolrField.SolrType.TEXT_GENERAL;
+            solrType = fieldType.equals("date") ? SolrField.SolrType.INTEGER : SolrField.SolrType.TEXT_EN;
 
             Boolean parentAllowsMultiplesBoolean = false;
             if (isHasParent()) {
@@ -355,22 +355,22 @@ public SolrField getSolrField() {
 
             boolean makeSolrFieldMultivalued;
             // http://stackoverflow.com/questions/5800762/what-is-the-use-of-multivalued-field-type-in-solr
-            if (solrType2 == SolrField.SolrType.TEXT_GENERAL) {
+            if (solrType == SolrField.SolrType.TEXT_EN) {
                 makeSolrFieldMultivalued = (allowMultiples || parentAllowsMultiplesBoolean);
             } else {
                 makeSolrFieldMultivalued = false;
             }
 
-            return new SolrField(name, solrType2, makeSolrFieldMultivalued, facetable);
+            return new SolrField(name, solrType, makeSolrFieldMultivalued, facetable);
 
         } else {
             /**
              * @todo: clean this up
              */
             String oddValue = name + getTmpNullFieldTypeIdentifier();
             boolean makeSolrFieldMultivalued = false;
-            SolrField solrField2 = new SolrField(oddValue, solrType2, makeSolrFieldMultivalued, facetable);
-            return solrField2;
+            SolrField solrField = new SolrField(oddValue, solrType, makeSolrFieldMultivalued, facetable);
+            return solrField;
         }
     }
 

diff --git a/src/main/java/edu/harvard/iq/dataverse/SolrField.java b/src/main/java/edu/harvard/iq/dataverse/SolrField.java
@@ -46,7 +46,12 @@ public boolean isFacetable() {
 
     public enum SolrType {
 
-        STRING("string"), TEXT_GENERAL("text_general"), INTEGER("int"), LONG("long");
+        /**
+         * @todo: make this configurable from text_en to text_general or
+         * non-English languages? We changed it to text_en to improve English
+         * language searching in https://redmine.hmdc.harvard.edu/issues/3859
+         */
+        STRING("string"), TEXT_EN("text_en"), INTEGER("int"), LONG("long");
 
         private String type;
 

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Config.java b/src/main/java/edu/harvard/iq/dataverse/api/Config.java
@@ -26,7 +26,7 @@ public String getSolrSchema() {
             String type = datasetField.getSolrField().getSolrType().getType();
             String multivalued = datasetField.getSolrField().isAllowedToBeMultivalued().toString();
             // <field name="datasetId" type="text_general" multiValued="false" stored="true" indexed="true"/>
-            sb.append("<field name=\"" + nameSearchable + "\" type=\"" + type + "\" multiValued=\"" + multivalued + "\" stored=\"true\" indexed=\"true\"/>\n");
+            sb.append("   <field name=\"" + nameSearchable + "\" type=\"" + type + "\" multiValued=\"" + multivalued + "\" stored=\"true\" indexed=\"true\"/>\n");
         }
 
         List<String> listOfStaticFields = new ArrayList();
@@ -70,7 +70,7 @@ public String getSolrSchema() {
             }
 
             // <copyField source="*_i" dest="text" maxChars="3000"/>
-            sb.append("<copyField source=\"" + nameSearchable + "\" dest=\"text\" maxChars=\"3000\"/>\n");
+            sb.append("   <copyField source=\"" + nameSearchable + "\" dest=\"text\" maxChars=\"3000\"/>\n");
         }
 
         return sb.toString();