commit b82ae59df5db3188c331cbdd2a281341297eafda

Author: Clement David <clement.david@esi-group.com>

Date:   Wed Jan 15 11:57:58 2020 +0100

    * Bug 15451 fixed: reduce Lucene API usage

    This commit has been checked with lucene 8.4.0 and 5.2.1.

     http://bugzilla.scilab.org/15451

    Change-Id: I9f49d3a2bf9f882fe58f571d7acbcd74cc0a5773

diff --git a/scilab/etc/classpath.xml.vc b/scilab/etc/classpath.xml.vc

index 7ca970baf97..ec367d98d03 100644

--- a/scilab/etc/classpath.xml.vc

+++ b/scilab/etc/classpath.xml.vc

@@ -33,9 +33,9 @@ The option is "disableUnderMode" and can be:

     <path value="$SCILAB/thirdparty/looks-2.1.1.jar"/>

     <path value="$SCILAB/thirdparty/commons-logging.jar"/>

     <path value="$SCILAB/thirdparty/jhall.jar"/>

-    <path value="$SCILAB/thirdparty/lucene-core-5.2.1.jar"/>

-    <path value="$SCILAB/thirdparty/lucene-analyzers-common-5.2.1.jar"/>

-    <path value="$SCILAB/thirdparty/lucene-queryparser-5.2.1.jar"/>

+    <path value="$SCILAB/thirdparty/lucene-core-8.4.0.jar"/>

+    <path value="$SCILAB/thirdparty/lucene-analyzers-common-8.4.0.jar"/>

+    <path value="$SCILAB/thirdparty/lucene-queryparser-8.4.0.jar"/>

     <!-- Load when needed -->

diff --git a/scilab/modules/helptools/helptools.iss b/scilab/modules/helptools/helptools.iss

index 137573891f2..0c8a60805f6 100644

--- a/scilab/modules/helptools/helptools.iss

+++ b/scilab/modules/helptools/helptools.iss

@@ -47,9 +47,9 @@ Source: thirdparty\xml-apis.jar;DestDir: {app}\thirdparty; Components: {#COMPN_S

 Source: thirdparty\xml-apis-ext.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

 Source: thirdparty\xmlgraphics-commons.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

 Source: thirdparty\batik-all.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

-Source: thirdparty\lucene-analyzers-common-5.2.1.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

-Source: thirdparty\lucene-core-5.2.1.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

-Source: thirdparty\lucene-queryparser-5.2.1.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

+Source: thirdparty\lucene-analyzers-common-8.4.0.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

+Source: thirdparty\lucene-core-8.4.0.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

+Source: thirdparty\lucene-queryparser-8.4.0.jar;DestDir: {app}\thirdparty; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

 ;

 Source: modules\{#HELPTOOLS}\license.txt; DestDir: {app}\modules\{#HELPTOOLS}; Components: {#COMPN_SCILAB} and {#COMPN_JVM_MODULE}

 ;

diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java

index d31a0a7c356..455353270d3 100644

--- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java

+++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteIndexer.java

@@ -24,6 +24,7 @@ import java.nio.file.Path;

 import java.nio.file.SimpleFileVisitor;

 import java.nio.file.attribute.BasicFileAttributes;

 import java.util.ArrayList;

+import java.util.HashSet;

 import java.util.List;

 import java.util.Map;

 import java.util.Optional;

@@ -70,11 +71,8 @@ public final class PaletteIndexer {

     public void createIndex(Map<String, PaletteBlock> blockNameToPalette) {

         try {

             mgr.getIndexWriter().deleteAll();

-

-            // insert all block names

-            for (String blk : blockNameToPalette.keySet()) {

-                index(blk);

-            }

+            

+            HashSet<String> blocks = new HashSet<>(blockNameToPalette.keySet());

             // insert all help pages

             for (File r : roots) {

@@ -98,7 +96,8 @@ public final class PaletteIndexer {

                             }

                         } else if (fname.endsWith(".html")) {

                             // this is a regular file

-                            if (blockNameToPalette.containsKey(basename)) {

+                            if (blocks.contains(basename)) {

+                                blocks.remove(basename);

                                 index(basename, file.toUri().toURL());

                             }

                         }

@@ -107,6 +106,11 @@ public final class PaletteIndexer {

                     }

                 });

             }

+            

+            // insert all missing block names

+            for (String blk : blocks) {

+                index(blk);

+            }

             mgr.getIndexWriter().commit();

         } catch (IOException ex) {

@@ -161,8 +165,7 @@ public final class PaletteIndexer {

             Document doc = new Document();

             // add the block name

-            Field refname = new TextField("refname", basename, Field.Store.YES);

-            refname.setBoost(100f);

+            Field refname = new StringField("refname", basename, Field.Store.YES);

             doc.add(refname);

             // add the refpurpose

@@ -171,21 +174,20 @@ public final class PaletteIndexer {

                 Field refpurpose;

                 if (found.isPresent()) {

-                    refpurpose = new TextField("refpurpose", found.get(), Field.Store.YES);

+                    refpurpose = new TextField("refpurpose", found.get(), Field.Store.NO);

                 } else {

-                    refpurpose = new TextField("refpurpose", "", Field.Store.YES);

+                    refpurpose = new TextField("refpurpose", "", Field.Store.NO);

                 }

-                refpurpose.setBoost(10f);

                 doc.add(refpurpose);

             }

             // add the html content

             try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {

                 doc.add(new TextField("content", r));

-            }

-            mgr.getIndexWriter().addDocument(doc);

+		mgr.getIndexWriter().addDocument(doc);

+            }

         } catch (IOException e) {

             Logger.getLogger(PaletteIndexer.class.getName()).log(Level.SEVERE, null, e);

         }

@@ -195,8 +197,8 @@ public final class PaletteIndexer {

         try {

             Document doc = new Document();

             doc.add(new StringField("refname", block, Field.Store.YES));

-            doc.add(new StringField("refpurpose", block, Field.Store.YES));

-            doc.add(new TextField("content", block, Field.Store.YES));

+            doc.add(new TextField("refpurpose", block, Field.Store.NO));

+            doc.add(new TextField("content", block, Field.Store.NO));

             mgr.getIndexWriter().addDocument(doc);

         } catch (IOException e) {

diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java

index 16516d75100..1325ad2e656 100644

--- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java

+++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearchManager.java

@@ -15,11 +15,16 @@ package org.scilab.modules.xcos.palette;

 import java.io.File;

 import java.io.IOException;

+import static java.util.Comparator.reverseOrder;

 import java.util.HashMap;

 import java.util.List;

 import java.util.Map;

+import static java.util.Map.Entry.comparingByValue;

 import java.util.logging.Level;

 import java.util.logging.Logger;

+import static java.util.stream.Collectors.counting;

+import static java.util.stream.Collectors.groupingBy;

+import static java.util.stream.Collectors.toList;

 import javax.swing.JScrollPane;

 import javax.swing.tree.TreeModel;

@@ -94,14 +99,24 @@ public final class PaletteSearchManager {

             indexIsOutdated = false;

         }

-        List<Document> found = paletteSearcher.search(query);

-        for (Document doc : found) {

-            PaletteBlock block = nameToPalette.get(doc.get("refname"));

+        List<Document> found = paletteSearcher.search(query.toLowerCase());

+        // sort results per maximum frequency of the block name

+        Map<String, Long> freq = found.stream()

+                .map(doc -> doc.get("refname"))

+                .collect(groupingBy(x->x, counting()));

+        List<String> foundNames = freq.entrySet()

+                .stream()

+                .sorted(comparingByValue(reverseOrder()))

+                .map(Map.Entry::getKey)

+                .collect(toList());

+        

+        for (String b : foundNames) {

+            PaletteBlock block = nameToPalette.get(b);

             if (block != null) {

                 view.addBlock(block);

             }

         }

-        view.setText(queryLabel + found.size() + " " + XcosMessages.MATCHES);

+        view.setText(queryLabel + foundNames.size() + " " + XcosMessages.MATCHES);

         view.revalidate();

         scrollPane.revalidate();

     }

diff --git a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java

index 26490582834..334c3e51cfb 100644

--- a/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java

+++ b/scilab/modules/xcos/src/java/org/scilab/modules/xcos/palette/PaletteSearcher.java

@@ -51,13 +51,13 @@ public final class PaletteSearcher {

      * @return paths to the found blocks

      */

     public List<Document> search(String str) {

-        List<Document> found = new ArrayList<>();

+        

+        ArrayList<Document> found = new ArrayList<>();

         try (IndexReader reader = DirectoryReader.open(mgr.getDirectory())) {

             IndexSearcher searcher = new IndexSearcher(reader);

             StandardQueryParser queryParserHelper = new StandardQueryParser();

             queryParserHelper.setAllowLeadingWildcard(true);

-            queryParserHelper.setLowercaseExpandedTerms(true);

             queryParserHelper.setAnalyzer(mgr.getAnalyzer());

             queryParserHelper.setMultiFields(new String[] {"refname", "refpurpose", "content"});

@@ -78,6 +78,7 @@ public final class PaletteSearcher {

         } catch (IOException | QueryNodeException e) {

             Logger.getLogger(PaletteSearcher.class.getName()).log(Level.SEVERE, null, e);

         }

+        

         return found;

     }

 }

diff --git a/scilab/scilab-lib.properties.vc b/scilab/scilab-lib.properties.vc

index c3390dddd57..708f4d23d84 100644

--- a/scilab/scilab-lib.properties.vc

+++ b/scilab/scilab-lib.properties.vc

@@ -18,11 +18,11 @@ jogl2.jar=${thirdparty.dir}/jogl2.jar

 jhall.jar=${thirdparty.dir}/jhall.jar

-lucene-core.jar=${thirdparty.dir}/lucene-core-5.2.1.jar

+lucene-core.jar=${thirdparty.dir}/lucene-core-8.4.0jar

-lucene-analyzers-common.jar=${thirdparty.dir}/lucene-analyzers-common-5.2.1.jar

+lucene-analyzers-common.jar=${thirdparty.dir}/lucene-analyzers-common-8.4.0.jar

-lucene-queryparser.jar=${thirdparty.dir}/lucene-queryparser-5.2.1.jar

+lucene-queryparser.jar=${thirdparty.dir}/lucene-queryparser-8.4.0.jar

 gluegen2.jar=${thirdparty.dir}/gluegen2-rt.jar