Добрый вечер коллеги!
Пытаюсь воспроизвести example из книги Lucene In Action 2nd Ed..
4.2.3 Visualizing analyzers
It’s important to understand what various analyzers do with your text. Seeing the effect of an analyzer is
a powerful and immediate aid to this understanding. We’ll also describe the Attribute class, which
represents each element of a Token, and we’ll discuss each of the Token’s attributes: term,
positionIncrement, offset, type, flags and payload. Listing 4.2 provides a quick and easy way
to get visual feedback about the four primary built-in analyzers on a couple of text examples.
AnalyzerDemo includes two predefined phrases and an array of the four analyzers we’re focusing on in
this section. Each phrase is analyzed by all the analyzers, with bracketed output to indicate the terms that
would be indexed.
С изменениями ибо deprecated и не собирается. Задач несколько:
1) Разобраться как рабоатет StandardAnalyzer.
2) Разобраться с локалями. Проверить анализ смешанного русско-английского текста.
3) Смигрировать с 3.6.x на 4.2.x
4) Написать свой Analyzer
AnalyzerDemo.java
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
package lucene.in.action;
import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.Version;
public class AnalyzerDemo {
private static final String[] examples = {
"The quick brown fox jumped over the lazy dogs",
"XY&Z Corporation - xyz@example.com"
};
private static final Analyzer[] analyzers = new Analyzer[]{
new WhitespaceAnalyzer(Version.LUCENE_42),
new SimpleAnalyzer(Version.LUCENE_42),
new StopAnalyzer(Version.LUCENE_42),
new StandardAnalyzer(Version.LUCENE_42)
};
public static void main(String[] args) throws IOException {
// Use the embedded example strings, unless
// command line arguments are specified, then use those.
String[] strings = examples;
if (args.length > 0) {
strings = args;
}
for (int i = 0; i < strings.length; i++) {
analyze(strings[i]);
}
}
private static void analyze(String text) throws IOException {
System.out.println("Analyzing \"" + text + "\"");
for (int i = 0; i < analyzers.length; i++) {
Analyzer analyzer = analyzers[i];
String name = analyzer.getClass().getName();
name = name.substring(name.lastIndexOf(".") + 1);
System.out.println(" " + name + ":");
System.out.print(" ");
AnalyzerUtils.displayTokens(analyzer, text);
System.out.println("\n");
}
}
}
AnalyzerUtils.java
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
package lucene.in.action;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.AttributeSource.State;
import org.apache.lucene.util.Version;
public class AnalyzerUtils {
public static AttributeSource[] tokensFromAnalysis(Analyzer analyzer,
String text) throws IOException {
TokenStream stream = analyzer.tokenStream("contents", new StringReader(text)); //1
ArrayList tokenList = new ArrayList();
while (true) {
if (!stream.incrementToken()) {
break;
}
tokenList.add(stream.captureState());
}
return (AttributeSource[]) tokenList.toArray(new AttributeSource[0]);
}
public static void displayTokens(Analyzer analyzer,String text) throws IOException {
AttributeSource[] tokens = tokensFromAnalysis(analyzer, text);
for (int i = 0; i < tokens.length; i++) {
AttributeSource token = tokens[i];
// CharTermAttribute term = (CharTermAttribute) token.addAttribute(CharTermAttribute.class);
CharTermAttribute term = (CharTermAttribute) token.addAttribute(CharTermAttribute.class);
//System.out.print("[" + term.term() + "] ");
System.out.print("[" + term.toString() + "] ");
}
}
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
Analyzing "The quick brown fox jumped over the lazy dogs"
WhitespaceAnalyzer:
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: -1
at java.lang.Character.codePointAtImpl(Character.java:4739)
at java.lang.Character.codePointAt(Character.java:4702)
at org.apache.lucene.analysis.util.CharacterUtils$Java5CharacterUtils.codePointAt(CharacterUtils.java:164)
at org.apache.lucene.analysis.util.CharTokenizer.incrementToken(CharTokenizer.java:166)
at lucene.in.action.AnalyzerUtils.tokensFromAnalysis(AnalyzerUtils.java:27)
at lucene.in.action.AnalyzerUtils.displayTokens(AnalyzerUtils.java:36)
at lucene.in.action.AnalyzerDemo.analyze(AnalyzerDemo.java:50)
at lucene.in.action.AnalyzerDemo.main(AnalyzerDemo.java:38)
Java Result: 1
Пока застрял.
