hankcs
commented
8 years ago 我没有触发这个问题：
public void testHightlight() throws Exception
{
// Lucene Document的主要域名
String fieldName = "text";
// 实例化Analyzer分词器
Analyzer analyzer = new HanLPAnalyzer();
Directory directory = null;
IndexWriter iwriter;
IndexReader ireader = null;
IndexSearcher isearcher;
try
{
//索引过程**********************************
//建立内存索引对象
directory = new RAMDirectory();
//配置IndexWriterConfig
IndexWriterConfig iwConfig = new IndexWriterConfig(analyzer);
iwConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
iwriter = new IndexWriter(directory, iwConfig);
{
// 加入一个文档
Document doc = new Document();
doc.add(new TextField(fieldName, " 软件工程专业是2002年国家教育部新增专业，随着计算机应用领域的不\n" +
"断扩大及中国经济建设的不断发展，软件工程专业将成为一个新的热门专业。软\n" +
"件工程专业以计算机科学与技术学科为基础，\n", Field.Store.YES));
doc.add(new TextField("title", "test.docx", Field.Store.YES));
iwriter.addDocument(doc);
}
{
// 再加入一个
Document doc = new Document();
doc.add(new TextField(fieldName, "\n\n \n程序员\n\n喜欢黑夜", Field.Store.YES));
doc.add(new TextField("title", "关于程序员", Field.Store.YES));
iwriter.addDocument(doc);
}
iwriter.close();
//搜索过程**********************************
//实例化搜索器
ireader = DirectoryReader.open(directory);
isearcher = new IndexSearcher(ireader);
String keyword = "专业";
//使用QueryParser查询分析器构造Query对象
QueryParser qp = new QueryParser(fieldName, analyzer);
Query query = qp.parse(keyword);
System.out.println("Query = " + query);
//搜索相似度最高的5条记录
TopDocs topDocs = isearcher.search(query, 5);
System.out.println("命中：" + topDocs.totalHits);
//输出结果
ScoreDoc[] scoreDocs = topDocs.scoreDocs;
for (int i = 0; i < Math.min(5, scoreDocs.length); ++i)
{
Document targetDoc = isearcher.doc(scoreDocs[i].doc);
System.out.print(targetDoc.getField("title").stringValue());
System.out.println(" , " + scoreDocs[i].score);
String text = targetDoc.get(fieldName);
System.out.println(displayHtmlHighlight(query, analyzer, fieldName, text, 200));
}
}
catch (CorruptIndexException e)
{
e.printStackTrace();
}
catch (LockObtainFailedException e)
{
e.printStackTrace();
}
catch (IOException e)
{
e.printStackTrace();
}
catch (ParseException e)
{
e.printStackTrace();
}
catch (InvalidTokenOffsetsException e)
{
e.printStackTrace();
}
finally
{
if (ireader != null)
{
try
{
ireader.close();
}
catch (IOException e)
{
e.printStackTrace();
}
}
if (directory != null)
{
try
{
directory.close();
}
catch (IOException e)
{
e.printStackTrace();
}
}
}
}
/**
* 获取高亮显示结果的html代码
*
* @param query 查询
* @param analyzer 分词器
* @param fieldName 域名
* @param fieldContent 域内容
* @param fragmentSize 结果的长度（不含html标签长度）
* @return 结果（一段html代码）
* @throws IOException
* @throws InvalidTokenOffsetsException
*/
static String displayHtmlHighlight(Query query, Analyzer analyzer, String fieldName, String fieldContent, int fragmentSize) throws IOException, InvalidTokenOffsetsException
{
//创建一个高亮器
Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("【", "】"), new QueryScorer(query));
Fragmenter fragmenter = new SimpleFragmenter(fragmentSize);
highlighter.setTextFragmenter(fragmenter);
return highlighter.getBestFragment(analyzer, fieldName, fieldContent);
}
drmam
作者你好,我用tika读取文件之后就会出现这个错误 tika是1.6的，hanlp-solr是最新版，下面代码就是您文件中单元测试的代码。测试文件test 测试结果如下： 软件工程专业是2002年国家教育部新增专业，随着计算机应用领域的不 断扩大及中国经济建设的不断发展，软件工程专业将成为一个新的热门专业。软 件工程专业以计算机科学与技术学科为基础，
Query = text:专业 命中：1 测试回测换行符 , 0.2795085 文中有3个词汇”专业“只命中一个，如果使用ik分词器是就不会报错的，希望解决一下。
test.docx
org.apache.lucene.search.highlight.InvalidTokenOffsetsException: Token ， exceeds length of provided text sized 98 at org.apache.lucene.search.highlight.Highlighter.getBestTextFragments(Highlighter.java:225) at org.apache.lucene.search.highlight.Highlighter.getBestFragments(Highlighter.java:156) at org.apache.lucene.search.highlight.Highlighter.getBestFragment(Highlighter.java:102) at org.apache.lucene.search.highlight.Highlighter.getBestFragment(Highlighter.java:80) at com.lib.service.user.HighLighterTest.displayHtmlHighlight(HighLighterTest.java:181) at com.lib.service.user.HighLighterTest.testHightlight(HighLighterTest.java:116) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:483) at junit.framework.TestCase.runTest(TestCase.java:176) at junit.framework.TestCase.runBare(TestCase.java:141) at junit.framework.TestResult$1.protect(TestResult.java:122) at junit.framework.TestResult.runProtected(TestResult.java:142) at junit.framework.TestResult.run(TestResult.java:125) at junit.framework.TestCase.run(TestCase.java:129) at junit.framework.TestSuite.runTest(TestSuite.java:255) at junit.framework.TestSuite.run(TestSuite.java:250) at org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:84) at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86) at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38) at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459) at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675) at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382) at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)
/*
*
Copyright (c) 2008-2015, 码农场. All Right Reserved, http://www.hankcs.com/
This source is subject to Hankcs. Please contact Hankcs to get more information.
*/ package com.lib.service.user;
import junit.framework.TestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.; import org.apache.lucene.search.highlight.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.apache.tika.metadata.Metadata; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.parser.txt.TXTParser; import org.apache.tika.sax.BodyContentHandler; import org.junit.Test; import org.xml.sax.ContentHandler;
import com.hankcs.lucene.HanLPAnalyzer;
import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Scanner;
/**
@author hankcs */ public class HighLighterTest extends TestCase { @Test public void testHightlight() throws Exception { // Lucene Document的主要域名 String fieldName = "text";
}
/**
Parser parser = new AutoDetectParser(); InputStream input = null;
try{
Metadata metadata = new Metadata();
metadata.set(Metadata.CONTENT_ENCODING, "utf-8");
//metadata.set(Metadata.RESOURCE_NAME_KEY, file.getName());
input = new FileInputStream(new File(filePath));
ContentHandler handler = new BodyContentHandler(1024_1024_10);//当文件大于100000时，new BodyContentHandler(1024_1024_10);
ParseContext context = new ParseContext();
//context.set(Parser.class,parser);
parser.parse(input,handler, metadata,context);
/for(String name:metadata.names()) {
System.out.println(name+":"+metadata.get(name));
} / System.out.println(handler.toString()); return handler.toString();
}catch (Exception e){
e.printStackTrace();
}finally {
try {
if(input!=null)input.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
} }