BlogJava-DANCE WITH JAVA-随笔分类-搜索引擎lucence

lucene入门合集

dreamstone — Sun, 29 Jul 2007 12:09:00 GMT

dreamstone 2007-07-29 20:09 发表评论

lucene的中文分词器

dreamstone — Fri, 22 Jun 2007 01:15:00 GMT

lucene的中文分词器到现在还没有好的解决办法。下边介绍了两个lucene自己提供的分词器和一个javaeye上的网友实现的分词器。关于各个分词器的不同见代码中的print信息。直接运行得到console的输出结果更容易对比不同。

package analyzer;

import java.io.Reader;
import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.cn.ChineseAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.mira.lucene.analysis.IK_CAnalyzer;
import org.mira.lucene.analysis.MIK_CAnalyzer;

import com.sohospace.lucene.analysis.xanalyzer.XAnalyzer;
import com.sohospace.lucene.analysis.xanalyzer.XFactory;
import com.sohospace.lucene.analysis.xanalyzer.XTokenizer;
//中文分词使用了Paoding的分词技术，特表示感谢
public class TestCJKAnalyzer {
    private static String testString1 = "中华人民共和国在1949年建立，从此开始了新中国的伟大篇章";
    private static String testString2 = "比尔盖茨从事餐饮业和服务业方面的工作";
    public static void testStandard(String testString) throws Exception{
        Analyzer analyzer = new StandardAnalyzer();
        Reader r = new StringReader(testString);
        StopFilter sf = (StopFilter) analyzer.tokenStream("", r);
        System.err.println("=====standard analyzer====");
        System.err.println("分析方法：默认没有词只有字");
        Token t;
        while ((t = sf.next()) != null) {
            System.out.println(t.termText());
        }
    }
    public static void testCJK(String testString) throws Exception{
        Analyzer analyzer = new CJKAnalyzer();
        Reader r = new StringReader(testString);
        StopFilter sf = (StopFilter) analyzer.tokenStream("", r);
        System.err.println("=====cjk analyzer====");
        System.err.println("分析方法:交叉双字分割");
        Token t;
        while ((t = sf.next()) != null) {
            System.out.println(t.termText());
        }
    }
    public static void testChiniese(String testString) throws Exception{
        Analyzer analyzer = new ChineseAnalyzer();
        Reader r = new StringReader(testString);
        TokenFilter tf = (TokenFilter) analyzer.tokenStream("", r);
        System.err.println("=====chinese analyzer====");
        System.err.println("分析方法:基本等同StandardAnalyzer");
        Token t;
        while ((t = tf.next()) != null) {
            System.out.println(t.termText());
        }
    }
    public static void testPaoding(String testString) throws Exception{
        XAnalyzer analyzer = XFactory.getQueryAnalyzer();
        Reader r = new StringReader(testString);
        XTokenizer ts = (XTokenizer) analyzer.tokenStream("", r);
        System.err.println("=====paoding analyzer====");
        System.err.println("分析方法:字典分词,去掉停止词。在字典不能匹配的情况下使用CJKAnalyzer的分割发。");
        Token t;
        while ((t = ts.next()) != null) {
           System.out.println(t.termText());
        }
    }
    public static void testJe(String testString) throws Exception{
//        Analyzer analyzer = new MIK_CAnalyzer();
        Analyzer analyzer = new IK_CAnalyzer();
        Reader r = new StringReader(testString);
        TokenStream ts = (TokenStream)analyzer.tokenStream("", r);
        System.err.println("=====je analyzer====");
        System.err.println("分析方法:字典分词,正反双向搜索，具体不明");
        Token t;
        while ((t = ts.next()) != null) {
           System.out.println(t.termText());
        }
    }
    public static void main(String[] args) throws Exception{
//        String testString = testString1;
        String testString = testString1;
        System.out.println(testString);

        testStandard(testString);
        testCJK(testString);
        testPaoding(testString);

//        testChiniese(testString);
//        testJe(testString);
    }

}

dreamstone 2007-06-22 09:15 发表评论

lucene的丰富的各种查询（二）

dreamstone — Thu, 21 Jun 2007 07:08:00 GMT

和上文一样，列写lucene的查询用法
包括了RangeQuery prefixQuery phraseQuery wildcastQuery fuzzyQuery
被索引查询的文件，按照需求自己构造即可。

package search;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;

public class SearcherShow {
    private static Directory directory = new RAMDirectory();
    public static void preIndex() throws Exception{
        String fileName1 = "./data/searchShow.txt";
        String fileName2 = "./data/searchShow2.txt";
        String fileName3 = "./data/test.txt";
        IndexWriter writer = new IndexWriter(directory,new StandardAnalyzer(),true);
        Document doc1 = getDocument(fileName1);
        Document doc2 = getDocument(fileName2);
        Document doc3 = getDocument(fileName3);
        writer.addDocument(doc1);
        writer.addDocument(doc2);
        writer.addDocument(doc3);
        writer.close();

    }
    //先建立索引才能执行

    //termQuery   rangeQuery   booleanQuery的查询在Searcher类中
    public static void rangeQuery() throws Exception{
        Term startTerm = new Term("lastmodified","20070620");
        Term endTerm = new Term("lastmodified","20070622");
        RangeQuery query = new RangeQuery(startTerm,endTerm,true);
        IndexSearcher searcher = new IndexSearcher(directory);
        Hits hits = searcher.search(query);
        prtHits(hits);
    }
    public static void prefixQuery() throws Exception{
        Term term = new Term("fileName","searchShow.txt");
        Term prefixterm = new Term("fileName","searchShow");
        IndexSearcher searcher = new IndexSearcher(directory);
        Query query = new TermQuery(term);
        Query prefixQuery = new PrefixQuery(prefixterm);
        Hits hits = searcher.search(query);
        Hits prefixHits = searcher.search(prefixQuery);
        prtHits(hits);
        System.out.println("----------");
        prtHits(prefixHits);
    }
    public static void phraseQuery() throws Exception{
        IndexSearcher searcher = new IndexSearcher(directory);
        PhraseQuery query = new PhraseQuery();
        query.setSlop(2);
        query.add(new Term("contents","quick"));
        query.add(new Term("contents","fox"));
        Hits hits = searcher.search(query);
        prtHits(hits);
    }
    public static void wildcardQuery() throws Exception{
        IndexSearcher searcher = new IndexSearcher(directory);
        Query query = new WildcardQuery(new Term("contents","?ild*"));
        Hits hits = searcher.search(query);
        prtHits(hits);
    }
    public static void fuzzyQuery() throws Exception{
        IndexSearcher searcher = new IndexSearcher(directory);
        Term term = new Term("contents","wuzza");
        FuzzyQuery query = new FuzzyQuery(term);
        Hits hits = searcher.search(query);
        prtHits(hits);
    }
    public static Document getDocument(String fileName) throws Exception{
        File file = new File(fileName);
        Document doc = new Document();
        doc.add(Field.Keyword("fileName",file.getName() ));
        Date modified = new Date(file.lastModified());
        String lastmodified = new SimpleDateFormat("yyyyMMdd").format(modified);
        doc.add(Field.Keyword("lastmodified", lastmodified));
        BufferedReader br = new BufferedReader(new InputStreamReader(
                new FileInputStream(file)));
        StringBuffer sb = new StringBuffer();
        String line = null;
        while ((line = br.readLine()) != null) {
            sb.append(line);
        }
        br.close();
        doc.add(Field.Text("contents",sb.toString() ));
        return doc;
    }
    public static void prtHits(Hits hits) throws Exception{
        for(int i=0;i<hits.length();i++){
            Document doc = hits.doc(i);
            System.out.println(doc.get("fileName"));
            System.out.println(doc.get("lastmodified"));
        }
    }
    public static void main(String[] args) throws Exception{
        preIndex();
//        rangeQuery();
//        prefixQuery();
//        phraseQuery();
//        wildcardQuery();
        fuzzyQuery();
    }
}

dreamstone 2007-06-21 15:08 发表评论

lucene的丰富的各种查询(一)

dreamstone — Thu, 21 Jun 2007 07:06:00 GMT

lucene支持十分丰富的查询，这里列写其中一些比较常用的查询的用法。
term查询、queryParser查询 ,booleanQuery

package search;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

public class Searcher {
    public static void termQuery() throws Exception{
        Directory directory = FSDirectory.getDirectory("./index", false);
        IndexSearcher searcher = new IndexSearcher(directory);
        Term t = new Term("body","document");
        Query query = new TermQuery(t);
        Hits hits = searcher.search(query);
        System.out.println(hits.length());
    }
    public static void queryParser() throws Exception{
        Directory directory = FSDirectory.getDirectory("./index", false);
        IndexSearcher searcher = new IndexSearcher(directory);
        Query query = QueryParser.parse("text","body",new StandardAnalyzer());
        Hits hits = searcher.search(query);
        System.out.println(hits.length());
    }
    public static void booleanQuery() throws Exception{
        Query parseQuery = QueryParser.parse("text","body",new StandardAnalyzer());
        Term t = new Term("body","document");
        Query termQuery = new TermQuery(t);
        BooleanQuery boolQuery = new BooleanQuery();
        boolQuery.add(parseQuery,true,false);
        boolQuery.add(termQuery,true,false);

        Directory directory = FSDirectory.getDirectory("./index", false);
        IndexSearcher searcher = new IndexSearcher(directory);
        Hits hits = searcher.search(boolQuery);
        System.out.println(hits.length());
    }
    public static void main(String[] args) throws Exception{
        termQuery();
        queryParser();
        booleanQuery();
    }
}

dreamstone 2007-06-21 15:06 发表评论