回复: [Neo4j] Re: Create FULLTEXT index is no useful in LuceneBatchInserterIndexProvider ?

[shensh...@gmail.com]

hi peter

i found the reason.
Thank you for your help

发送自HTC手机

----- Reply message -----
发件人： "Peter Neubauer" <peter.n...@neotechnology.com>
收件人： <ne...@googlegroups.com>
主题： [Neo4j] Re: Create FULLTEXT index is no useful in LuceneBatchInserterIndexProvider ?
日期： 周四, 3 月 8 日, 2012 年 17:45


Hi there,
you can see a number of setups that are tested under
https://github.com/neo4j/community/blob/master/lucene-index/src/test/java/org/neo4j/index/impl/lucene/TestLuceneBatchInsert.java

Is there something that you could reuse, or write a failing test?

Cheers,

/peter neubauer

G:  neubauer.peter
S:  peter.neubauer
P:  +46 704 106975
L:   http://www.linkedin.com/in/neubauer
T:   @peterneubauer

Neo4j 1.6 released                 - dzone.com/6S4K
The Neo4j Heroku Challenge   - http://neo4j-challenge.herokuapp.com/



2012/3/6 Shouer.Shen <shensh...@gmail.com>:
> Hi, Michael
> the type of nodeIndex and relationshipIndex is BatchInserterIndex,
> there is no shutdown method except flush();
>
> Shen
> On 3月6日, 下午3时37分, Michael Hunger <michael.hun...@neotechnology.com>
> wrote:
>> The same. You have to shutdown both index providers in finish()
>> Am 05.03.2012 um 17:31 schrieb Shouer.Shen:
>>
>>
>>
>>
>>
>>
>>
>> > Hi, peter
>>
>> > I am sure the store dir is right,
>> > and I had tried it again, and the result is the same.
>>
>> > I do not know why !
>> > here is my code:
>>
>> > Importer.java
>>
>> > package com.run.data.importData;
>>
>> > import static org.neo4j.helpers.collection.MapUtil.map;
>> > import static org.neo4j.helpers.collection.MapUtil.stringMap;
>>
>> > import java.io.BufferedReader;
>> > import java.io.File;
>> > import java.io.FileReader;
>> > import java.io.IOException;
>> > import java.io.InputStreamReader;
>> > import java.util.Map;
>>
>> > import org.neo4j.graphdb.index.BatchInserterIndex;
>> > import org.neo4j.graphdb.index.BatchInserterIndexProvider;
>> > import org.neo4j.index.impl.lucene.LuceneBatchInserterIndexProvider;
>> > import org.neo4j.index.impl.lucene.LuceneIndexImplementation;
>> > import org.neo4j.kernel.impl.batchinsert.BatchInserter;
>> > import org.neo4j.kernel.impl.batchinsert.BatchInserterImpl;
>>
>> > import com.run.data.Data;
>> > import com.run.data.RelType;
>> > import com.run.data.Report;
>>
>> > /**
>> > * 批量数据导入工具
>> > * @author shenshouer
>> > *
>> > */
>> > public class Importer {
>>
>> >    private final String NODE_INDEX_NAME = "nodes";
>> >    private final String RELATIONSHIP_INDEX_NAME = "relationships";
>> >    /* 导入报告 */
>> >    private static Report report;
>> >    /* 批量导入工具 */
>> >    private BatchInserter db;
>> >    /* 批量索引建立工具 */
>> >    private BatchInserterIndexProvider indexProvider;
>> >    /* 节点索引工具 */
>> >    private BatchInserterIndex nodeIndex;
>> >    /* 关系索引工具 */
>> >    private BatchInserterIndex relationshipIndex;
>>
>> >    public Importer(File graphDb) {
>> >            final Map<String, String> config = getConfig();
>> >            db = new BatchInserterImpl(graphDb.getAbsolutePath(), config);
>>
>> >            indexProvider = new LuceneBatchInserterIndexProvider(db);
>> >            nodeIndex = indexProvider.nodeIndex(NODE_INDEX_NAME,
>> > LuceneIndexImplementation.FULLTEXT_CONFIG);
>> >            relationshipIndex =
>> > indexProvider.relationshipIndex(RELATIONSHIP_INDEX_NAME,
>> > LuceneIndexImplementation.FULLTEXT_CONFIG);
>>
>> >            report = new Report(10 * 1000 * 1000, 100);
>> >    }
>>
>> >    public static void main(String[] args)  throws Exception {
>>
>> >            String dbPath = "";
>> >            String nodeFilePath = "";
>> >            String relsFilePath = "";
>> >            int i = 0;
>> >            do{
>> >                    BufferedReader stdin = new BufferedReader(new
>> > InputStreamReader(System.in));
>> >                    switch(i){
>> >                    case 0 :
>> >                            System.out.println("请指定数据库目录：");
>> >                            dbPath = stdin.readLine();
>> >                            break;
>> >                    case 1 :
>> >                            System.out.println("请指定node数据文件位置：");
>> >                            nodeFilePath = ""+stdin.readLine();
>> >                            break;
>> >                    case 2 :
>> >                            System.out.println("请指定relationship数据文件位置：");
>> >                            relsFilePath = ""+stdin.readLine();
>> >                            break;
>> >                    }
>>
>> >                    i++;
>> >            }while( i < 3);
>>
>> >            System.out.println("db dir is : "+ dbPath);
>> >            System.out.println("node file path is : "+ nodeFilePath);
>> >            System.out.println("relationship file path is : " + relsFilePath);
>>
>> >            File graphDb = new File(dbPath);
>> >            File nodesFile = new File(nodeFilePath);
>> >            File relationshipsFile = new File(relsFilePath);
>> >            if (!graphDb.exists()) graphDb.mkdirs();
>> >            Importer importBatch = new Importer(graphDb);
>> >            try {
>> >                    if (nodesFile.exists())
>> >                            importBatch.importNodes(nodesFile);
>> >                    if (relationshipsFile.exists())
>> >                            importBatch.importRelationships(relationshipsFile);
>> >            } finally {
>> >                    importBatch.finish();
>> >            }
>> >    }
>>
>> >    private void importNodes(File file) throws IOException {
>> >            BufferedReader bf = new BufferedReader(new FileReader(file));
>> >            final Data data = new Data(bf.readLine(), "\t", 0);
>> >            String line;
>> >            report.reset();
>> >            Map<String, Object> properties;
>> >            long nodeId;
>>
>> >            while ((line = bf.readLine()) != null) {
>> >                    properties = map(data.update(line));
>> >                    nodeId = db.createNode(properties);
>> >                    nodeIndex.add(nodeId, properties);
>> >                    report.dots();
>> >            }
>>
>> >            report.finishImport("Nodes");
>> >    }
>>
>> >    private void importRelationships(File file) throws IOException {
>> >            BufferedReader bf = new BufferedReader(new FileReader(file));
>> >            final Data data = new Data(bf.readLine(), "\t", 3);
>> >            Object[] rel = new Object[3];
>> >            final RelType type = new RelType();
>> >            String line;
>> >            report.reset();
>> >            Map<String, Object> properties;
>> >            long relationshipId;
>>
>> >            while ((line = bf.readLine()) != null) {
>>
>> >                    properties = map(data.update(line, rel));
>> >                    relationshipId = db.createRelationship(id(rel[0]), id(rel[1]),
>> > type.update(rel[2]), properties);
>> >                    relationshipIndex.add(relationshipId, properties);
>>
>> >                    report.dots();
>> >            }
>> >            report.finishImport("Relationships");
>> >    }
>>
>> >    /**
>> >     * 获取配置文件
>> >     * @return
>> >     */
>> >    private Map<String, String> getConfig() {
>> >            if (new File("batch.properties").exists()) {
>> >                    return BatchInserterImpl.loadProperties("batch.properties");
>> >            } else {
>> >                    return stringMap(
>> >                                    "dump_configuration", "true",
>> >                                    "cache_type", "none",
>> >                                    "neostore.propertystore.db.index.keys.mapped_memory", "5M",
>> >                                    "neostore.propertystore.db.index.mapped_memory", "5M",
>> >                                    "neostore.nodestore.db.mapped_memory", "50M",
>> >                                    "neostore.relationshipstore.db.mapped_memory", "250M",
>> >                                    "neostore.propertystore.db.mapped_memory", "200M",
>> >                                    "neostore.propertystore.db.strings.mapped_memory", "100M");
>> >            }
>> >    }
>>
>> >    private long id(Object id) {
>> >            return Long.parseLong(id.toString());
>> >    }
>>
>> >    private void finish() {
>> >            db.shutdown();
>> >            report.finish();
>> >    }
>> > }
>>
>> > On 3月5日, 下午9时39分, Peter Neubauer <peter.neuba...@neotechnology.com>
>> > wrote:
>> >> Do you have the store somewhere? I guess the index querying is not
>> >> correct, but no idea without poking :)
>>
>> >> Cheers,
>>
>> >> /peter neubauer
>>
>> >> G:  neubauer.peter
>> >> S:  peter.neubauer
>> >> P:  +46 704 106975
>> >> L:  http://www.linkedin.com/in/neubauer
>> >> T:   @peterneubauer
>>
>> >> Neo4j 1.6 released                 - dzone.com/6S4K
>> >> The Neo4j Heroku Challenge   -http://neo4j-challenge.herokuapp.com/
>>
>> >> 2012/3/5 Shouer.Shen <shenshoue...@gmail.com>:
>>
>> >>> The Node I query watched from Data browser like :
>>
>> >>> Node 1
>> >>>http://localhost:7474/db/data/node/1
>>
>> >>> Node            1
>> >>> Rels             2
>> >>> Property       "TEST_1"
>>
>> >>> Shen
>>
>> >>> On 3月5日, 下午6时24分, "Shouer.Shen" <shenshoue...@gmail.com> wrote:
>> >>>> On windows 7 64 bit platform !
>>
>> >>>> shen
>>
>> >>>> On 3月5日, 下午6时21分, "Shouer.Shen" <shenshoue...@gmail.com> wrote:
>>
>> >>>>> The edition is neo4j-enterprise-1.6.1
>>
>> >>>>> who can help me ?
>>
>> >>>>> shen !
>>
>> >>>>> On 3月5日, 下午6时19分, "Shouer.Shen" <shenshoue...@gmail.com> wrote:
>>
>> >>>>>> here is my code :
>>
>> >>>>>> // define index name
>> >>>>>> private final String NODE_INDEX_NAME = "nodes";
>> >>>>>> private final String RELATIONSHIP_INDEX_NAME = "relationships";
>>
>> >>>>>> // create indexProvider
>> >>>>>> db = new BatchInserterImpl(graphDb.getAbsolutePath(), config);
>>
>> >>>>>> indexProvider = new LuceneBatchInserterIndexProvider(db);
>> >>>>>> nodeIndex = indexProvider.nodeIndex(NODE_INDEX_NAME,
>> >>>>>> LuceneIndexImplementation.FULLTEXT_CONFIG);
>> >>>>>> relationshipIndex =
>> >>>>>> indexProvider.relationshipIndex(RELATIONSHIP_INDEX_NAME,
>> >>>>>> LuceneIndexImplementation.FULLTEXT_CONFIG);
>>
>> >>>>>> // import nodes
>> >>>>>> private void importNodes(File file) throws IOException {
>> >>>>>>                BufferedReader bf = new BufferedReader(new FileReader(file));
>> >>>>>>                final Data data = new Data(bf.readLine(), "\t", 0);
>> >>>>>>                String line;
>> >>>>>>                report.reset();
>> >>>>>>                Map<String, Object> properties;
>> >>>>>>                long nodeId;
>>
>> >>>>>>                while ((line = bf.readLine()) != null) {
>> >>>>>>                        properties = map(data.update(line));
>> >>>>>>                        nodeId = db.createNode(properties);
>> >>>>>>                        nodeIndex.add(nodeId, properties);
>> >>>>>>                        report.dots();
>> >>>>>>                }
>>
>> >>>>>>                report.finishImport("Nodes");
>> >>>>>>        }
>>
>> >>>>>> // import relationship
>> >>>>>> private void importRelationships(File file) throws IOException {
>> >>>>>>                BufferedReader bf = new BufferedReader(new FileReader(file));
>> >>>>>>                final Data data = new Data(bf.readLine(), "\t", 3);
>> >>>>>>                Object[] rel = new Object[3];
>> >>>>>>                final RelType type = new RelType();
>> >>>>>>                String line;
>> >>>>>>                report.reset();
>> >>>>>>                Map<String, Object> properties;
>> >>>>>>                long relationshipId;
>>
>> >>>>>>                while ((line = bf.readLine()) != null) {
>>
>> >>>>>>                        properties = map(data.update(line, rel));
>> >>>>>>                        relationshipId = db.createRelationship(id(rel[0]), id(rel[1]),
>> >>>>>> type.update(rel[2]), properties);
>> >>>>>>                        relationshipIndex.add(relationshipId, properties);
>>
>> >>>>>>                        report.dots();
>> >>>>>>                }
>> >>>>>>                report.finishImport("Relationships");
>> >>>>>>        }
>>
>> >>>>>> // import result
>> >>>>>> db dir is : E:\document\neo4j\Neo4j\neo4j-enterprise-1.6.1\data
>> >>>>>> \graph.db
>> >>>>>> node file path is : E:\document\neo4j\rest\java\nodes.csv
>> >>>>>> relationship file path is : E:\document\neo4j\rest\java\rels.csv
>> >>>>>> Physical mem: 4007MB, Heap size: 890MB
>> >>>>>> use_memory_mapped_buffers=false
>> >>>>>> neostore.propertystore.db.index.keys.mapped_memory=5M
>> >>>>>> neostore.propertystore.db.strings.mapped_memory=100M
>> >>>>>> neostore.propertystore.db.arrays.mapped_memory=52M
>> >>>>>> neo_store=E:\document\neo4j\Neo4j\neo4j-enterprise-1.6.1\data\graph.db
>> >>>>>> \neostore
>> >>>>>> neostore.relationshipstore.db.mapped_memory=250M
>>
>> ...
>>
>> 阅读更多 >>

[Peter Neubauer]

Great,
glad you sorted it out!

Cheers,

/peter neubauer

G: neubauer.peter
S: peter.neubauer
P: +46 704 106975
L: http://www.linkedin.com/in/neubauer
T: @peterneubauer

Neo4j 1.6 released - dzone.com/6S4K
The Neo4j Heroku Challenge - http://neo4j-challenge.herokuapp.com/