[Neo4j] Index Performance(Was: Event framework has landed)

[Atle Prange]

Started a new thread since the old got a bit long, if you want to
catch up read the thread "The event framework has landed".

Okay, i changed the tests to reflect a bit more realistic usage.

The tests first inserts 1M entries to create a base of data. After
that it makes reads and writes 1000 entries a thousand times.

BabuDB:
First million: 4s
1000 inserts, 4ms
1000 lookups: 30ms

Lucene:
First million entries took 1 ms. This shows the async behavior of Lucene.
1000 inserts: about 4 seconds (!)
1000 lookups: under 1 ms.

(All numbers extremely approximated, and the numbers can only be seen
as relative performance indicators)


This is what i excpected. Lucene is optimized towards collecting large
amount of data batchwise, and then handle many searches. (Correct me
if i am wrong)
BabuDB "just writes" data and "just reads" them later on.

The test can of course be flawed.

BabuDB test:



package org.ogrm.test;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.commons.lang.math.RandomUtils;
import org.xtreemfs.babudb.BabuDB;
import org.xtreemfs.babudb.BabuDBException;
import org.xtreemfs.babudb.BabuDBFactory;
import org.xtreemfs.babudb.config.BabuDBConfig;
import org.xtreemfs.babudb.log.DiskLogger.SyncMode;
import org.xtreemfs.babudb.lsmdb.BabuDBInsertGroup;
import org.xtreemfs.babudb.lsmdb.Database;

public class TestBabuDb {

	private static Database db;

	public static void main( String[] args ) throws Exception {
		deleteFileOrDirectory( new File( "babudb" ) );
		BabuDB babuDb = BabuDBFactory.createBabuDB( new BabuDBConfig(
"babudb/db", "babudb/log", 1, 1024 * 1024 * 20,
				10, SyncMode.ASYNC, 0, 0, false, 512, 1024 * 1024 * 100 ) );
		db = babuDb.getDatabaseManager().createDatabase( "test", 1 );
		int init = 1000000;
		int num = 1000;
		int base = 0;
		int iterations = 1000;
		insert( init, base );
		base = init;
		for (int i = 0; i < iterations; i++) {
			lookup( num, base );
			insert( num, base );
			base = base + num;
		}

		db.shutdown();
		babuDb.shutdown();
	}

	private static byte[] fastToBytes( long value ) throws IOException {
		byte[] array = new byte[8];
		for (int i = 0; i < 8; i++) {
			array[7 - i] = (byte) (value >>> (i * 8));
		}
		return array;
	}

	private static long fastToLong( byte[] array ) throws IOException {
		long value = 0;
		for (int i = 0; i < array.length; i++) {
			value <<= 8;
			value ^= (long) array[i] & 0xFF;
		}
		return value;
	}

	private static byte[] lookupKey( String key, Object value ) {
		return String.valueOf( key + "|" + value + "|" ).getBytes();
	}

	private static byte[] key( long id, String key, Object value ) {
		return String.valueOf( key + "|" + value + "|" + id ).getBytes();
	}

	private static void lookup( int num, int start ) throws Exception {
		long t = System.currentTimeMillis();
		for (int i = start; i < (start + num); i++) {
			Iterator<Entry<byte[], byte[]>> entries = db.prefixLookup( 0,
lookupKey( "key", "value" + i ), null ).get();
			while (entries.hasNext()) {
				Entry<byte[], byte[]> entry = entries.next();
				fastToLong( entry.getValue() );
			}
		}
		System.out.println( num + " lookups:" + (System.currentTimeMillis() - t) );
	}

	private static void insert( int num, int start ) throws Exception {
		long t = System.currentTimeMillis();
		BabuDBInsertGroup group = db.createInsertGroup();

		for (int i = start; i < (num + start); i++) {
			long id = i;
			group.addInsert( 0, key( id, "key", "value" + i % 10000 ),
fastToBytes( id ) );
		}
		db.insert( group, null ).get();
		System.out.println( "insert time (" + num + "):" +
(System.currentTimeMillis() - t) );
	}

	public static void deleteFileOrDirectory( File file ) {
		if (!file.exists()) {
			return;
		}

		if (file.isDirectory()) {
			for (File child : file.listFiles()) {
				deleteFileOrDirectory( child );
			}
			file.delete();
		} else {
			file.delete();
		}
	}

	private static long randomId() {
		return RandomUtils.nextLong();
	}
}


TestLucene


package org.ogrm.test;

import java.io.File;
import java.io.IOException;

import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;

public class TestLucene {
	private static IndexWriter writer;
	private static IndexSearcher searcher;

	public static void main( String[] args ) throws Exception {
		File path = new File( "lcn" );
		deleteFileOrDirectory( path );
		Directory dir = FSDirectory.open( path );
		writer = new IndexWriter( dir, new KeywordAnalyzer(),
MaxFieldLength.UNLIMITED );
		writer.setMaxBufferedDocs( 100000 );

		IndexReader reader = writer.getReader();
		searcher = new IndexSearcher( reader );

		int init = 1000000;
		int num = 1000;
		int base = 0;
		int iterations = 1000;
		insert( init, base );
		base = init;
		for (int i = 0; i < iterations; i++) {
			lookup( num, base );
			insert( num, base );
			base = base + num;
		}
	}

	public static void deleteFileOrDirectory( File file ) {
		if (!file.exists()) {
			return;
		}

		if (file.isDirectory()) {
			for (File child : file.listFiles()) {
				deleteFileOrDirectory( child );
			}
			file.delete();
		} else {
			file.delete();
		}
	}

	private static void insert( int num, int base ) throws Exception {
		long t = System.currentTimeMillis();
		for (int i = num; i < (num + base); i++) {
			Document doc = new Document();
			doc.add( new Field( "_id_", fastToBytes( i ), Store.YES ) );
			doc.add( new Field( "key", "value" + i % 10000, Store.NO,
Index.NOT_ANALYZED ) );
			writer.addDocument( doc );
		}
		writer.commit();
		System.out.println( num + " inserts in " + (System.currentTimeMillis() - t) );
	}

	private static void lookup( int num, int base ) throws Exception {
		long t = System.currentTimeMillis();
		for (int i = 0; i < 100; i++) {
			Query query = new TermQuery( new Term( "key", "value" + (i + 2500000) ) );
			TopDocs docs = searcher.search( query, 100 );
			for (ScoreDoc scoreDoc : docs.scoreDocs) {
				Document doc = searcher.doc( scoreDoc.doc );
				fastToLong( doc.getBinaryValue( "_id_" ) );
			}
		}
		System.out.println( num + " get " + (System.currentTimeMillis() - t) );
	}

	private static byte[] fastToBytes( long value ) throws IOException {
		byte[] array = new byte[8];
		for (int i = 0; i < 8; i++) {
			array[7 - i] = (byte) (value >>> (i * 8));
		}
		return array;
	}

	private static long fastToLong( byte[] array ) throws IOException {
		long value = 0;
		for (int i = 0; i < array.length; i++) {
			value <<= 8;
			value ^= (long) array[i] & 0xFF;
		}
		return value;
	}
}



-atle