bleve/index/store/goleveldb/store.go

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package goleveldb

import (
	"bytes"
	"fmt"
	"os"

	"github.com/blevesearch/bleve/index/store"
	"github.com/blevesearch/bleve/registry"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

const (
	Name                    = "goleveldb"
	defaultCompactBatchSize = 250
)

type Store struct {
	path string
	opts *opt.Options
	db   *leveldb.DB
	mo   store.MergeOperator

	defaultWriteOptions *opt.WriteOptions
	defaultReadOptions  *opt.ReadOptions
}

func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {

	path, ok := config["path"].(string)
	if !ok {
		return nil, fmt.Errorf("must specify path")
	}
	if path == "" {
		return nil, os.ErrInvalid
	}

	opts, err := applyConfig(&opt.Options{}, config)
	if err != nil {
		return nil, err
	}

	db, err := leveldb.OpenFile(path, opts)
	if err != nil {
		return nil, err
	}

	rv := Store{
		path:                path,
		opts:                opts,
		db:                  db,
		mo:                  mo,
		defaultReadOptions:  &opt.ReadOptions{},
		defaultWriteOptions: &opt.WriteOptions{},
	}
	rv.defaultWriteOptions.Sync = true
	return &rv, nil
}

func (ldbs *Store) Close() error {
	return ldbs.db.Close()
}

func (ldbs *Store) Reader() (store.KVReader, error) {
	snapshot, _ := ldbs.db.GetSnapshot()
	return &Reader{
		store:    ldbs,
		snapshot: snapshot,
	}, nil
}

func (ldbs *Store) Writer() (store.KVWriter, error) {
	return &Writer{
		store: ldbs,
	}, nil
}

// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches), then
// compacts the underlying goleveldb store.  Removing entries is a workaround for github issue #374.
func (ldbs *Store) CompactWithBatchSize(batchSize int) error {
	// workaround for github issue #374 - remove DictionaryTerm keys with count=0
	batch := &leveldb.Batch{}
	for {
		t, err := ldbs.db.OpenTransaction()
		if err != nil {
			return err
		}
		iter := t.NewIterator(util.BytesPrefix([]byte("d")), ldbs.defaultReadOptions)

		for iter.Next() {
			if bytes.Equal(iter.Value(), []byte{0}) {
				k := append([]byte{}, iter.Key()...)
				batch.Delete(k)
			}
			if batch.Len() == batchSize {
				break
			}
		}
		iter.Release()
		if iter.Error() != nil {
			t.Discard()
			return iter.Error()
		}

		if batch.Len() > 0 {
			err := t.Write(batch, ldbs.defaultWriteOptions)
			if err != nil {
				t.Discard()
				return err
			}
			err = t.Commit()
			if err != nil {
				return err
			}
		} else {
			t.Discard()
			break
		}
		batch.Reset()
	}

	return ldbs.db.CompactRange(util.Range{nil, nil})
}

// Compact compacts the underlying goleveldb store.  The current implementation includes a workaround
// for github issue #374 (see CompactWithBatchSize).
func (ldbs *Store) Compact() error {
	return ldbs.CompactWithBatchSize(defaultCompactBatchSize)
}

func init() {
	registry.RegisterKVStore(Name, New)
}