Merge pull request #381 from MachineShop-IOT/master

Compact for boltdb (workaround for #374)

index/store/boltdb/store.go

@@ -18,6 +18,7 @@
 package boltdb
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 
@@ -26,7 +27,10 @@ import (
 	"github.com/boltdb/bolt"
 )
 
-const Name = "boltdb"
+const (
+	Name                    = "boltdb"
+	defaultCompactBatchSize = 100
+)
 
 type Store struct {
 	path   string
@@ -102,6 +106,46 @@ func (bs *Store) Stats() json.Marshaler {
 	}
 }
 
+// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
+// Removing entries is a workaround for github issue #374.
+func (bs *Store) CompactWithBatchSize(batchSize int) error {
+	for {
+		cnt := 0
+		err := bs.db.Batch(func(tx *bolt.Tx) error {
+			c := tx.Bucket([]byte(bs.bucket)).Cursor()
+			prefix := []byte("d")
+
+			for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
+				if bytes.Equal(v, []byte{0}) {
+					cnt++
+					if err := c.Delete(); err != nil {
+						return err
+					}
+					if cnt == batchSize {
+						break
+					}
+				}
+
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		if cnt == 0 {
+			break
+		}
+	}
+	return nil
+}
+
+// Compact calls CompactWithBatchSize with a default batch size of 100.  This is a workaround
+// for github issue #374.
+func (bs *Store) Compact() error {
+	return bs.CompactWithBatchSize(defaultCompactBatchSize)
+}
+
 func init() {
 	registry.RegisterKVStore(Name, New)
 }

index/store/goleveldb/store.go

@@ -21,8 +21,8 @@ import (
 )
 
 const (
-	Name             = "goleveldb"
-	defaultBatchSize = 250
+	Name                    = "goleveldb"
+	defaultCompactBatchSize = 250
 )
 
 type Store struct {
@@ -132,7 +132,7 @@ func (ldbs *Store) CompactWithBatchSize(batchSize int) error {
 // Compact compacts the underlying goleveldb store.  The current implementation includes a workaround
 // for github issue #374 (see CompactWithBatchSize).
 func (ldbs *Store) Compact() error {
-	return ldbs.CompactWithBatchSize(defaultBatchSize)
+	return ldbs.CompactWithBatchSize(defaultCompactBatchSize)
 }
 
 func init() {