renamed/moved examples/bleve_index_json to utils/bleve_index
This commit is contained in:
Marty Schoch
parent
7c0ea53ea2
commit
3bc165d77b
|
|
@ -7,10 +7,10 @@
|
|||
.DS_Store
|
||||
/analysis/token_filters/cld2/cld2-read-only
|
||||
/analysis/token_filters/cld2/libcld2_full.a
|
||||
/examples/bleve_index_json/bleve_index_json
|
||||
/examples/bleve_index_json/index.bleve/
|
||||
/utils/bleve_create/bleve_create
|
||||
/utils/bleve_dump/bleve_dump
|
||||
/utils/bleve_index/bleve_index
|
||||
/utils/bleve_index/index.bleve/
|
||||
/utils/bleve_query/bleve_query
|
||||
/utils/bleve_registry/bleve_registry
|
||||
/y.output
|
||||
|
|
|
|||
|
|
@ -1,76 +0,0 @@
|
|||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
||||
// except in compliance with the License. You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the
|
||||
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions
|
||||
// and limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
|
||||
"github.com/blevesearch/bleve"
|
||||
)
|
||||
|
||||
var jsonDir = flag.String("jsonDir", "json", "json directory")
|
||||
var indexPath = flag.String("index", "index.bleve", "index path")
|
||||
|
||||
func main() {
|
||||
|
||||
flag.Parse()
|
||||
|
||||
// create a new default mapping
|
||||
mapping := bleve.NewIndexMapping()
|
||||
|
||||
// open the index
|
||||
index, err := bleve.New(*indexPath, mapping)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer index.Close()
|
||||
|
||||
for jsonFile := range walkDirectory(*jsonDir) {
|
||||
// index the json files
|
||||
err = index.Index(jsonFile.filename, jsonFile.contents)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type jsonFile struct {
|
||||
filename string
|
||||
contents []byte
|
||||
}
|
||||
|
||||
func walkDirectory(dir string) chan jsonFile {
|
||||
rv := make(chan jsonFile)
|
||||
go func() {
|
||||
defer close(rv)
|
||||
|
||||
// open the directory
|
||||
dirEntries, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// walk the directory entries
|
||||
for _, dirEntry := range dirEntries {
|
||||
// read the bytes
|
||||
jsonBytes, err := ioutil.ReadFile(dir + "/" + dirEntry.Name())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
rv <- jsonFile{
|
||||
filename: dirEntry.Name(),
|
||||
contents: jsonBytes,
|
||||
}
|
||||
}
|
||||
}()
|
||||
return rv
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
||||
// except in compliance with the License. You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the
|
||||
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions
|
||||
// and limitations under the License.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/blevesearch/bleve"
|
||||
)
|
||||
|
||||
var indexPath = flag.String("index", "", "index path")
|
||||
var keepExt = flag.Bool("keepExt", false, "keep extension in doc id")
|
||||
var keepDir = flag.Bool("keepDir", false, "keep dir in doc id")
|
||||
|
||||
func main() {
|
||||
|
||||
flag.Parse()
|
||||
|
||||
if *indexPath == "" {
|
||||
log.Fatal("must specify index path")
|
||||
}
|
||||
|
||||
// open the index
|
||||
index, err := bleve.Open(*indexPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer index.Close()
|
||||
|
||||
if flag.NArg() < 1 {
|
||||
log.Fatal("must specify at least one path to index")
|
||||
}
|
||||
|
||||
for file := range handleArgs(flag.Args()) {
|
||||
// index the files
|
||||
docID := file.filename
|
||||
if !*keepDir {
|
||||
_, docID = filepath.Split(docID)
|
||||
}
|
||||
if !*keepExt {
|
||||
ext := filepath.Ext(docID)
|
||||
docID = docID[0 : len(docID)-len(ext)]
|
||||
}
|
||||
log.Printf("Indexing: %s", docID)
|
||||
err = index.Index(docID, file.contents)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type file struct {
|
||||
filename string
|
||||
contents []byte
|
||||
}
|
||||
|
||||
func handleArgs(args []string) chan file {
|
||||
rv := make(chan file)
|
||||
|
||||
go func() {
|
||||
for _, arg := range args {
|
||||
arg = filepath.Clean(arg)
|
||||
handleArgRecursive(arg, rv)
|
||||
}
|
||||
close(rv)
|
||||
}()
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func handleArgRecursive(arg string, results chan file) {
|
||||
stat, err := os.Stat(arg)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return
|
||||
}
|
||||
if stat.IsDir() {
|
||||
// open the directory
|
||||
dirEntries, err := ioutil.ReadDir(arg)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// walk the directory entries
|
||||
for _, dirEntry := range dirEntries {
|
||||
handleArgRecursive(arg+string(os.PathSeparator)+dirEntry.Name(), results)
|
||||
}
|
||||
} else {
|
||||
bytes, err := ioutil.ReadFile(arg)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
results <- file{
|
||||
filename: arg,
|
||||
contents: bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue