Merge pull request #56 from Tom-debug110/main

使用os包中的ReadFile、ReadDir，详情见golang/go#42026

Author: newpanjing

core/parser.go

@@ -13,7 +13,7 @@ import (
 // Parser 解析器
 func Parser() *global.Config {
 
-	var addr = flag.String("addr", "0.0.0.0:5678", "设置监听地址和端口")
+	var addr = flag.String("addr", "127.0.0.1:5678", "设置监听地址和端口")
 	//兼容windows
 	dir := fmt.Sprintf(".%sdata", string(os.PathSeparator))
 

main.go

@@ -1,6 +1,8 @@
 package main
 
-import "gofound/core"
+import (
+	"gofound/core"
+)
 
 func main() {
 	//初始化容器和参数解析

sdk/SDK 设计指南.md

@@ -0,0 +1,7 @@
+# GoFound SDK设计指南
+
+## 支持自定义配置
+在支持自定义配置的时候，同时提供默认配置项
+## 
+支持`gofound` 提供的所有操作,增删改查等
+

sdk/base.go

@@ -0,0 +1,33 @@
+package gofound
+
+import (
+	"gofound/searcher/model"
+	"gofound/searcher/system"
+	"runtime"
+)
+
+// Query 查询
+func (c *Client) Query(req *model.SearchRequest) (*model.SearchResult, error) {
+	r, err := c.container.GetDataBase(req.Database).MultiSearch(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+func (*Client) GC() {
+	runtime.GC()
+}
+func (c *Client) Status() (map[string]interface{}, error) {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	// TODO 其他系统信息
+	r := map[string]interface{}{
+		"memory": system.GetMemStat(),
+		"cpu":    system.GetCPUStatus(),
+		"disk":   system.GetDiskStat(),
+	}
+	return r, nil
+}

sdk/client.go

@@ -0,0 +1,82 @@
+package gofound
+
+import (
+	"fmt"
+	"gofound/core"
+	"gofound/global"
+	"gofound/searcher"
+	"os"
+	"runtime"
+	"sync"
+)
+
+var once sync.Once
+
+// Client 应该对外部屏蔽细节
+// 尽量少的提供接口，但是又要保证功能性
+type Client struct {
+	config    *global.Config      //服务配置
+	container *searcher.Container //运行实体
+}
+
+func newDefaultConfig() *global.Config {
+	return &global.Config{
+		Addr:        "127.0.0.1:5678",
+		Data:        fmt.Sprintf(".%sdata", string(os.PathSeparator)),
+		Debug:       true,
+		Dictionary:  "./data/dictionary.txt",
+		EnableAdmin: true,
+		Gomaxprocs:  runtime.NumCPU() * 2,
+		Shard:       0,
+		Auth:        "",
+		EnableGzip:  true,
+		Timeout:     10 * 60,
+	}
+}
+func newTokenizerAndContainer(config *global.Config) *searcher.Container {
+	tokenizer := core.NewTokenizer(global.CONFIG.Dictionary)
+	return core.NewContainer(tokenizer)
+}
+
+// NewClient 通过参数进行配置,必须指定全部参数
+func NewClient(config *global.Config) *Client {
+	global.CONFIG = config
+	//初始化分词器
+	container := newTokenizerAndContainer(config)
+	global.Container = container
+	return &Client{
+		config:    config,
+		container: container,
+	}
+}
+
+// Default 使用默认参数创建服务
+func Default() *Client {
+	global.CONFIG = newDefaultConfig()
+	container := newTokenizerAndContainer(global.CONFIG)
+	global.Container = container
+	return &Client{
+		config:    global.CONFIG,
+		container: container,
+	}
+}
+
+// SetAddr 设置Web服务地址
+func (c *Client) SetAddr(addr string) *Client {
+	if addr == "" {
+		return c
+	}
+	c.config.Addr = addr
+	return c
+}
+
+// SetData 设置数据存放地址
+func (c *Client) SetData(dir string) *Client {
+	if dir == "" {
+		return c
+	}
+	c.config.Data = dir
+	return c
+}
+
+//TODO 其他配置项

sdk/database.go

@@ -0,0 +1,32 @@
+package gofound
+
+import (
+	"gofound/searcher"
+
+	"github.com/syndtr/goleveldb/leveldb/errors"
+)
+
+// Show 查看数据库
+func (c *Client) Show() (map[string]*searcher.Engine, error) {
+	// 保持分格一致
+	return c.container.GetDataBases(), nil
+}
+
+// Drop 删除数据库
+func (c *Client) Drop(dbName string) error {
+	if dbName == "" {
+		return errors.New("database not exist")
+	}
+	if err := c.container.DropDataBase(dbName); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Create 创建数据库
+func (c *Client) Create(dbName string) (*searcher.Engine, error) {
+	if dbName == "" {
+		return nil, errors.New("database name is empty")
+	}
+	return c.container.GetDataBase(dbName), nil
+}

sdk/index.go

@@ -0,0 +1,43 @@
+package gofound
+
+import (
+	"errors"
+	"gofound/searcher/model"
+)
+
+// AddIndex 添加索引
+func (c *Client) AddIndex(dbName string, request *model.IndexDoc) error {
+	if request.Text == "" {
+		return errors.New("text is empty")
+	}
+	c.container.GetDataBase(dbName).IndexDocument(request)
+
+	return nil
+}
+
+// BatchAddIndex 批次添加索引
+func (c *Client) BatchAddIndex(dbName string, documents []*model.IndexDoc) error {
+	db := c.container.GetDataBase(dbName)
+	// 数据预处理
+	for _, doc := range documents {
+		if doc.Text == "" {
+			return errors.New("text is empty")
+		}
+		if doc.Document == nil {
+			return errors.New("document is empty")
+		}
+	}
+	for _, doc := range documents {
+		go db.IndexDocument(doc)
+	}
+	return nil
+}
+
+// RemoveIndex 删除索引
+func (c *Client) RemoveIndex(dbName string, data *model.RemoveIndexModel) error {
+	db := c.container.GetDataBase(dbName)
+	if err := db.RemoveIndex(data.Id); err != nil {
+		return err
+	}
+	return nil
+}

sdk/word.go

@@ -0,0 +1,15 @@
+package gofound
+
+// WordCut 分词
+func (c *Client) WordCut(keyword string) []string {
+	return c.container.Tokenizer.Cut(keyword)
+}
+
+// BatchWordCut 批量分词
+func (c *Client) BatchWordCut(keywords []string) *[][]string {
+	res := make([][]string, len(keywords))
+	for _, w := range keywords {
+		res = append(res, c.container.Tokenizer.Cut(w))
+	}
+	return &res
+}

searcher/container.go

@@ -81,7 +81,6 @@ func (c *Container) GetDataBase(name string) *Engine {
 		engine = c.NewEngine(name)
 		c.engines[name] = engine
 		//释放引擎
-
 	}
 
 	return engine

searcher/storage/leveldb_storage.go

@@ -1,12 +1,13 @@
 package storage
 
 import (
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/filter"
-	"github.com/syndtr/goleveldb/leveldb/opt"
 	"log"
 	"sync"
 	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 // LeveldbStorage TODO 要支持事务

searcher/words/tokenizer.go

@@ -2,9 +2,10 @@ package words
 
 import (
 	"embed"
-	"github.com/wangbin/jiebago"
 	"gofound/searcher/utils"
 	"strings"
+
+	"github.com/wangbin/jiebago"
 )
 
 var (

web/service/service.go

@@ -23,8 +23,8 @@ func Callback() map[string]interface{} {
 		"dataSize":       system.GetFloat64MB(utils.DirSizeB(global.CONFIG.Data)),
 		"executable":     os.Args[0],
 		"dbs":            global.Container.GetDataBaseNumber(),
-		//"indexCount":     global.Container.GetIndexCount(),
-		//"documentCount":  global.Container.GetDocumentCount(),
+		//"indexCount":     global.container.GetIndexCount(),
+		//"documentCount":  global.container.GetDocumentCount(),
 		"pid":        os.Getpid(),
 		"enableAuth": global.CONFIG.Auth != "",
 		"enableGzip": global.CONFIG.EnableGzip,