go-ego / riot

Go Open Source, Distributed, Simple and efficient Search Engine; Warning: This is V1 and beta version, because of big memory consume, and the V2 will be rewrite all code.
Apache License 2.0
6.11k stars 473 forks source link

use leveledb to store,but save failed #84

Open busyfree opened 5 years ago

busyfree commented 5 years ago

func main() { if beego.BConfig.RunMode == "dev" { beego.BConfig.WebConfig.DirectoryIndex = true beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" } initSearcher() beego.BeeApp.Server.RegisterOnShutdown( func() { initClose() }) beego.Run() }

func initSearcher() { // 初始化 var dictFile = filepath.Join(beego.AppPath, "conf/dictionary.txt") var stopTokenFile = filepath.Join(beego.AppPath, "conf/stop_tokens.txt") controllers.Searcher.Init(types.EngineOpts{ Using: 1, GseDict: dictFile, StopTokenFile: stopTokenFile, IndexerOpts: &types.IndexerOpts{ IndexType: types.LocsIndex, }, UseStore: true, StoreOnly: true, StoreFolder: "/tmp/shareyou_search", StoreEngine: "ldb", })

} func initClose() { controllers.Searcher.Flush() controllers.Searcher.Close() }

api post to save data into engine 
api.go

package controllers import ( "fmt" "github.com/go-ego/riot/types" "lib/utils" "reflect" "strings" )

var ( Searcher = riot.Engine{} )

const ( // SecondsInADay seconds in a day SecondsInADay = 86400 // MaxTokenProximity max token proximity MaxTokenProximity = 2 )

type ApiController struct { BaseController }

type DocItem struct { Id string json:"id" Timestamp uint64 json:"timestamp" UserName string json:"username" LikeCount uint64 json:"count" Text string json:"text" DocId string json:"docid" }

// WeiboScoringFields weibo scoring fields type DocItemScoringFields struct { Id int64 Type string Timestamp int64 LikeCount int64 }

// WeiboScoringCriteria custom weibo scoring criteria type WeiboScoringCriteria struct { }

// Score score and sort func (criteria WeiboScoringCriteria) Score(doc types.IndexedDoc, fields interface{}) []float32 { if reflect.TypeOf(fields) != reflect.TypeOf(DocItemScoringFields{}) { return []float32{} } wsf := fields.(DocItemScoringFields) output := make([]float32, 3) if doc.TokenProximity > MaxTokenProximity { output[0] = 1.0 / float32(doc.TokenProximity) } else { output[0] = 1.0 } output[1] = float32(wsf.Timestamp / (SecondsInADay 3)) output[2] = float32(doc.BM25 (1 + float32(wsf.LikeCount)/10000)) return output }

func (o *ApiController) Post() {

content := o.Input().Get("content")
typeStr := o.Input().Get("type")
ts := utils.MsTimestampNow()
id, _ := o.GetInt64("id", 0)
count, _ := o.GetInt64("count", 1)
sfId, err := SnowFlakeServer.NextID()
docId := fmt.Sprintf("%d", sfId)
Searcher.Index(docId, types.DocData{
    Content: content,
    Fields: DocItemScoringFields{
        Id:        id,
        Type:      typeStr,
        Timestamp: ts,
        LikeCount: count,
    },
})
Searcher.Flush()
return

}

func (o *ApiController) Get() { keyword := o.Input().Get("keyword") output := Searcher.SearchDoc(types.SearchReq{ Text: keyword, RankOpts: &types.RankOpts{ ScoringCriteria: &WeiboScoringCriteria{}, OutputOffset: 0, MaxOutputs: 100, }, })

docs := make([]*DocItem, 0, 0)
for _, doc := range output.Docs {
    item := DocItem{}
    field, _ := doc.Fields.(DocItemScoringFields)
    item.DocId = doc.DocId
    item.Text = doc.Content
    item.Timestamp = uint64(field.Timestamp)
    for _, t := range output.Tokens {
        item.Text = strings.Replace(item.Text, t, "<font color=red>"+t+"</font>", -1)
    }
    docs = append(docs, &item)
}
return

}



## Description
when ctrl+c to exit the server ,then run again, search the keyword ,but nothing find. any help?
busyfree commented 5 years ago

why leveldb opened at rito/engine.go ,but when write data into leveldb then return leveldb status is closed. image

busyfree commented 5 years ago

Add a retry label to control the error.

image