Open busyfree opened 5 years ago
func main() { if beego.BConfig.RunMode == "dev" { beego.BConfig.WebConfig.DirectoryIndex = true beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" } initSearcher() beego.BeeApp.Server.RegisterOnShutdown( func() { initClose() }) beego.Run() }
func initSearcher() { // 初始化 var dictFile = filepath.Join(beego.AppPath, "conf/dictionary.txt") var stopTokenFile = filepath.Join(beego.AppPath, "conf/stop_tokens.txt") controllers.Searcher.Init(types.EngineOpts{ Using: 1, GseDict: dictFile, StopTokenFile: stopTokenFile, IndexerOpts: &types.IndexerOpts{ IndexType: types.LocsIndex, }, UseStore: true, StoreOnly: true, StoreFolder: "/tmp/shareyou_search", StoreEngine: "ldb", })
} func initClose() { controllers.Searcher.Flush() controllers.Searcher.Close() }
api post to save data into engine api.go
package controllers import ( "fmt" "github.com/go-ego/riot/types" "lib/utils" "reflect" "strings" )
var ( Searcher = riot.Engine{} )
const ( // SecondsInADay seconds in a day SecondsInADay = 86400 // MaxTokenProximity max token proximity MaxTokenProximity = 2 )
type ApiController struct { BaseController }
type DocItem struct { Id string json:"id" Timestamp uint64 json:"timestamp" UserName string json:"username" LikeCount uint64 json:"count" Text string json:"text" DocId string json:"docid" }
json:"id"
json:"timestamp"
json:"username"
json:"count"
json:"text"
json:"docid"
// WeiboScoringFields weibo scoring fields type DocItemScoringFields struct { Id int64 Type string Timestamp int64 LikeCount int64 }
// WeiboScoringCriteria custom weibo scoring criteria type WeiboScoringCriteria struct { }
// Score score and sort func (criteria WeiboScoringCriteria) Score(doc types.IndexedDoc, fields interface{}) []float32 { if reflect.TypeOf(fields) != reflect.TypeOf(DocItemScoringFields{}) { return []float32{} } wsf := fields.(DocItemScoringFields) output := make([]float32, 3) if doc.TokenProximity > MaxTokenProximity { output[0] = 1.0 / float32(doc.TokenProximity) } else { output[0] = 1.0 } output[1] = float32(wsf.Timestamp / (SecondsInADay 3)) output[2] = float32(doc.BM25 (1 + float32(wsf.LikeCount)/10000)) return output }
func (o *ApiController) Post() {
content := o.Input().Get("content") typeStr := o.Input().Get("type") ts := utils.MsTimestampNow() id, _ := o.GetInt64("id", 0) count, _ := o.GetInt64("count", 1) sfId, err := SnowFlakeServer.NextID() docId := fmt.Sprintf("%d", sfId) Searcher.Index(docId, types.DocData{ Content: content, Fields: DocItemScoringFields{ Id: id, Type: typeStr, Timestamp: ts, LikeCount: count, }, }) Searcher.Flush() return
}
func (o *ApiController) Get() { keyword := o.Input().Get("keyword") output := Searcher.SearchDoc(types.SearchReq{ Text: keyword, RankOpts: &types.RankOpts{ ScoringCriteria: &WeiboScoringCriteria{}, OutputOffset: 0, MaxOutputs: 100, }, })
docs := make([]*DocItem, 0, 0) for _, doc := range output.Docs { item := DocItem{} field, _ := doc.Fields.(DocItemScoringFields) item.DocId = doc.DocId item.Text = doc.Content item.Timestamp = uint64(field.Timestamp) for _, t := range output.Tokens { item.Text = strings.Replace(item.Text, t, "<font color=red>"+t+"</font>", -1) } docs = append(docs, &item) } return
## Description when ctrl+c to exit the server ,then run again, search the keyword ,but nothing find. any help?
why leveldb opened at rito/engine.go ,but when write data into leveldb then return leveldb status is closed.
Add a retry label to control the error.
func main() { if beego.BConfig.RunMode == "dev" { beego.BConfig.WebConfig.DirectoryIndex = true beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" } initSearcher() beego.BeeApp.Server.RegisterOnShutdown( func() { initClose() }) beego.Run() }
func initSearcher() { // 初始化 var dictFile = filepath.Join(beego.AppPath, "conf/dictionary.txt") var stopTokenFile = filepath.Join(beego.AppPath, "conf/stop_tokens.txt") controllers.Searcher.Init(types.EngineOpts{ Using: 1, GseDict: dictFile, StopTokenFile: stopTokenFile, IndexerOpts: &types.IndexerOpts{ IndexType: types.LocsIndex, }, UseStore: true, StoreOnly: true, StoreFolder: "/tmp/shareyou_search", StoreEngine: "ldb", })
} func initClose() { controllers.Searcher.Flush() controllers.Searcher.Close() }
package controllers import ( "fmt" "github.com/go-ego/riot/types" "lib/utils" "reflect" "strings" )
var ( Searcher = riot.Engine{} )
const ( // SecondsInADay seconds in a day SecondsInADay = 86400 // MaxTokenProximity max token proximity MaxTokenProximity = 2 )
type ApiController struct { BaseController }
type DocItem struct { Id string
json:"id"
Timestamp uint64json:"timestamp"
UserName stringjson:"username"
LikeCount uint64json:"count"
Text stringjson:"text"
DocId stringjson:"docid"
}// WeiboScoringFields weibo scoring fields type DocItemScoringFields struct { Id int64 Type string Timestamp int64 LikeCount int64 }
// WeiboScoringCriteria custom weibo scoring criteria type WeiboScoringCriteria struct { }
// Score score and sort func (criteria WeiboScoringCriteria) Score(doc types.IndexedDoc, fields interface{}) []float32 { if reflect.TypeOf(fields) != reflect.TypeOf(DocItemScoringFields{}) { return []float32{} } wsf := fields.(DocItemScoringFields) output := make([]float32, 3) if doc.TokenProximity > MaxTokenProximity { output[0] = 1.0 / float32(doc.TokenProximity) } else { output[0] = 1.0 } output[1] = float32(wsf.Timestamp / (SecondsInADay 3)) output[2] = float32(doc.BM25 (1 + float32(wsf.LikeCount)/10000)) return output }
func (o *ApiController) Post() {
}
func (o *ApiController) Get() { keyword := o.Input().Get("keyword") output := Searcher.SearchDoc(types.SearchReq{ Text: keyword, RankOpts: &types.RankOpts{ ScoringCriteria: &WeiboScoringCriteria{}, OutputOffset: 0, MaxOutputs: 100, }, })
}