本项目是阿里云日志服务 (Log Service,简称SLS)API的Golang编程接口,提供了对于Log Service Rest API的封装和实现,帮助Golang开发人员更快编程使用阿里云日志服务。
本项目主要由3个部分组成:
详细API接口以及含义请参考:https://help.aliyun.com/document_detail/29007.html
go get -u github.com/aliyun/aliyun-log-go-sdk
前言: 所有的使用样例都位于example目录下,使用该目录下的所有样例前,请先在该目录下的config.go文件中的init函数中配置您的 project, logstore等所需要的配置参数,example目录下的所有样例都会使用config.go文件中配置的参数。
创建Client
参考 config.go 文件
AccessKeyID = "your ak id"
AccessKeySecret = "your ak secret"
credentialsProvider := NewStaticCredentialsProvider(AccessKeyID, AccessKeySecret, "")
Endpoint = "your endpoint" // just like cn-hangzhou.log.aliyuncs.com
Client = sls.CreateNormalInterfaceV2(Endpoint, credentialsProvider)
为了防止出现配置错误,您可以在创建 Client 之后,测试 Client 是否能成功调用 SLS API
_, err := Client.ListProject()
if err != nil {
panic(err)
}
创建project
参考 log_project.go文件
project, err := util.Client.CreateProject(ProjectName,"Project used for testing")
if err != nil {
fmt.Println(err)
}
fmt.Println(project)
创建logstore
err := util.Client.CreateLogStore(ProjectName, LogStoreName,2,2,true,64)
if err != nil {
fmt.Println(err)
}
创建索引
indexKeys := map[string]sls.IndexKey{
"col_0": {
Token: []string{" "},
CaseSensitive: false,
Type: "long",
},
"col_1": {
Token: []string{",", ":", " "},
CaseSensitive: false,
Type: "text",
},
}
index := sls.Index{
Keys: indexKeys,
Line: &sls.IndexLine{
Token: []string{",", ":", " "},
CaseSensitive: false,
IncludeKeys: []string{},
ExcludeKeys: []string{},
},
}
err = util.Client.CreateIndex(util.ProjectName, logstore_name, index)
if err != nil {
fmt.Printf("CreateIndex fail, err: %s", err)
return
}
fmt.Println("CreateIndex success")
写数据
这里展示了用sdk中原生的API接口去发送数据简单示例,但是我们不推荐用API直接向logstore写入数据,推荐使用SDK 中提供的producer 包向logstore 写入数据,自动压缩数据并且提供安全退出机制,不会使数据丢失, 对于MetricStore类型, 使用PutLogsWithMetricStoreURL
API 发送数据可以提升大基数时间线下查询性能 。
logs := []*sls.Log{}
for logIdx := 0; logIdx < 100; logIdx++ {
content := []*sls.LogContent{}
for colIdx := 0; colIdx < 10; colIdx++ {
content = append(content, &sls.LogContent{
Key: proto.String(fmt.Sprintf("col_%d", colIdx)),
Value: proto.String(fmt.Sprintf("loggroup idx: %d, log idx: %d, col idx: %d, value: %d", loggroupIdx, logIdx, colIdx, rand.Intn(10000000))),
})
}
log := &sls.Log{
Time: proto.Uint32(uint32(time.Now().Unix())),
Contents: content,
}
logs = append(logs, log)
}
loggroup := &sls.LogGroup{
Topic: proto.String(""),
Source: proto.String("10.230.201.117"),
Logs: logs,
}
// PostLogStoreLogs API Ref: https://intl.aliyun.com/help/doc-detail/29026.htm
for retryTimes := 0; retryTimes < 10; retryTimes++ {
err := util.Client.PutLogs(util.ProjectName, util.LogStoreName, loggroup)
if err == nil {
fmt.Printf("PutLogs success, retry: %d\n", retryTimes)
break
} else {
fmt.Printf("PutLogs fail, retry: %d, err: %s\n", retryTimes, err)
//handle exception here, you can add retryable erorrCode, set appropriate put_retry
if strings.Contains(err.Error(), sls.WRITE_QUOTA_EXCEED) || strings.Contains(err.Error(), sls.PROJECT_QUOTA_EXCEED) || strings.Contains(err.Error(), sls.SHARD_WRITE_QUOTA_EXCEED) {
//mayby you should split shard
time.Sleep(1000 * time.Millisecond)
} else if strings.Contains(err.Error(), sls.INTERNAL_SERVER_ERROR) || strings.Contains(err.Error(), sls.SERVER_BUSY) {
time.Sleep(200 * time.Millisecond)
}
}
}
6.读数据
这里展示了使用SDK中原生API接口调用去拉取数据的方式,我们不推荐使用这种方式去读取消费logstore中的数据,推荐使用SDK中 consumer 消费组去拉取数据,消费组提供自动负载均衡以及失败重试等机制,并且会自动保存拉取断点,再次拉取不会拉取重复数据。
shards, err := client.ListShards(project, logstore)
totalLogCount := 0
for _, shard := range shards {
fmt.Printf("[shard] %d begin\n", shard.ShardID)
beginCursor, err := client.GetCursor(project, logstore, shard.ShardID, "begin")
if err != nil {
panic(err)
}
endCursor, err := client.GetCursor(project, logstore, shard.ShardID, "end")
if err != nil {
panic(err)
}
nextCursor := beginCursor
for nextCursor != endCursor {
gl, nc, err := client.PullLogs(project, logstore, shard.ShardID, nextCursor, endCursor, 10)
if err != nil {
fmt.Printf("pull log error : %s\n", err)
time.Sleep(time.Second)
continue
}
nextCursor = nc
fmt.Printf("now count %d \n", totalLogCount)
if gl != nil {
for _, lg := range gl.LogGroups {
for _, tag := range lg.LogTags {
fmt.Printf("[tag] %s : %s\n", tag.GetKey(), tag.GetValue())
}
for _, log := range lg.Logs {
totalLogCount++
// print log
for _, content := range log.Contents {
fmt.Printf("[log] %s : %s\n", content.GetKey(), content.GetValue())
}
}
}
}
}
创建机器组
机器组分为IP型机器组和标识型机器组,二选一
创建标识型机器组
attribute := sls.MachinGroupAttribute{
}
machineList := []string{"mac-user-defined-id-value_XX"}
var machineGroup = &sls.MachineGroup{
Name: groupName,
MachineIDType: "userdefined",
MachineIDList: machineList,
Attribute: attribute,
}
err = util.Client.CreateMachineGroup(ProjectName, machineGroup)
if err != nil {
fmt.Println(err)
}
创建IP型机器组
attribute := sls.MachinGroupAttribute{
}
machineList := []string{"192.168.XX.XX"}
var machineGroup = &sls.MachineGroup{
Name: groupName,
MachineIDType: "ip",
MachineIDList: machineList,
Attribute: attribute,
}
err = util.Client.CreateMachineGroup(ProjectName, machineGroup)
if err != nil {
fmt.Println(err)
}
创建logtail 采集配置
logtail 采集配置,目前通过sdk 支持创建下列几种模式的采集配置,分别为 完整正则,分隔符模式,json模式 ,插件模式,这里展示的完整正则模式的创建。
regexConfig := new(sls.RegexConfigInputDetail)
regexConfig.DiscardUnmatch = false
regexConfig.Key = []string{"logger", "time", "cluster", "hostname", "sr", "app", "workdir", "exe", "corepath", "signature", "backtrace"}
regexConfig.Regex = "\\S*\\s+(\\S*)\\s+(\\S*\\s+\\S*)\\s+\\S*\\s+(\\S*)\\s+(\\S*)\\s+(\\S*)\\s+(\\S*)\\s+(\\S*)\\s+(\\S*)\\s+(\\S*)\\s+\\S*\\s+(\\S*)\\s*([^$]+)"
regexConfig.TimeFormat = "%Y/%m/%d %H:%M:%S"
regexConfig.LogBeginRegex = `INFO core_dump_info_data .*`
regexConfig.LogPath = "/cloud/log/tianji/TianjiClient#/core_dump_manager"
regexConfig.FilePattern = "core_dump_info_data.log*"
regexConfig.MaxDepth = 0
sls.InitRegexConfigInputDetail(regexConfig)
outputDetail := sls.OutputDetail{
ProjectName: projectName,
LogStoreName: logstore,
}
logConfig := &sls.LogConfig{
Name: configName,
InputType: "file",
OutputType: "LogService", // Now only supports LogService
InputDetail: regexConfig,
OutputDetail: outputDetail,
}
err = util.Client.CreateConfig(projectName, logConfig)
if err != nil {
fmt.Println(err)
}
logtail配置到机器组
err = util.Client.ApplyConfigToMachineGroup(ProjectName, confName, mgname)
if err != nil {
fmt.Println(err)
}
protoc -I=. -I=$GOPATH/src -I=$GOPATH/src/github.com/gogo/protobuf/protobuf --gofast_out=. log.proto