Q: 误报情况？

[xiaoyuer11223344]

你好，我用如下的代码，对相同的10个url进行反复测试，发现返回的结果不相同，请问是逻辑出现问题了还是我自己代码调度方式有问题？

package main

import (
    "bufio"
    "errors"
    "fmt"
    "github.com/chainreactors/neutron/protocols"
    "github.com/chainreactors/neutron/templates"
    "gopkg.in/yaml.v3"
    "io/ioutil"
    "os"
)

// FileExists checks if the file exists in the provided path
func FileExists(filename string) bool {
    info, err := os.Stat(filename)
    if os.IsNotExist(err) {
        return false
    }
    if err != nil {
        return false
    }
    return !info.IsDir()
}

// ReadFile with filename
func ReadFile(filename string) (chan string, error) {
    if !FileExists(filename) {
        return nil, errors.New("file doesn't exist")
    }
    out := make(chan string)
    go func() {
        defer close(out)
        f, err := os.Open(filename)
        if err != nil {
            return
        }
        defer f.Close()
        scanner := bufio.NewScanner(f)
        for scanner.Scan() {
            out <- scanner.Text()
        }
    }()

    return out, nil
}

var ExecuterOptions *protocols.ExecuterOptions

func NewExecuterOptions() *protocols.ExecuterOptions {
    var options = &protocols.ExecuterOptions{
        Options: &protocols.Options{
            Opsec:       false,
            VarsPayload: map[string]interface{}{},
        },
    }
    return options
}

func LoadTemplate() (*templates.Template, error) {
    var err error

    template := &templates.Template{}

    var bytes []byte
    bytes, err = ioutil.ReadFile("./poc/harbor-public-images.yml")
    if err != nil {
        return nil, err
    }

    err = yaml.Unmarshal(bytes, template)
    if err != nil {
        return nil, err
    }

    // compile template
    err = template.Compile(ExecuterOptions)
    if err != nil {
        return nil, err
    }

    return template, nil
}

func CustomConcurrent(target string, template *templates.Template) (bool, error) {
    var status bool

    res, err := template.Execute(target, nil)
    if err == nil && res != nil {
        status = true
    }

    return status, err
}

func main() {
    ExecuterOptions = NewExecuterOptions()

    template, err := LoadTemplate()
    if err != nil {
        return
    }

    fc, err := ReadFile("url.txt")
    if err != nil {
        return
    }

    // init
    var webList = make(map[string][]string, len(fc))
    for url := range fc {
        webList[url] = []string{"harbor"}
    }

    for url := range webList {
        status, _err := CustomConcurrent(url, template)
        if _err != nil {
            fmt.Printf("%s is not ok, error is %v\n", url, _err)
            return
        }

        if status {
            fmt.Printf("%s is ok\n", url)
        } else {
            fmt.Printf("%s is not ok\n", url)
        }
    }
}

harbor-public-images.yml

id: harbor-public-images

info:
  name: Harbor-unauth
  author: qqq
  severity: high
  tags: harbor

http:
  - method: GET
    path:
      - "{{BaseURL}}/api/search?q="

    matchers-condition: and
    matchers:
      - type: status
        status:
          - 200

      - type: word
        words:
          - 'project_id'
        condition: or

# /api/v2.0/projects/{project_name}/repositories/{repositorie_name}/artifacts?with_tag=true&with_scan_overview=true&with_label=true&page_size=15&page=1

url.txt

http://ycs-newcenter.ssl.ysten.com
http://notary-registry.kanongyun.com
http://39.107.35.12:7444
http://39.107.141.11
https://registry-1.pve1.die-lobby.dev
https://artifacth.sail-cloud.com
http://chbm.hafnkj.com:30002
https://reg.deltawaters.online
https://docker.yc345.tv

[xiaoyuer11223344]

module NeutronDemo

go 1.21.10

require ( github.com/chainreactors/neutron v0.0.0-20240628163951-679753df51f6 gopkg.in/yaml.v3 v3.0.1 )

require ( github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/chainreactors/files v0.0.0-20230731174853-acee21c8c45a // indirect github.com/chainreactors/logs v0.0.0-20231027080134-7a11bb413460 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect )

[XiaoliChan]

这个并不是误报，而是你没理解逻辑，res != nil 意味着可能进入到作者定义的chain阶段，类似原版的workflow

真正判断，返回的struct里面有个matched，只需要判断上即可

res.Matched

[xiaoyuer11223344]

师傅 非常感谢你能够解答，但是我尝试了你说的根据matched字段来进行判断，但是还是出现上面的情况，要么就是正确 要么都是错误

func CustomConcurrent(target string, template *templates.Template) (bool, error) {
    var status bool

    res, err := template.Execute(target, nil)
    if err == nil && res != nil && res.Matched {
        status = true
        return status, nil
    } else {
        return status, err
    }
}

实际情况中只有https://218.253.255.187/api/search?q= 是有效的

[XiaoliChan]

你的代码我没仔细看，自己调优一下吧（可以确定的是我已经运用到实战中）

issue can be close

[xiaoyuer11223344]

模版对象templates.Template不能复用，每次对一个目标需要重新创建

[xiaoyuer11223344]

实际情况： 37.139.42.217不存在harbor未授权,218.253.255.187:80存在harbor未授权

重复十次，结果均是要么全有，要么全无

cat url.txt

37.139.42.217
218.253.255.187

./gogo -l url.txt -p 80,443 -E harbor

[*] gogo: , 2024-07-10 01:27.28
[*] Current goroutines: 1000, Version Level: 0,Exploit: harbor, PortSpray: false , 2024-07-10 01:27.28
[*] Start task url.txt ,total ports: 2 , mod: default , 2024-07-10 01:27.28
[*] ports: 80,443 , 2024-07-10 01:27.28
[*] Default Scan is expected to take 4 seconds , 2024-07-10 01:27.28
[+] https://218.253.255.187:443 nginx   harbor||nginx   icon-info.com.hk,www.icon-info.com.hk,mail2.icon-info.com.hk,iconvdi.iconsystems.com.hk,security.icon-info.com.hk,mail1.icon-info.com.hk,access.iconsystems.com.hk,vpn.iconsystems.com.hk,mail.icon-info.com.hk,autodiscover.icon-info.com.hk,autodiscover.ibslaudit.com,e-leave.iconsystems.com.hk,hcpaw.iconsystems.com.hk,spp.iconsystems.com.hk,harbor.iconsystems.com.hk [200] Harbor [ high: harbor-public-images ]
[+] http://218.253.255.187:80   nginx/1.16.1    nginx    [308] 308 Permanent Redirect [ high: harbor-public-images ]
[+] http://37.139.42.217:80 nginx/1.14.1    nginx    [200] Test Page for the Nginx HTTP Server on AlmaLinux [ high: harbor-public-images ]
[+] https://37.139.42.217:443   nginx/1.14.1    harbor||nginx   harbor.logijet.ru,harbor.lorus-scm.com [200] Harbor [ high: harbor-public-images ]
[*] Alived: 4, Total: 4 , 2024-07-10 01:27.33
[*] Time consuming: 4.866362s , 2024-07-10 01:27.33

[*] gogo: , 2024-07-10 01:28.49
[*] Current goroutines: 1000, Version Level: 0,Exploit: harbor, PortSpray: false , 2024-07-10 01:28.49
[*] Start task url.txt ,total ports: 2 , mod: default , 2024-07-10 01:28.49
[*] ports: 80,443 , 2024-07-10 01:28.49
[*] Default Scan is expected to take 4 seconds , 2024-07-10 01:28.49
[+] http://218.253.255.187:80   nginx/1.16.1    nginx    [308] 308 Permanent Redirect
[+] https://218.253.255.187:443 nginx   harbor||nginx   icon-info.com.hk,www.icon-info.com.hk,mail2.icon-info.com.hk,iconvdi.iconsystems.com.hk,security.icon-info.com.hk,mail1.icon-info.com.hk,access.iconsystems.com.hk,vpn.iconsystems.com.hk,mail.icon-info.com.hk,autodiscover.icon-info.com.hk,autodiscover.ibslaudit.com,e-leave.iconsystems.com.hk,hcpaw.iconsystems.com.hk,spp.iconsystems.com.hk,harbor.iconsystems.com.hk [200] Harbor
[+] http://37.139.42.217:80 nginx/1.14.1    nginx    [200] Test Page for the Nginx HTTP Server on AlmaLinux
[+] https://37.139.42.217:443   nginx/1.14.1    harbor||nginx   harbor.logijet.ru,harbor.lorus-scm.com [200] Harbor
[*] Alived: 4, Total: 4 , 2024-07-10 01:28.55
[*] Time consuming: 5.275849916s , 2024-07-10 01:28.55

重复十次，结果均是如下

cat url.txt

37.139.42.217

./gogo -l url.txt -p 80,443 -E harbor

[*] gogo: , 2024-07-10 01:25.47
[*] Current goroutines: 1000, Version Level: 0,Exploit: harbor, PortSpray: false , 2024-07-10 01:25.47
[*] Start task url.txt ,total ports: 2 , mod: default , 2024-07-10 01:25.47
[*] ports: 80,443 , 2024-07-10 01:25.47
[*] Default Scan is expected to take 4 seconds , 2024-07-10 01:25.47
[+] http://37.139.42.217:80 nginx/1.14.1    nginx    [200] Test Page for the Nginx HTTP Server on AlmaLinux
[+] https://37.139.42.217:443   nginx/1.14.1    harbor||nginx   harbor.logijet.ru,harbor.lorus-scm.com [200] Harbor
[*] Alived: 2, Total: 2 , 2024-07-10 01:25.55
[*] Time consuming: 7.661643666s , 2024-07-10 01:25.55

[xiaoyuer11223344]

是我不会用 还是有bug？

[xiaoyuer11223344]

你的代码我没仔细看，自己调优一下吧（可以确定的是我已经运用到实战中）

issue can be close

师傅已经运用到实战中 没出现这个情况？

[XiaoliChan]

@xiaoyuer11223344 好像确实存在bug，第一个目标命中后，后面的目标都会命中

晚点我测测旧版的neutron

[xiaoyuer11223344]

自己项目扫描的时候neutron有这种问题的情况，发现是模版对象templates.Template不能复用，解决的办法是每次对一个目标需要重新创建对应的poc。

[XiaoliChan]

自己项目扫描的时候neutron有这种问题的情况，发现是模版对象templates.Template不能复用，解决的办法是每次对一个目标需要重新创建对应的poc。

这样的话，大量poc会占用大量内存吧

[M09Ic]

我确认下问题, 似乎并没有这么好复现. 我测试了几十次, 都出现了正确的结果. 我怀疑是template没正确初始化?

[M09Ic]

应该是在2月份对neutron同步nuclei大量新特性的时候引入的。

bug出现在 protocols/http/request_generator.go:235

    for i, value := range r.request.Path {
        r.request.Path[i], err = common.Evaluate(value, values)
        if err != nil {
            return nil, common.EvalError
        }
    }

覆盖了executor中的request的各种参数, 导致了后续结果全被第一个命中的结果覆盖

[M09Ic]

尝试在 https://github.com/chainreactors/neutron/commit/2972f33344a3f7b8a5e86fc84b5c515a4a73fb32 中修复.

本地测试已经不存在之前的问题, 调用同一个template能返回正确的结果

func TestTemplate_Execute(t1 *testing.T) {
    common.NeutronLog.SetLevel(logs.Debug)
    content, _ := os.ReadFile("tmp.yaml")
    t := &Template{}
    err := yaml.Unmarshal(content, t)
    if err != nil {
        println(err.Error())
        return
    }
    if t != nil {
        err := t.Compile(ExecuterOptions)
        if err != nil {
            println(err.Error())
            return
        }
    }

    println("load success")

    res, err := t.Execute("https://218.253.255.187", nil)
    if err == nil {
        fmt.Println("no err", res)
        if res.Matched {
            fmt.Printf("matched, %s\n", t.Id)
        }
    } else {

        fmt.Println(res)
    }
    res, err = t.Execute("https://37.139.42.217", nil)
    if err == nil {
        fmt.Println("no err", res)
        if res != nil && res.Matched {
            fmt.Printf("matched, %s\n", t.Id)
        } else {
            fmt.Println("no result")
        }
    } else {
        fmt.Println(res)
    }
}

@xiaoyuer11223344 @XiaoliChan 是否可以帮忙在之前的环境中测试一下. 测试通过后我会尽快合并到gogo

[XiaoliChan]

@M09Ic Confirm the bug has been fixed on my side.

[XiaoliChan]

More test result

[xiaoyuer11223344]

目前单模块批量检测无问题，结果准确

[M09Ic]

这个改动对整体的poc解析有一定影响, 我也在测试一些复杂poc的结果. 如果可以, 也可以帮我测试一些复杂poc是否正常运作

[M09Ic]

果然上个改动中影响了randstr的解析。尝试修复, 并新增了对annotation的解析。

https://github.com/chainreactors/neutron/commit/c1aba15b5e1ee89baaf7c32635939ecf4d99bcfd