search

gocsaf / csaf

Tools to download or provide CSAF (Common Security Advisory Framework) documents.
https://csaf.io
40 stars 23 forks source link

Race condition on csaf_downloader #546

Closed koplas closed 5 months ago

koplas commented 5 months ago

Each worker shares the same util.PathEval, which is not thread-safe. To fix this issue, each worker should have their own util.PathEval or util.PathEval should be made thread-safe. Here is an example of how util.PathEval can be made thread-safe using a lock:

diff --git a/util/json.go b/util/json.go
index f66ab86..c8ffbca 100644
--- a/util/json.go
+++ b/util/json.go
@@ -13,6 +13,7 @@ import (
        "encoding/json"
        "errors"
        "fmt"
+       "sync"
        "time"

        "github.com/Intevation/gval"
@@ -32,6 +33,7 @@ func ReMarshalJSON(dst, src any) error {
 type PathEval struct {
        builder gval.Language
        exprs   map[string]gval.Evaluable
+       exprsMU sync.RWMutex
 }

 // NewPathEval creates a new PathEval.
@@ -45,14 +47,22 @@ func NewPathEval() *PathEval {
 // Compile compiles an expression and stores it in the
 // internal cache on success.
 func (pe *PathEval) Compile(expr string) (gval.Evaluable, error) {
-       if eval := pe.exprs[expr]; eval != nil {
+       pe.exprsMU.RLock()
+       eval := pe.exprs[expr]
+       pe.exprsMU.RUnlock()
+
+       if eval != nil {
                return eval, nil
        }
        eval, err := pe.builder.NewEvaluable(expr)
        if err != nil {
                return nil, err
        }
+
+       pe.exprsMU.Lock()
        pe.exprs[expr] = eval
+       pe.exprsMU.Unlock()
+
        return eval, nil
 }

@@ -62,13 +72,17 @@ func (pe *PathEval) Eval(expr string, doc any) (any, error) {
        if doc == nil {
                return nil, errors.New("no document to extract data from")
        }
+       pe.exprsMU.RLock()
        eval := pe.exprs[expr]
+       pe.exprsMU.RUnlock()
        if eval == nil {
                var err error
                if eval, err = pe.builder.NewEvaluable(expr); err != nil {
                        return nil, err
                }
+               pe.exprsMU.Lock()
                pe.exprs[expr] = eval
+               pe.exprsMU.Unlock()
        }
        return eval(context.Background(), doc)
 }

To reproduce, run the following command: go run -race ./cmd/csaf_downloader -d ~/Documents/csaf_json_files https://cert-portal.siemens.com/productcert/csaf/provider-metadata.json

Which results in this output:

==================
WARNING: DATA RACE
Read at 0x00c0003c4270 by goroutine 44:
  runtime.mapaccess1_faststr()
      /usr/lib/go/src/runtime/map_faststr.go:13 +0x0
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:65 +0xae
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  github.com/csaf-poc/csaf_distribution/v3/util.IDMatchesFilename()
      /home/paul/Documents/delete_me_csaf/util/file.go:47 +0xc6
  main.(*downloader).downloadWorker.func6()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:591 +0xc4
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:627 +0x237a
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Previous write at 0x00c0003c4270 by goroutine 43:
  runtime.mapassign_faststr()
      /usr/lib/go/src/runtime/map_faststr.go:203 +0x0
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:71 +0x204
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:654 +0x2eeb
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Goroutine 44 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247

Goroutine 43 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247
==================
==================
WARNING: DATA RACE
Read at 0x00c000331c38 by goroutine 44:
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:65 +0xbb
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  github.com/csaf-poc/csaf_distribution/v3/util.IDMatchesFilename()
      /home/paul/Documents/delete_me_csaf/util/file.go:47 +0xc6
  main.(*downloader).downloadWorker.func6()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:591 +0xc4
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:627 +0x237a
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Previous write at 0x00c000331c38 by goroutine 43:
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:71 +0x213
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  github.com/csaf-poc/csaf_distribution/v3/util.IDMatchesFilename()
      /home/paul/Documents/delete_me_csaf/util/file.go:47 +0xc6
  main.(*downloader).downloadWorker.func6()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:591 +0xc4
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:627 +0x237a
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Goroutine 44 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247

Goroutine 43 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247
==================
==================
WARNING: DATA RACE
Read at 0x00c0001ab790 by goroutine 44:
  github.com/Intevation/jsonpath.plainPath.evaluatePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:22 +0xa8
  github.com/Intevation/jsonpath.plainPath.evaluate()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:17 +0x97
  github.com/Intevation/jsonpath.(*plainPath).evaluate()
      <autogenerated>:1 +0x1f
  github.com/Intevation/jsonpath.path.evaluate-fm()
      <autogenerated>:1 +0x7b
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:73 +0x273
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  github.com/csaf-poc/csaf_distribution/v3/util.IDMatchesFilename()
      /home/paul/Documents/delete_me_csaf/util/file.go:47 +0xc6
  main.(*downloader).downloadWorker.func6()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:591 +0xc4
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:627 +0x237a
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Previous write at 0x00c0001ab790 by goroutine 43:
  github.com/Intevation/jsonpath.plainPath.withPlainSelector()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:51 +0xaa
  github.com/Intevation/jsonpath.(*plainPath).withPlainSelector()
      <autogenerated>:1 +0x1f
  github.com/Intevation/jsonpath.(*parser).appendPlainSelector()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:199 +0x413
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:85 +0x307
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:86 +0x476
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:86 +0x476
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parse()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:33 +0xa4
  github.com/Intevation/jsonpath.parseRootPath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:19 +0xac
  github.com/Intevation/gval.(*Parser).ParseNextExpression()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:44 +0x1fa
  github.com/Intevation/gval.(*Parser).ParseExpression()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:17 +0x65
  github.com/Intevation/gval.(*Parser).parse()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:76 +0xad
  github.com/Intevation/gval.Language.NewEvaluableWithContext()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/language.go:81 +0x11b
  github.com/Intevation/gval.Language.NewEvaluable()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/language.go:74 +0x1bc
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:68 +0xd0
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  github.com/csaf-poc/csaf_distribution/v3/util.IDMatchesFilename()
      /home/paul/Documents/delete_me_csaf/util/file.go:47 +0xc6
  main.(*downloader).downloadWorker.func6()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:591 +0xc4
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:627 +0x237a
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Goroutine 44 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247

Goroutine 43 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247
==================
==================
WARNING: DATA RACE
Read at 0x00c000331c40 by goroutine 44:
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:65 +0xbb
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:654 +0x2eeb
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Previous write at 0x00c000331c40 by goroutine 43:
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:71 +0x213
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:654 +0x2eeb
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Goroutine 44 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247

Goroutine 43 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247
==================
==================
WARNING: DATA RACE
Read at 0x00c0001ab7d0 by goroutine 44:
  github.com/Intevation/jsonpath.plainPath.evaluatePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:22 +0xa8
  github.com/Intevation/jsonpath.plainPath.evaluate()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:17 +0x97
  github.com/Intevation/jsonpath.(*plainPath).evaluate()
      <autogenerated>:1 +0x1f
  github.com/Intevation/jsonpath.path.evaluate-fm()
      <autogenerated>:1 +0x7b
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:73 +0x273
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:654 +0x2eeb
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Previous write at 0x00c0001ab7d0 by goroutine 43:
  github.com/Intevation/jsonpath.plainPath.withPlainSelector()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/path.go:51 +0xaa
  github.com/Intevation/jsonpath.(*plainPath).withPlainSelector()
      <autogenerated>:1 +0x1f
  github.com/Intevation/jsonpath.(*parser).appendPlainSelector()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:199 +0x413
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:85 +0x307
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:86 +0x476
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parseSelect()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:86 +0x476
  github.com/Intevation/jsonpath.(*parser).parsePath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:44 +0x9e4
  github.com/Intevation/jsonpath.(*parser).parse()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:33 +0xa4
  github.com/Intevation/jsonpath.parseRootPath()
      /home/paul/go/pkg/mod/github.com/!intevation/jsonpath@v0.2.1/parse.go:19 +0xac
  github.com/Intevation/gval.(*Parser).ParseNextExpression()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:44 +0x1fa
  github.com/Intevation/gval.(*Parser).ParseExpression()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:17 +0x65
  github.com/Intevation/gval.(*Parser).parse()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/parse.go:76 +0xad
  github.com/Intevation/gval.Language.NewEvaluableWithContext()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/language.go:81 +0x11b
  github.com/Intevation/gval.Language.NewEvaluable()
      /home/paul/go/pkg/mod/github.com/!intevation/gval@v1.3.0/language.go:74 +0x1bc
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Eval()
      /home/paul/Documents/delete_me_csaf/util/json.go:68 +0xd0
  github.com/csaf-poc/csaf_distribution/v3/util.(*PathEval).Extract()
      /home/paul/Documents/delete_me_csaf/util/json.go:178 +0xf6
  main.(*downloader).downloadWorker()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:654 +0x2eeb
  main.(*downloader).downloadFiles.gowrap1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0xaa

Goroutine 44 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247

Goroutine 43 (running) created at:
  main.(*downloader).downloadFiles()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:276 +0x28a
  main.(*downloader).download.func1()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:243 +0x94
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).processROLIE()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:378 +0xda3
  github.com/csaf-poc/csaf_distribution/v3/csaf.(*AdvisoryFileProcessor).Process()
      /home/paul/Documents/delete_me_csaf/csaf/advisories.go:154 +0xa4c
  main.(*downloader).download()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:242 +0x784
  main.(*downloader).run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/downloader.go:768 +0x179
  main.run()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:48 +0x539
  main.main()
      /home/paul/Documents/delete_me_csaf/cmd/csaf_downloader/main.go:62 +0x247
==================
s-l-teichmann commented 5 months ago

PR #547 addresses this. Instead of using a mutex around the global one its easier to use a PathEval per worker. The Aggregator already does this, too. With the PR the downloader runs fine under the race detector.