moby / buildkit-bench

Set of benchmarks for BuildKit
https://moby.github.io/buildkit-bench/
Apache License 2.0
0 stars 2 forks source link

Cache not working for parent targets #3

Closed crazy-max closed 2 months ago

crazy-max commented 2 months ago

In the HCL definition we are building multiple buildkit refs using the matrix feature: https://github.com/moby/buildkit-bench/blob/b788ea06409c567b48cfd2aa5f632bd647b87367/docker-bake.hcl#L47-L58

Then we use another target to merge results: https://github.com/moby/buildkit-bench/blob/b788ea06409c567b48cfd2aa5f632bd647b87367/docker-bake.hcl#L60-L66

So we can inject it in the tests image using named contexts: https://github.com/moby/buildkit-bench/blob/b788ea06409c567b48cfd2aa5f632bd647b87367/docker-bake.hcl#L68-L77

When running on ci and using the gha cache exporter, it seems to build all buildkit refs all over without using any cache blobs. This happens for all parent targets such as tests-base and buildkit-binaries: https://github.com/moby/buildkit-bench/actions/runs/10454128293/job/28946051235#step:6:844

But solves the cache for buildkit-build-* targets like https://github.com/moby/buildkit-bench/actions/runs/10454128293/job/28946051235#step:6:1228

Note that I see some [tests-base] importing cache manifest from gha: but does not look to make anything useful: https://github.com/moby/buildkit-bench/actions/runs/10454128293/job/28946051235#step:6:743

Here is the canonical representation of the build: https://github.com/moby/buildkit-bench/actions/runs/10454128293/job/28946051235#step:6:325

  {
    "group": {
      "default": {
        "targets": [
          "tests-base"
        ]
      }
    },
    "target": {
      "buildkit-binaries": {
        "attest": [
          "type=provenance,disabled=true"
        ],
        "context": ".",
        "contexts": {
          "buildkit-build-master": "target:buildkit-build-master",
          "buildkit-build-v0-14-1": "target:buildkit-build-v0-14-1",
          "buildkit-build-v0-15-2": "target:buildkit-build-v0-15-2"
        },
        "dockerfile": "Dockerfile",
        "dockerfile-inline": "FROM scratch\nCOPY --link --from=buildkit-build-master / /master\nCOPY --link --from=buildkit-build-v0-15-2 / /v0.15.2\nCOPY --link --from=buildkit-build-v0-14-1 / /v0.14.1\n",
        "cache-from": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb",
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e",
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "output": [
          "type=cacheonly"
        ]
      },
      "buildkit-build-master": {
        "attest": [
          "type=provenance,disabled=true"
        ],
        "context": "https://github.com/moby/buildkit.git#master",
        "dockerfile": "Dockerfile",
        "args": {
          "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
          "BUILDKIT_DEBUG": "1"
        },
        "cache-from": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb",
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e",
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "cache-to": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb"
        ],
        "target": "binaries",
        "output": [
          "type=cacheonly"
        ]
      },
      "buildkit-build-v0-14-1": {
        "attest": [
          "type=provenance,disabled=true"
        ],
        "context": "https://github.com/moby/buildkit.git#v0.14.1",
        "dockerfile": "Dockerfile",
        "args": {
          "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
          "BUILDKIT_DEBUG": "1"
        },
        "cache-from": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb",
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e",
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "cache-to": [
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "target": "binaries",
        "output": [
          "type=cacheonly"
        ]
      },
      "buildkit-build-v0-15-2": {
        "attest": [
          "type=provenance,disabled=true"
        ],
        "context": "https://github.com/moby/buildkit.git#v0.15.2",
        "dockerfile": "Dockerfile",
        "args": {
          "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
          "BUILDKIT_DEBUG": "1"
        },
        "cache-from": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb",
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e",
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "cache-to": [
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e"
        ],
        "target": "binaries",
        "output": [
          "type=cacheonly"
        ]
      },
      "tests-base": {
        "attest": [
          "type=provenance,disabled=true"
        ],
        "context": ".",
        "contexts": {
          "buildkit-binaries": "target:buildkit-binaries"
        },
        "dockerfile": "Dockerfile",
        "args": {
          "BUILDKIT_REFS": "master,v0.15.2,v0.14.1"
        },
        "cache-from": [
          "type=gha,scope=buildkit-49f3d8f8294d40a3c37a1e2ea3053be7c88f47cb",
          "type=gha,scope=buildkit-9e14164a1099d3e41b58fc879cbdd6f2b2edb04e",
          "type=gha,scope=buildkit-eb864a84592468ee9b434326cb7efd66f58555af"
        ],
        "target": "tests-base",
        "output": [
          "type=cacheonly"
        ]
      }
    }
  }
crazy-max commented 2 months ago

As discussed we should use the registry cache exporter instead.

crazy-max commented 2 months ago

With build for each ref split in a dedicated job in https://github.com/moby/buildkit-bench/pull/17, this is not an issue anymore.