From a3e7a0690d6dde3140ee016d4df40f7eac084c00 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 25 Aug 2022 21:20:05 -0700 Subject: [PATCH 1/9] vendor: update in-toto-golang Signed-off-by: Tonis Tiigi --- go.mod | 10 +- go.sum | 17 +- .../cpuguy83/go-md2man/v2/md2man/roff.go | 105 +- .../in-toto/in-toto-golang/in_toto/keylib.go | 4 +- .../in-toto/in-toto-golang/in_toto/model.go | 103 +- .../slsa_provenance/v0.2/provenance.go | 66 + .../in-toto/in-toto-golang/pkg/ssl/verify.go | 71 - .../russross/blackfriday/v2/README.md | 90 +- .../russross/blackfriday/v2/block.go | 30 +- .../github.com/russross/blackfriday/v2/doc.go | 28 + .../russross/blackfriday/v2/entities.go | 2236 +++++++++++++++++ .../github.com/russross/blackfriday/v2/esc.go | 42 +- .../russross/blackfriday/v2/html.go | 9 +- .../russross/blackfriday/v2/inline.go | 2 +- .../russross/blackfriday/v2/node.go | 12 +- .../go-securesystemslib}/LICENSE | 4 +- .../cjson}/canonicaljson.go | 2 +- .../go-securesystemslib/dsse}/sign.go | 61 +- .../go-securesystemslib/dsse/verify.go | 146 ++ .../github.com/shibumi/go-pathspec/.gitignore | 3 + .../sanitized_anchor_name/.travis.yml | 16 - .../shurcooL/sanitized_anchor_name/README.md | 36 - .../shurcooL/sanitized_anchor_name/main.go | 29 - vendor/modules.txt | 21 +- 24 files changed, 2783 insertions(+), 360 deletions(-) create mode 100644 vendor/github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2/provenance.go delete mode 100644 vendor/github.com/in-toto/in-toto-golang/pkg/ssl/verify.go create mode 100644 vendor/github.com/russross/blackfriday/v2/entities.go rename vendor/github.com/{shurcooL/sanitized_anchor_name => secure-systems-lab/go-securesystemslib}/LICENSE (94%) rename vendor/github.com/{in-toto/in-toto-golang/in_toto => secure-systems-lab/go-securesystemslib/cjson}/canonicaljson.go (99%) rename vendor/github.com/{in-toto/in-toto-golang/pkg/ssl => secure-systems-lab/go-securesystemslib/dsse}/sign.go (70%) create mode 100644 vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/verify.go delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/.travis.yml delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/README.md delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/main.go diff --git a/go.mod b/go.mod index 68c9ead24ae6..b2483f9cb780 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru v0.5.4 - github.com/in-toto/in-toto-golang v0.3.3 + github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add github.com/klauspost/compress v1.15.12 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/moby/locker v1.0.1 @@ -111,7 +111,7 @@ require ( github.com/containerd/fifo v1.0.0 // indirect github.com/containerd/ttrpc v1.1.0 // indirect github.com/containernetworking/cni v1.1.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect @@ -140,9 +140,9 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/russross/blackfriday/v2 v2.0.1 // indirect - github.com/shibumi/go-pathspec v1.2.0 // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect + github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/vbatts/tar-split v0.11.2 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.4.1 // indirect diff --git a/go.sum b/go.sum index 2b89ee34a0b3..7d9488728fcb 100644 --- a/go.sum +++ b/go.sum @@ -475,8 +475,9 @@ github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -891,8 +892,8 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/in-toto/in-toto-golang v0.3.3 h1:tkkEBU5i09UEeWKnrp6Rq4fXKAfpVXYMLRO5mDfnb3I= -github.com/in-toto/in-toto-golang v0.3.3/go.mod h1:dbXecHGZSqRubmm5TXtvDSZT5JyaKD7ebVTiC2aMLWY= +github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add h1:DAh7mHiRT7wc6kKepYdCpH16ElPciMPQWJaJ7H3l/ng= +github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add/go.mod h1:DQI8vlV6h6qSY/tCOoYKtxjWrkyiNpJ3WTV/WoBllmQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= @@ -1252,8 +1253,9 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.0.4/go.mod h1:9T/Cfuxs5StfsocWr4WzDL36HqnX0fVb9d5fSEaLhoE= github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1267,19 +1269,20 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= +github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ= github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/shibumi/go-pathspec v1.2.0 h1:KVKEDHYk7bQolRMs7nfzjT3SBOCgcXFJzccnj9bsGbA= -github.com/shibumi/go-pathspec v1.2.0/go.mod h1:bDxCftD0fST3qXIlHoQ/fChsU4mWMVklXp1yPErQaaY= +github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= +github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go index 0668a66cf707..be2b3436062d 100644 --- a/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go +++ b/vendor/github.com/cpuguy83/go-md2man/v2/md2man/roff.go @@ -15,7 +15,7 @@ type roffRenderer struct { extensions blackfriday.Extensions listCounters []int firstHeader bool - defineTerm bool + firstDD bool listDepth int } @@ -42,7 +42,8 @@ const ( quoteCloseTag = "\n.RE\n" listTag = "\n.RS\n" listCloseTag = "\n.RE\n" - arglistTag = "\n.TP\n" + dtTag = "\n.TP\n" + dd2Tag = "\n" tableStart = "\n.TS\nallbox;\n" tableEnd = ".TE\n" tableCellStart = "T{\n" @@ -90,7 +91,7 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering switch node.Type { case blackfriday.Text: - r.handleText(w, node, entering) + escapeSpecialChars(w, node.Literal) case blackfriday.Softbreak: out(w, crTag) case blackfriday.Hardbreak: @@ -150,40 +151,21 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering out(w, codeCloseTag) case blackfriday.Table: r.handleTable(w, node, entering) - case blackfriday.TableCell: - r.handleTableCell(w, node, entering) case blackfriday.TableHead: case blackfriday.TableBody: case blackfriday.TableRow: // no action as cell entries do all the nroff formatting return blackfriday.GoToNext + case blackfriday.TableCell: + r.handleTableCell(w, node, entering) + case blackfriday.HTMLSpan: + // ignore other HTML tags default: fmt.Fprintln(os.Stderr, "WARNING: go-md2man does not handle node type "+node.Type.String()) } return walkAction } -func (r *roffRenderer) handleText(w io.Writer, node *blackfriday.Node, entering bool) { - var ( - start, end string - ) - // handle special roff table cell text encapsulation - if node.Parent.Type == blackfriday.TableCell { - if len(node.Literal) > 30 { - start = tableCellStart - end = tableCellEnd - } else { - // end rows that aren't terminated by "tableCellEnd" with a cr if end of row - if node.Parent.Next == nil && !node.Parent.IsHeader { - end = crTag - } - } - } - out(w, start) - escapeSpecialChars(w, node.Literal) - out(w, end) -} - func (r *roffRenderer) handleHeading(w io.Writer, node *blackfriday.Node, entering bool) { if entering { switch node.Level { @@ -230,15 +212,20 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering if node.ListFlags&blackfriday.ListTypeOrdered != 0 { out(w, fmt.Sprintf(".IP \"%3d.\" 5\n", r.listCounters[len(r.listCounters)-1])) r.listCounters[len(r.listCounters)-1]++ + } else if node.ListFlags&blackfriday.ListTypeTerm != 0 { + // DT (definition term): line just before DD (see below). + out(w, dtTag) + r.firstDD = true } else if node.ListFlags&blackfriday.ListTypeDefinition != 0 { - // state machine for handling terms and following definitions - // since blackfriday does not distinguish them properly, nor - // does it seperate them into separate lists as it should - if !r.defineTerm { - out(w, arglistTag) - r.defineTerm = true + // DD (definition description): line that starts with ": ". + // + // We have to distinguish between the first DD and the + // subsequent ones, as there should be no vertical + // whitespace between the DT and the first DD. + if r.firstDD { + r.firstDD = false } else { - r.defineTerm = false + out(w, dd2Tag) } } else { out(w, ".IP \\(bu 2\n") @@ -251,7 +238,7 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering bool) { if entering { out(w, tableStart) - //call walker to count cells (and rows?) so format section can be produced + // call walker to count cells (and rows?) so format section can be produced columns := countColumns(node) out(w, strings.Repeat("l ", columns)+"\n") out(w, strings.Repeat("l ", columns)+".\n") @@ -261,28 +248,41 @@ func (r *roffRenderer) handleTable(w io.Writer, node *blackfriday.Node, entering } func (r *roffRenderer) handleTableCell(w io.Writer, node *blackfriday.Node, entering bool) { - var ( - start, end string - ) - if node.IsHeader { - start = codespanTag - end = codespanCloseTag - } if entering { + var start string if node.Prev != nil && node.Prev.Type == blackfriday.TableCell { - out(w, "\t"+start) - } else { - out(w, start) + start = "\t" + } + if node.IsHeader { + start += codespanTag + } else if nodeLiteralSize(node) > 30 { + start += tableCellStart } + out(w, start) } else { - // need to carriage return if we are at the end of the header row - if node.IsHeader && node.Next == nil { - end = end + crTag + var end string + if node.IsHeader { + end = codespanCloseTag + } else if nodeLiteralSize(node) > 30 { + end = tableCellEnd + } + if node.Next == nil && end != tableCellEnd { + // Last cell: need to carriage return if we are at the end of the + // header row and content isn't wrapped in a "tablecell" + end += crTag } out(w, end) } } +func nodeLiteralSize(node *blackfriday.Node) int { + total := 0 + for n := node.FirstChild; n != nil; n = n.FirstChild { + total += len(n.Literal) + } + return total +} + // because roff format requires knowing the column count before outputting any table // data we need to walk a table tree and count the columns func countColumns(node *blackfriday.Node) int { @@ -309,15 +309,6 @@ func out(w io.Writer, output string) { io.WriteString(w, output) // nolint: errcheck } -func needsBackslash(c byte) bool { - for _, r := range []byte("-_&\\~") { - if c == r { - return true - } - } - return false -} - func escapeSpecialChars(w io.Writer, text []byte) { for i := 0; i < len(text); i++ { // escape initial apostrophe or period @@ -328,7 +319,7 @@ func escapeSpecialChars(w io.Writer, text []byte) { // directly copy normal characters org := i - for i < len(text) && !needsBackslash(text[i]) { + for i < len(text) && text[i] != '\\' { i++ } if i > org { diff --git a/vendor/github.com/in-toto/in-toto-golang/in_toto/keylib.go b/vendor/github.com/in-toto/in-toto-golang/in_toto/keylib.go index 15d098acdf9e..8811d32d9332 100644 --- a/vendor/github.com/in-toto/in-toto-golang/in_toto/keylib.go +++ b/vendor/github.com/in-toto/in-toto-golang/in_toto/keylib.go @@ -16,6 +16,8 @@ import ( "io/ioutil" "os" "strings" + + "github.com/secure-systems-lab/go-securesystemslib/cjson" ) // ErrFailedPEMParsing gets returned when PKCS1, PKCS8 or PKIX key parsing fails @@ -106,7 +108,7 @@ func (k *Key) generateKeyID() error { "public": k.KeyVal.Public, }, } - keyCanonical, err := EncodeCanonical(keyToBeHashed) + keyCanonical, err := cjson.EncodeCanonical(keyToBeHashed) if err != nil { return err } diff --git a/vendor/github.com/in-toto/in-toto-golang/in_toto/model.go b/vendor/github.com/in-toto/in-toto-golang/in_toto/model.go index 82c1e09041ad..25e12c033e69 100644 --- a/vendor/github.com/in-toto/in-toto-golang/in_toto/model.go +++ b/vendor/github.com/in-toto/in-toto-golang/in_toto/model.go @@ -15,7 +15,10 @@ import ( "strings" "time" - "github.com/in-toto/in-toto-golang/pkg/ssl" + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + + "github.com/secure-systems-lab/go-securesystemslib/cjson" + "github.com/secure-systems-lab/go-securesystemslib/dsse" ) /* @@ -76,10 +79,10 @@ const ( // The SPDX mandates 'spdxVersion' field, so predicate type can omit // version. PredicateSPDX = "https://spdx.dev/Document" + // PredicateCycloneDX represents a CycloneDX SBOM + PredicateCycloneDX = "https://cyclonedx.org/schema" // PredicateLinkV1 represents an in-toto 0.9 link. PredicateLinkV1 = "https://in-toto.io/Link/v1" - // PredicateSLSAProvenanceV01 represents a build provenance for an artifact. - PredicateSLSAProvenanceV01 = "https://slsa.dev/provenance/v0.1" ) // ErrInvalidPayloadType indicates that the envelope used an unkown payload type @@ -873,7 +876,7 @@ Signed field of the Metablock on which it was called. If canonicalization fails the first return value is nil and the second return value is the error. */ func (mb *Metablock) GetSignableRepresentation() ([]byte, error) { - return EncodeCanonical(mb.Signed) + return cjson.EncodeCanonical(mb.Signed) } /* @@ -959,16 +962,10 @@ func (mb *Metablock) Sign(key Key) error { return nil } -/* -DigestSet contains a set of digests. It is represented as a map from -algorithm name to lowercase hex-encoded value. -*/ -type DigestSet map[string]string - // Subject describes the set of software artifacts the statement applies to. type Subject struct { - Name string `json:"name"` - Digest DigestSet `json:"digest"` + Name string `json:"name"` + Digest slsa.DigestSet `json:"digest"` } // StatementHeader defines the common fields for all statements @@ -988,59 +985,10 @@ type Statement struct { Predicate interface{} `json:"predicate"` } -// ProvenanceBuilder idenfifies the entity that executed the build steps. -type ProvenanceBuilder struct { - ID string `json:"id"` -} - -// ProvenanceRecipe describes the actions performed by the builder. -type ProvenanceRecipe struct { - Type string `json:"type"` - // DefinedInMaterial can be sent as the null pointer to indicate that - // the value is not present. - DefinedInMaterial *int `json:"definedInMaterial,omitempty"` - EntryPoint string `json:"entryPoint"` - Arguments interface{} `json:"arguments,omitempty"` - Environment interface{} `json:"environment,omitempty"` -} - -// ProvenanceComplete indicates wheter the claims in build/recipe are complete. -// For in depth information refer to the specifictaion: -// https://github.com/in-toto/attestation/blob/v0.1.0/spec/predicates/provenance.md -type ProvenanceComplete struct { - Arguments bool `json:"arguments"` - Environment bool `json:"environment"` - Materials bool `json:"materials"` -} - -// ProvenanceMetadata contains metadata for the built artifact. -type ProvenanceMetadata struct { - // Use pointer to make sure that the abscense of a time is not - // encoded as the Epoch time. - BuildStartedOn *time.Time `json:"buildStartedOn,omitempty"` - BuildFinishedOn *time.Time `json:"buildFinishedOn,omitempty"` - Completeness ProvenanceComplete `json:"completeness"` - Reproducible bool `json:"reproducible"` -} - -// ProvenanceMaterial defines the materials used to build an artifact. -type ProvenanceMaterial struct { - URI string `json:"uri"` - Digest DigestSet `json:"digest,omitempty"` -} - -// ProvenancePredicate is the provenance predicate definition. -type ProvenancePredicate struct { - Builder ProvenanceBuilder `json:"builder"` - Recipe ProvenanceRecipe `json:"recipe"` - Metadata *ProvenanceMetadata `json:"metadata,omitempty"` - Materials []ProvenanceMaterial `json:"materials,omitempty"` -} - // ProvenanceStatement is the definition for an entire provenance statement. type ProvenanceStatement struct { StatementHeader - Predicate ProvenancePredicate `json:"predicate"` + Predicate slsa.ProvenancePredicate `json:"predicate"` } // LinkStatement is the definition for an entire link statement. @@ -1051,7 +999,7 @@ type LinkStatement struct { /* SPDXStatement is the definition for an entire SPDX statement. -Currently not implemented. Some tooling exists here: +This is currently not implemented. Some tooling exists here: https://github.com/spdx/tools-golang, but this software is still in early state. This struct is the same as the generic Statement struct but is added for @@ -1063,35 +1011,46 @@ type SPDXStatement struct { } /* -SSLSigner provides signature generation and validation based on the SSL +CycloneDXStatement defines a cyclonedx sbom in the predicate. It is not +currently serialized just as its SPDX counterpart. It is an empty +interface, like the generic Statement. +*/ +type CycloneDXStatement struct { + StatementHeader + Predicate interface{} `json:"predicate"` +} + +/* +DSSESigner provides signature generation and validation based on the SSL Signing Spec: https://github.com/secure-systems-lab/signing-spec as describe by: https://github.com/MarkLodato/ITE/tree/media-type/ITE/5 It wraps the generic SSL envelope signer and enforces the correct payload type both during signature generation and validation. */ -type SSLSigner struct { - signer *ssl.EnvelopeSigner +type DSSESigner struct { + signer *dsse.EnvelopeSigner } -func NewSSLSigner(p ...ssl.SignVerifier) (*SSLSigner, error) { - es, err := ssl.NewEnvelopeSigner(p...) +func NewDSSESigner(p ...dsse.SignVerifier) (*DSSESigner, error) { + es, err := dsse.NewEnvelopeSigner(p...) if err != nil { return nil, err } - return &SSLSigner{ + return &DSSESigner{ signer: es, }, nil } -func (s *SSLSigner) SignPayload(body []byte) (*ssl.Envelope, error) { +func (s *DSSESigner) SignPayload(body []byte) (*dsse.Envelope, error) { return s.signer.SignPayload(PayloadType, body) } -func (s *SSLSigner) Verify(e *ssl.Envelope) error { +func (s *DSSESigner) Verify(e *dsse.Envelope) error { if e.PayloadType != PayloadType { return ErrInvalidPayloadType } - return s.signer.Verify(e) + _, err := s.signer.Verify(e) + return err } diff --git a/vendor/github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2/provenance.go b/vendor/github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2/provenance.go new file mode 100644 index 000000000000..1cecf3a4989f --- /dev/null +++ b/vendor/github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2/provenance.go @@ -0,0 +1,66 @@ +package v02 + +import "time" + +const ( + // PredicateSLSAProvenance represents a build provenance for an artifact. + PredicateSLSAProvenance = "https://slsa.dev/provenance/v0.2" +) + +// ProvenancePredicate is the provenance predicate definition. +type ProvenancePredicate struct { + Builder ProvenanceBuilder `json:"builder"` + BuildType string `json:"buildType"` + Invocation ProvenanceInvocation `json:"invocation,omitempty"` + BuildConfig interface{} `json:"buildConfig,omitempty"` + Metadata *ProvenanceMetadata `json:"metadata,omitempty"` + Materials []ProvenanceMaterial `json:"materials,omitempty"` +} + +// ProvenanceBuilder idenfifies the entity that executed the build steps. +type ProvenanceBuilder struct { + ID string `json:"id"` +} + +// ProvenanceInvocation identifies the event that kicked off the build. +type ProvenanceInvocation struct { + ConfigSource ConfigSource `json:"configSource,omitempty"` + Parameters interface{} `json:"parameters,omitempty"` + Environment interface{} `json:"environment,omitempty"` +} + +type ConfigSource struct { + URI string `json:"uri,omitempty"` + Digest DigestSet `json:"digest,omitempty"` + EntryPoint string `json:"entryPoint,omitempty"` +} + +// ProvenanceMetadata contains metadata for the built artifact. +type ProvenanceMetadata struct { + BuildInvocationID string `json:"buildInvocationID,omitempty"` + // Use pointer to make sure that the abscense of a time is not + // encoded as the Epoch time. + BuildStartedOn *time.Time `json:"buildStartedOn,omitempty"` + BuildFinishedOn *time.Time `json:"buildFinishedOn,omitempty"` + Completeness ProvenanceComplete `json:"completeness"` + Reproducible bool `json:"reproducible"` +} + +// ProvenanceMaterial defines the materials used to build an artifact. +type ProvenanceMaterial struct { + URI string `json:"uri,omitempty"` + Digest DigestSet `json:"digest,omitempty"` +} + +// ProvenanceComplete indicates wheter the claims in build/recipe are complete. +// For in depth information refer to the specifictaion: +// https://github.com/in-toto/attestation/blob/v0.1.0/spec/predicates/provenance.md +type ProvenanceComplete struct { + Parameters bool `json:"parameters"` + Environment bool `json:"environment"` + Materials bool `json:"materials"` +} + +// DigestSet contains a set of digests. It is represented as a map from +// algorithm name to lowercase hex-encoded value. +type DigestSet map[string]string diff --git a/vendor/github.com/in-toto/in-toto-golang/pkg/ssl/verify.go b/vendor/github.com/in-toto/in-toto-golang/pkg/ssl/verify.go deleted file mode 100644 index 426968debe0f..000000000000 --- a/vendor/github.com/in-toto/in-toto-golang/pkg/ssl/verify.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Package ssl implements the Secure Systems Lab signing-spec (sometimes -abbreviated SSL Siging spec. -https://github.com/secure-systems-lab/signing-spec -*/ -package ssl - -/* -Verifier verifies a complete message against a signature and key. -If the message was hashed prior to signature generation, the verifier -must perform the same steps. -If the key is not recognized ErrUnknownKey shall be returned. -*/ -type Verifier interface { - Verify(keyID string, data, sig []byte) error -} - -type EnvelopeVerifier struct { - providers []Verifier -} - -func (ev *EnvelopeVerifier) Verify(e *Envelope) error { - if len(e.Signatures) == 0 { - return ErrNoSignature - } - - // Decode payload (i.e serialized body) - body, err := b64Decode(e.Payload) - if err != nil { - return err - } - // Generate PAE(payloadtype, serialized body) - paeEnc := PAE(e.PayloadType, string(body)) - - // If *any* signature is found to be incorrect, the entire verification - // step fails even if *some* signatures are correct. - verified := false - for _, s := range e.Signatures { - sig, err := b64Decode(s.Sig) - if err != nil { - return err - } - - // Loop over the providers. If a provider recognizes the key, we exit - // the loop and use the result. - for _, v := range ev.providers { - err := v.Verify(s.KeyID, paeEnc, sig) - if err != nil { - if err == ErrUnknownKey { - continue - } - return err - } - - verified = true - break - } - } - if !verified { - return ErrUnknownKey - } - - return nil -} - -func NewEnvelopeVerifier(p ...Verifier) EnvelopeVerifier { - ev := EnvelopeVerifier{ - providers: p, - } - return ev -} diff --git a/vendor/github.com/russross/blackfriday/v2/README.md b/vendor/github.com/russross/blackfriday/v2/README.md index d5a8649bd532..d9c08a22fc54 100644 --- a/vendor/github.com/russross/blackfriday/v2/README.md +++ b/vendor/github.com/russross/blackfriday/v2/README.md @@ -1,4 +1,6 @@ -Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday) +Blackfriday +[![Build Status][BuildV2SVG]][BuildV2URL] +[![PkgGoDev][PkgGoDevV2SVG]][PkgGoDevV2URL] =========== Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It @@ -16,19 +18,21 @@ It started as a translation from C of [Sundown][3]. Installation ------------ -Blackfriday is compatible with any modern Go release. With Go 1.7 and git -installed: +Blackfriday is compatible with modern Go releases in module mode. +With Go installed: - go get gopkg.in/russross/blackfriday.v2 + go get github.com/russross/blackfriday/v2 -will download, compile, and install the package into your `$GOPATH` -directory hierarchy. Alternatively, you can achieve the same if you -import it into a project: +will resolve and add the package to the current development module, +then build and install it. Alternatively, you can achieve the same +if you import it in a package: - import "gopkg.in/russross/blackfriday.v2" + import "github.com/russross/blackfriday/v2" and `go get` without parameters. +Legacy GOPATH mode is unsupported. + Versions -------- @@ -36,13 +40,9 @@ Versions Currently maintained and recommended version of Blackfriday is `v2`. It's being developed on its own branch: https://github.com/russross/blackfriday/tree/v2 and the documentation is available at -https://godoc.org/gopkg.in/russross/blackfriday.v2. +https://pkg.go.dev/github.com/russross/blackfriday/v2. -It is `go get`-able via via [gopkg.in][6] at `gopkg.in/russross/blackfriday.v2`, -but we highly recommend using package management tool like [dep][7] or -[Glide][8] and make use of semantic versioning. With package management you -should import `github.com/russross/blackfriday` and specify that you're using -version 2.0.0. +It is `go get`-able in module mode at `github.com/russross/blackfriday/v2`. Version 2 offers a number of improvements over v1: @@ -62,6 +62,11 @@ Potential drawbacks: v2. See issue [#348](https://github.com/russross/blackfriday/issues/348) for tracking. +If you are still interested in the legacy `v1`, you can import it from +`github.com/russross/blackfriday`. Documentation for the legacy v1 can be found +here: https://pkg.go.dev/github.com/russross/blackfriday. + + Usage ----- @@ -91,7 +96,7 @@ Here's an example of simple usage of Blackfriday together with Bluemonday: ```go import ( "github.com/microcosm-cc/bluemonday" - "github.com/russross/blackfriday" + "github.com/russross/blackfriday/v2" ) // ... @@ -104,6 +109,8 @@ html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) If you want to customize the set of options, use `blackfriday.WithExtensions`, `blackfriday.WithRenderer` and `blackfriday.WithRefOverride`. +### `blackfriday-tool` + You can also check out `blackfriday-tool` for a more complete example of how to use it. Download and install it using: @@ -114,7 +121,7 @@ markdown file using a standalone program. You can also browse the source directly on github if you are just looking for some example code: -* +* Note that if you have not already done so, installing `blackfriday-tool` will be sufficient to download and install @@ -123,6 +130,22 @@ installed in `$GOPATH/bin`. This is a statically-linked binary that can be copied to wherever you need it without worrying about dependencies and library versions. +### Sanitized anchor names + +Blackfriday includes an algorithm for creating sanitized anchor names +corresponding to a given input text. This algorithm is used to create +anchors for headings when `AutoHeadingIDs` extension is enabled. The +algorithm has a specification, so that other packages can create +compatible anchor names and links to those anchors. + +The specification is located at https://pkg.go.dev/github.com/russross/blackfriday/v2#hdr-Sanitized_Anchor_Names. + +[`SanitizedAnchorName`](https://pkg.go.dev/github.com/russross/blackfriday/v2#SanitizedAnchorName) exposes this functionality, and can be used to +create compatible links to the anchor names generated by blackfriday. +This algorithm is also implemented in a small standalone package at +[`github.com/shurcooL/sanitized_anchor_name`](https://pkg.go.dev/github.com/shurcooL/sanitized_anchor_name). It can be useful for clients +that want a small package and don't need full functionality of blackfriday. + Features -------- @@ -199,6 +222,15 @@ implements the following extensions: You can use 3 or more backticks to mark the beginning of the block, and the same number to mark the end of the block. + To preserve classes of fenced code blocks while using the bluemonday + HTML sanitizer, use the following policy: + + ```go + p := bluemonday.UGCPolicy() + p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code") + html := p.SanitizeBytes(unsafe) + ``` + * **Definition lists**. A simple definition list is made of a single-line term followed by a colon and the definition for that term. @@ -250,7 +282,7 @@ Other renderers Blackfriday is structured to allow alternative rendering engines. Here are a few of note: -* [github_flavored_markdown](https://godoc.org/github.com/shurcooL/github_flavored_markdown): +* [github_flavored_markdown](https://pkg.go.dev/github.com/shurcooL/github_flavored_markdown): provides a GitHub Flavored Markdown renderer with fenced code block highlighting, clickable heading anchor links. @@ -261,20 +293,28 @@ are a few of note: * [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt, but for markdown. -* [LaTeX output](https://github.com/Ambrevar/Blackfriday-LaTeX): +* [LaTeX output](https://gitlab.com/ambrevar/blackfriday-latex): renders output as LaTeX. +* [bfchroma](https://github.com/Depado/bfchroma/): provides convenience + integration with the [Chroma](https://github.com/alecthomas/chroma) code + highlighting library. bfchroma is only compatible with v2 of Blackfriday and + provides a drop-in renderer ready to use with Blackfriday, as well as + options and means for further customization. + * [Blackfriday-Confluence](https://github.com/kentaro-m/blackfriday-confluence): provides a [Confluence Wiki Markup](https://confluence.atlassian.com/doc/confluence-wiki-markup-251003035.html) renderer. +* [Blackfriday-Slack](https://github.com/karriereat/blackfriday-slack): converts markdown to slack message style + -Todo +TODO ---- * More unit testing -* Improve unicode support. It does not understand all unicode +* Improve Unicode support. It does not understand all Unicode rules (about what constitutes a letter, a punctuation symbol, etc.), so it may fail to detect word boundaries correctly in - some instances. It is safe on all utf-8 input. + some instances. It is safe on all UTF-8 input. License @@ -286,6 +326,10 @@ License [1]: https://daringfireball.net/projects/markdown/ "Markdown" [2]: https://golang.org/ "Go Language" [3]: https://github.com/vmg/sundown "Sundown" - [4]: https://godoc.org/gopkg.in/russross/blackfriday.v2#Parse "Parse func" + [4]: https://pkg.go.dev/github.com/russross/blackfriday/v2#Parse "Parse func" [5]: https://github.com/microcosm-cc/bluemonday "Bluemonday" - [6]: https://labix.org/gopkg.in "gopkg.in" + + [BuildV2SVG]: https://travis-ci.org/russross/blackfriday.svg?branch=v2 + [BuildV2URL]: https://travis-ci.org/russross/blackfriday + [PkgGoDevV2SVG]: https://pkg.go.dev/badge/github.com/russross/blackfriday/v2 + [PkgGoDevV2URL]: https://pkg.go.dev/github.com/russross/blackfriday/v2 diff --git a/vendor/github.com/russross/blackfriday/v2/block.go b/vendor/github.com/russross/blackfriday/v2/block.go index b8607474e599..dcd61e6e35bc 100644 --- a/vendor/github.com/russross/blackfriday/v2/block.go +++ b/vendor/github.com/russross/blackfriday/v2/block.go @@ -18,8 +18,7 @@ import ( "html" "regexp" "strings" - - "github.com/shurcooL/sanitized_anchor_name" + "unicode" ) const ( @@ -259,7 +258,7 @@ func (p *Markdown) prefixHeading(data []byte) int { } if end > i { if id == "" && p.extensions&AutoHeadingIDs != 0 { - id = sanitized_anchor_name.Create(string(data[i:end])) + id = SanitizedAnchorName(string(data[i:end])) } block := p.addBlock(Heading, data[i:end]) block.HeadingID = id @@ -673,6 +672,7 @@ func (p *Markdown) fencedCodeBlock(data []byte, doRender bool) int { if beg == 0 || beg >= len(data) { return 0 } + fenceLength := beg - 1 var work bytes.Buffer work.Write([]byte(info)) @@ -706,6 +706,7 @@ func (p *Markdown) fencedCodeBlock(data []byte, doRender bool) int { if doRender { block := p.addBlock(CodeBlock, work.Bytes()) // TODO: get rid of temp buffer block.IsFenced = true + block.FenceLength = fenceLength finalizeCodeBlock(block) } @@ -1503,7 +1504,7 @@ func (p *Markdown) paragraph(data []byte) int { id := "" if p.extensions&AutoHeadingIDs != 0 { - id = sanitized_anchor_name.Create(string(data[prev:eol])) + id = SanitizedAnchorName(string(data[prev:eol])) } block := p.addBlock(Heading, data[prev:eol]) @@ -1588,3 +1589,24 @@ func skipUntilChar(text []byte, start int, char byte) int { } return i } + +// SanitizedAnchorName returns a sanitized anchor name for the given text. +// +// It implements the algorithm specified in the package comment. +func SanitizedAnchorName(text string) string { + var anchorName []rune + futureDash := false + for _, r := range text { + switch { + case unicode.IsLetter(r) || unicode.IsNumber(r): + if futureDash && len(anchorName) > 0 { + anchorName = append(anchorName, '-') + } + futureDash = false + anchorName = append(anchorName, unicode.ToLower(r)) + default: + futureDash = true + } + } + return string(anchorName) +} diff --git a/vendor/github.com/russross/blackfriday/v2/doc.go b/vendor/github.com/russross/blackfriday/v2/doc.go index 5b3fa9876ac8..57ff152a0568 100644 --- a/vendor/github.com/russross/blackfriday/v2/doc.go +++ b/vendor/github.com/russross/blackfriday/v2/doc.go @@ -15,4 +15,32 @@ // // If you're interested in calling Blackfriday from command line, see // https://github.com/russross/blackfriday-tool. +// +// Sanitized Anchor Names +// +// Blackfriday includes an algorithm for creating sanitized anchor names +// corresponding to a given input text. This algorithm is used to create +// anchors for headings when AutoHeadingIDs extension is enabled. The +// algorithm is specified below, so that other packages can create +// compatible anchor names and links to those anchors. +// +// The algorithm iterates over the input text, interpreted as UTF-8, +// one Unicode code point (rune) at a time. All runes that are letters (category L) +// or numbers (category N) are considered valid characters. They are mapped to +// lower case, and included in the output. All other runes are considered +// invalid characters. Invalid characters that precede the first valid character, +// as well as invalid character that follow the last valid character +// are dropped completely. All other sequences of invalid characters +// between two valid characters are replaced with a single dash character '-'. +// +// SanitizedAnchorName exposes this functionality, and can be used to +// create compatible links to the anchor names generated by blackfriday. +// This algorithm is also implemented in a small standalone package at +// github.com/shurcooL/sanitized_anchor_name. It can be useful for clients +// that want a small package and don't need full functionality of blackfriday. package blackfriday + +// NOTE: Keep Sanitized Anchor Name algorithm in sync with package +// github.com/shurcooL/sanitized_anchor_name. +// Otherwise, users of sanitized_anchor_name will get anchor names +// that are incompatible with those generated by blackfriday. diff --git a/vendor/github.com/russross/blackfriday/v2/entities.go b/vendor/github.com/russross/blackfriday/v2/entities.go new file mode 100644 index 000000000000..a2c3edb691c8 --- /dev/null +++ b/vendor/github.com/russross/blackfriday/v2/entities.go @@ -0,0 +1,2236 @@ +package blackfriday + +// Extracted from https://html.spec.whatwg.org/multipage/entities.json +var entities = map[string]bool{ + "Æ": true, + "Æ": true, + "&": true, + "&": true, + "Á": true, + "Á": true, + "Ă": true, + "Â": true, + "Â": true, + "А": true, + "𝔄": true, + "À": true, + "À": true, + "Α": true, + "Ā": true, + "⩓": true, + "Ą": true, + "𝔸": true, + "⁡": true, + "Å": true, + "Å": true, + "𝒜": true, + "≔": true, + "Ã": true, + "Ã": true, + "Ä": true, + "Ä": true, + "∖": true, + "⫧": true, + "⌆": true, + "Б": true, + "∵": true, + "ℬ": true, + "Β": true, + "𝔅": true, + "𝔹": true, + "˘": true, + "ℬ": true, + "≎": true, + "Ч": true, + "©": true, + "©": true, + "Ć": true, + "⋒": true, + "ⅅ": true, + "ℭ": true, + "Č": true, + "Ç": true, + "Ç": true, + "Ĉ": true, + "∰": true, + "Ċ": true, + "¸": true, + "·": true, + "ℭ": true, + "Χ": true, + "⊙": true, + "⊖": true, + "⊕": true, + "⊗": true, + "∲": true, + "”": true, + "’": true, + "∷": true, + "⩴": true, + "≡": true, + "∯": true, + "∮": true, + "ℂ": true, + "∐": true, + "∳": true, + "⨯": true, + "𝒞": true, + "⋓": true, + "≍": true, + "ⅅ": true, + "⤑": true, + "Ђ": true, + "Ѕ": true, + "Џ": true, + "‡": true, + "↡": true, + "⫤": true, + "Ď": true, + "Д": true, + "∇": true, + "Δ": true, + "𝔇": true, + "´": true, + "˙": true, + "˝": true, + "`": true, + "˜": true, + "⋄": true, + "ⅆ": true, + "𝔻": true, + "¨": true, + "⃜": true, + "≐": true, + "∯": true, + "¨": true, + "⇓": true, + "⇐": true, + "⇔": true, + "⫤": true, + "⟸": true, + "⟺": true, + "⟹": true, + "⇒": true, + "⊨": true, + "⇑": true, + "⇕": true, + "∥": true, + "↓": true, + "⤓": true, + "⇵": true, + "̑": true, + "⥐": true, + "⥞": true, + "↽": true, + "⥖": true, + "⥟": true, + "⇁": true, + "⥗": true, + "⊤": true, + "↧": true, + "⇓": true, + "𝒟": true, + "Đ": true, + "Ŋ": true, + "Ð": true, + "Ð": true, + "É": true, + "É": true, + "Ě": true, + "Ê": true, + "Ê": true, + "Э": true, + "Ė": true, + "𝔈": true, + "È": true, + "È": true, + "∈": true, + "Ē": true, + "◻": true, + "▫": true, + "Ę": true, + "𝔼": true, + "Ε": true, + "⩵": true, + "≂": true, + "⇌": true, + "ℰ": true, + "⩳": true, + "Η": true, + "Ë": true, + "Ë": true, + "∃": true, + "ⅇ": true, + "Ф": true, + "𝔉": true, + "◼": true, + "▪": true, + "𝔽": true, + "∀": true, + "ℱ": true, + "ℱ": true, + "Ѓ": true, + ">": true, + ">": true, + "Γ": true, + "Ϝ": true, + "Ğ": true, + "Ģ": true, + "Ĝ": true, + "Г": true, + "Ġ": true, + "𝔊": true, + "⋙": true, + "𝔾": true, + "≥": true, + "⋛": true, + "≧": true, + "⪢": true, + "≷": true, + "⩾": true, + "≳": true, + "𝒢": true, + "≫": true, + "Ъ": true, + "ˇ": true, + "^": true, + "Ĥ": true, + "ℌ": true, + "ℋ": true, + "ℍ": true, + "─": true, + "ℋ": true, + "Ħ": true, + "≎": true, + "≏": true, + "Е": true, + "IJ": true, + "Ё": true, + "Í": true, + "Í": true, + "Î": true, + "Î": true, + "И": true, + "İ": true, + "ℑ": true, + "Ì": true, + "Ì": true, + "ℑ": true, + "Ī": true, + "ⅈ": true, + "⇒": true, + "∬": true, + "∫": true, + "⋂": true, + "⁣": true, + "⁢": true, + "Į": true, + "𝕀": true, + "Ι": true, + "ℐ": true, + "Ĩ": true, + "І": true, + "Ï": true, + "Ï": true, + "Ĵ": true, + "Й": true, + "𝔍": true, + "𝕁": true, + "𝒥": true, + "Ј": true, + "Є": true, + "Х": true, + "Ќ": true, + "Κ": true, + "Ķ": true, + "К": true, + "𝔎": true, + "𝕂": true, + "𝒦": true, + "Љ": true, + "<": true, + "<": true, + "Ĺ": true, + "Λ": true, + "⟪": true, + "ℒ": true, + "↞": true, + "Ľ": true, + "Ļ": true, + "Л": true, + "⟨": true, + "←": true, + "⇤": true, + "⇆": true, + "⌈": true, + "⟦": true, + "⥡": true, + "⇃": true, + "⥙": true, + "⌊": true, + "↔": true, + "⥎": true, + "⊣": true, + "↤": true, + "⥚": true, + "⊲": true, + "⧏": true, + "⊴": true, + "⥑": true, + "⥠": true, + "↿": true, + "⥘": true, + "↼": true, + "⥒": true, + "⇐": true, + "⇔": true, + "⋚": true, + "≦": true, + "≶": true, + "⪡": true, + "⩽": true, + "≲": true, + "𝔏": true, + "⋘": true, + "⇚": true, + "Ŀ": true, + "⟵": true, + "⟷": true, + "⟶": true, + "⟸": true, + "⟺": true, + "⟹": true, + "𝕃": true, + "↙": true, + "↘": true, + "ℒ": true, + "↰": true, + "Ł": true, + "≪": true, + "⤅": true, + "М": true, + " ": true, + "ℳ": true, + "𝔐": true, + "∓": true, + "𝕄": true, + "ℳ": true, + "Μ": true, + "Њ": true, + "Ń": true, + "Ň": true, + "Ņ": true, + "Н": true, + "​": true, + "​": true, + "​": true, + "​": true, + "≫": true, + "≪": true, + " ": true, + "𝔑": true, + "⁠": true, + " ": true, + "ℕ": true, + "⫬": true, + "≢": true, + "≭": true, + "∦": true, + "∉": true, + "≠": true, + "≂̸": true, + "∄": true, + "≯": true, + "≱": true, + "≧̸": true, + "≫̸": true, + "≹": true, + "⩾̸": true, + "≵": true, + "≎̸": true, + "≏̸": true, + "⋪": true, + "⧏̸": true, + "⋬": true, + "≮": true, + "≰": true, + "≸": true, + "≪̸": true, + "⩽̸": true, + "≴": true, + "⪢̸": true, + "⪡̸": true, + "⊀": true, + "⪯̸": true, + "⋠": true, + "∌": true, + "⋫": true, + "⧐̸": true, + "⋭": true, + "⊏̸": true, + "⋢": true, + "⊐̸": true, + "⋣": true, + "⊂⃒": true, + "⊈": true, + "⊁": true, + "⪰̸": true, + "⋡": true, + "≿̸": true, + "⊃⃒": true, + "⊉": true, + "≁": true, + "≄": true, + "≇": true, + "≉": true, + "∤": true, + "𝒩": true, + "Ñ": true, + "Ñ": true, + "Ν": true, + "Œ": true, + "Ó": true, + "Ó": true, + "Ô": true, + "Ô": true, + "О": true, + "Ő": true, + "𝔒": true, + "Ò": true, + "Ò": true, + "Ō": true, + "Ω": true, + "Ο": true, + "𝕆": true, + "“": true, + "‘": true, + "⩔": true, + "𝒪": true, + "Ø": true, + "Ø": true, + "Õ": true, + "Õ": true, + "⨷": true, + "Ö": true, + "Ö": true, + "‾": true, + "⏞": true, + "⎴": true, + "⏜": true, + "∂": true, + "П": true, + "𝔓": true, + "Φ": true, + "Π": true, + "±": true, + "ℌ": true, + "ℙ": true, + "⪻": true, + "≺": true, + "⪯": true, + "≼": true, + "≾": true, + "″": true, + "∏": true, + "∷": true, + "∝": true, + "𝒫": true, + "Ψ": true, + """: true, + """: true, + "𝔔": true, + "ℚ": true, + "𝒬": true, + "⤐": true, + "®": true, + "®": true, + "Ŕ": true, + "⟫": true, + "↠": true, + "⤖": true, + "Ř": true, + "Ŗ": true, + "Р": true, + "ℜ": true, + "∋": true, + "⇋": true, + "⥯": true, + "ℜ": true, + "Ρ": true, + "⟩": true, + "→": true, + "⇥": true, + "⇄": true, + "⌉": true, + "⟧": true, + "⥝": true, + "⇂": true, + "⥕": true, + "⌋": true, + "⊢": true, + "↦": true, + "⥛": true, + "⊳": true, + "⧐": true, + "⊵": true, + "⥏": true, + "⥜": true, + "↾": true, + "⥔": true, + "⇀": true, + "⥓": true, + "⇒": true, + "ℝ": true, + "⥰": true, + "⇛": true, + "ℛ": true, + "↱": true, + "⧴": true, + "Щ": true, + "Ш": true, + "Ь": true, + "Ś": true, + "⪼": true, + "Š": true, + "Ş": true, + "Ŝ": true, + "С": true, + "𝔖": true, + "↓": true, + "←": true, + "→": true, + "↑": true, + "Σ": true, + "∘": true, + "𝕊": true, + "√": true, + "□": true, + "⊓": true, + "⊏": true, + "⊑": true, + "⊐": true, + "⊒": true, + "⊔": true, + "𝒮": true, + "⋆": true, + "⋐": true, + "⋐": true, + "⊆": true, + "≻": true, + "⪰": true, + "≽": true, + "≿": true, + "∋": true, + "∑": true, + "⋑": true, + "⊃": true, + "⊇": true, + "⋑": true, + "Þ": true, + "Þ": true, + "™": true, + "Ћ": true, + "Ц": true, + " ": true, + "Τ": true, + "Ť": true, + "Ţ": true, + "Т": true, + "𝔗": true, + "∴": true, + "Θ": true, + "  ": true, + " ": true, + "∼": true, + "≃": true, + "≅": true, + "≈": true, + "𝕋": true, + "⃛": true, + "𝒯": true, + "Ŧ": true, + "Ú": true, + "Ú": true, + "↟": true, + "⥉": true, + "Ў": true, + "Ŭ": true, + "Û": true, + "Û": true, + "У": true, + "Ű": true, + "𝔘": true, + "Ù": true, + "Ù": true, + "Ū": true, + "_": true, + "⏟": true, + "⎵": true, + "⏝": true, + "⋃": true, + "⊎": true, + "Ų": true, + "𝕌": true, + "↑": true, + "⤒": true, + "⇅": true, + "↕": true, + "⥮": true, + "⊥": true, + "↥": true, + "⇑": true, + "⇕": true, + "↖": true, + "↗": true, + "ϒ": true, + "Υ": true, + "Ů": true, + "𝒰": true, + "Ũ": true, + "Ü": true, + "Ü": true, + "⊫": true, + "⫫": true, + "В": true, + "⊩": true, + "⫦": true, + "⋁": true, + "‖": true, + "‖": true, + "∣": true, + "|": true, + "❘": true, + "≀": true, + " ": true, + "𝔙": true, + "𝕍": true, + "𝒱": true, + "⊪": true, + "Ŵ": true, + "⋀": true, + "𝔚": true, + "𝕎": true, + "𝒲": true, + "𝔛": true, + "Ξ": true, + "𝕏": true, + "𝒳": true, + "Я": true, + "Ї": true, + "Ю": true, + "Ý": true, + "Ý": true, + "Ŷ": true, + "Ы": true, + "𝔜": true, + "𝕐": true, + "𝒴": true, + "Ÿ": true, + "Ж": true, + "Ź": true, + "Ž": true, + "З": true, + "Ż": true, + "​": true, + "Ζ": true, + "ℨ": true, + "ℤ": true, + "𝒵": true, + "á": true, + "á": true, + "ă": true, + "∾": true, + "∾̳": true, + "∿": true, + "â": true, + "â": true, + "´": true, + "´": true, + "а": true, + "æ": true, + "æ": true, + "⁡": true, + "𝔞": true, + "à": true, + "à": true, + "ℵ": true, + "ℵ": true, + "α": true, + "ā": true, + "⨿": true, + "&": true, + "&": true, + "∧": true, + "⩕": true, + "⩜": true, + "⩘": true, + "⩚": true, + "∠": true, + "⦤": true, + "∠": true, + "∡": true, + "⦨": true, + "⦩": true, + "⦪": true, + "⦫": true, + "⦬": true, + "⦭": true, + "⦮": true, + "⦯": true, + "∟": true, + "⊾": true, + "⦝": true, + "∢": true, + "Å": true, + "⍼": true, + "ą": true, + "𝕒": true, + "≈": true, + "⩰": true, + "⩯": true, + "≊": true, + "≋": true, + "'": true, + "≈": true, + "≊": true, + "å": true, + "å": true, + "𝒶": true, + "*": true, + "≈": true, + "≍": true, + "ã": true, + "ã": true, + "ä": true, + "ä": true, + "∳": true, + "⨑": true, + "⫭": true, + "≌": true, + "϶": true, + "‵": true, + "∽": true, + "⋍": true, + "⊽": true, + "⌅": true, + "⌅": true, + "⎵": true, + "⎶": true, + "≌": true, + "б": true, + "„": true, + "∵": true, + "∵": true, + "⦰": true, + "϶": true, + "ℬ": true, + "β": true, + "ℶ": true, + "≬": true, + "𝔟": true, + "⋂": true, + "◯": true, + "⋃": true, + "⨀": true, + "⨁": true, + "⨂": true, + "⨆": true, + "★": true, + "▽": true, + "△": true, + "⨄": true, + "⋁": true, + "⋀": true, + "⤍": true, + "⧫": true, + "▪": true, + "▴": true, + "▾": true, + "◂": true, + "▸": true, + "␣": true, + "▒": true, + "░": true, + "▓": true, + "█": true, + "=⃥": true, + "≡⃥": true, + "⌐": true, + "𝕓": true, + "⊥": true, + "⊥": true, + "⋈": true, + "╗": true, + "╔": true, + "╖": true, + "╓": true, + "═": true, + "╦": true, + "╩": true, + "╤": true, + "╧": true, + "╝": true, + "╚": true, + "╜": true, + "╙": true, + "║": true, + "╬": true, + "╣": true, + "╠": true, + "╫": true, + "╢": true, + "╟": true, + "⧉": true, + "╕": true, + "╒": true, + "┐": true, + "┌": true, + "─": true, + "╥": true, + "╨": true, + "┬": true, + "┴": true, + "⊟": true, + "⊞": true, + "⊠": true, + "╛": true, + "╘": true, + "┘": true, + "└": true, + "│": true, + "╪": true, + "╡": true, + "╞": true, + "┼": true, + "┤": true, + "├": true, + "‵": true, + "˘": true, + "¦": true, + "¦": true, + "𝒷": true, + "⁏": true, + "∽": true, + "⋍": true, + "\": true, + "⧅": true, + "⟈": true, + "•": true, + "•": true, + "≎": true, + "⪮": true, + "≏": true, + "≏": true, + "ć": true, + "∩": true, + "⩄": true, + "⩉": true, + "⩋": true, + "⩇": true, + "⩀": true, + "∩︀": true, + "⁁": true, + "ˇ": true, + "⩍": true, + "č": true, + "ç": true, + "ç": true, + "ĉ": true, + "⩌": true, + "⩐": true, + "ċ": true, + "¸": true, + "¸": true, + "⦲": true, + "¢": true, + "¢": true, + "·": true, + "𝔠": true, + "ч": true, + "✓": true, + "✓": true, + "χ": true, + "○": true, + "⧃": true, + "ˆ": true, + "≗": true, + "↺": true, + "↻": true, + "®": true, + "Ⓢ": true, + "⊛": true, + "⊚": true, + "⊝": true, + "≗": true, + "⨐": true, + "⫯": true, + "⧂": true, + "♣": true, + "♣": true, + ":": true, + "≔": true, + "≔": true, + ",": true, + "@": true, + "∁": true, + "∘": true, + "∁": true, + "ℂ": true, + "≅": true, + "⩭": true, + "∮": true, + "𝕔": true, + "∐": true, + "©": true, + "©": true, + "℗": true, + "↵": true, + "✗": true, + "𝒸": true, + "⫏": true, + "⫑": true, + "⫐": true, + "⫒": true, + "⋯": true, + "⤸": true, + "⤵": true, + "⋞": true, + "⋟": true, + "↶": true, + "⤽": true, + "∪": true, + "⩈": true, + "⩆": true, + "⩊": true, + "⊍": true, + "⩅": true, + "∪︀": true, + "↷": true, + "⤼": true, + "⋞": true, + "⋟": true, + "⋎": true, + "⋏": true, + "¤": true, + "¤": true, + "↶": true, + "↷": true, + "⋎": true, + "⋏": true, + "∲": true, + "∱": true, + "⌭": true, + "⇓": true, + "⥥": true, + "†": true, + "ℸ": true, + "↓": true, + "‐": true, + "⊣": true, + "⤏": true, + "˝": true, + "ď": true, + "д": true, + "ⅆ": true, + "‡": true, + "⇊": true, + "⩷": true, + "°": true, + "°": true, + "δ": true, + "⦱": true, + "⥿": true, + "𝔡": true, + "⇃": true, + "⇂": true, + "⋄": true, + "⋄": true, + "♦": true, + "♦": true, + "¨": true, + "ϝ": true, + "⋲": true, + "÷": true, + "÷": true, + "÷": true, + "⋇": true, + "⋇": true, + "ђ": true, + "⌞": true, + "⌍": true, + "$": true, + "𝕕": true, + "˙": true, + "≐": true, + "≑": true, + "∸": true, + "∔": true, + "⊡": true, + "⌆": true, + "↓": true, + "⇊": true, + "⇃": true, + "⇂": true, + "⤐": true, + "⌟": true, + "⌌": true, + "𝒹": true, + "ѕ": true, + "⧶": true, + "đ": true, + "⋱": true, + "▿": true, + "▾": true, + "⇵": true, + "⥯": true, + "⦦": true, + "џ": true, + "⟿": true, + "⩷": true, + "≑": true, + "é": true, + "é": true, + "⩮": true, + "ě": true, + "≖": true, + "ê": true, + "ê": true, + "≕": true, + "э": true, + "ė": true, + "ⅇ": true, + "≒": true, + "𝔢": true, + "⪚": true, + "è": true, + "è": true, + "⪖": true, + "⪘": true, + "⪙": true, + "⏧": true, + "ℓ": true, + "⪕": true, + "⪗": true, + "ē": true, + "∅": true, + "∅": true, + "∅": true, + " ": true, + " ": true, + " ": true, + "ŋ": true, + " ": true, + "ę": true, + "𝕖": true, + "⋕": true, + "⧣": true, + "⩱": true, + "ε": true, + "ε": true, + "ϵ": true, + "≖": true, + "≕": true, + "≂": true, + "⪖": true, + "⪕": true, + "=": true, + "≟": true, + "≡": true, + "⩸": true, + "⧥": true, + "≓": true, + "⥱": true, + "ℯ": true, + "≐": true, + "≂": true, + "η": true, + "ð": true, + "ð": true, + "ë": true, + "ë": true, + "€": true, + "!": true, + "∃": true, + "ℰ": true, + "ⅇ": true, + "≒": true, + "ф": true, + "♀": true, + "ffi": true, + "ff": true, + "ffl": true, + "𝔣": true, + "fi": true, + "fj": true, + "♭": true, + "fl": true, + "▱": true, + "ƒ": true, + "𝕗": true, + "∀": true, + "⋔": true, + "⫙": true, + "⨍": true, + "½": true, + "½": true, + "⅓": true, + "¼": true, + "¼": true, + "⅕": true, + "⅙": true, + "⅛": true, + "⅔": true, + "⅖": true, + "¾": true, + "¾": true, + "⅗": true, + "⅜": true, + "⅘": true, + "⅚": true, + "⅝": true, + "⅞": true, + "⁄": true, + "⌢": true, + "𝒻": true, + "≧": true, + "⪌": true, + "ǵ": true, + "γ": true, + "ϝ": true, + "⪆": true, + "ğ": true, + "ĝ": true, + "г": true, + "ġ": true, + "≥": true, + "⋛": true, + "≥": true, + "≧": true, + "⩾": true, + "⩾": true, + "⪩": true, + "⪀": true, + "⪂": true, + "⪄": true, + "⋛︀": true, + "⪔": true, + "𝔤": true, + "≫": true, + "⋙": true, + "ℷ": true, + "ѓ": true, + "≷": true, + "⪒": true, + "⪥": true, + "⪤": true, + "≩": true, + "⪊": true, + "⪊": true, + "⪈": true, + "⪈": true, + "≩": true, + "⋧": true, + "𝕘": true, + "`": true, + "ℊ": true, + "≳": true, + "⪎": true, + "⪐": true, + ">": true, + ">": true, + "⪧": true, + "⩺": true, + "⋗": true, + "⦕": true, + "⩼": true, + "⪆": true, + "⥸": true, + "⋗": true, + "⋛": true, + "⪌": true, + "≷": true, + "≳": true, + "≩︀": true, + "≩︀": true, + "⇔": true, + " ": true, + "½": true, + "ℋ": true, + "ъ": true, + "↔": true, + "⥈": true, + "↭": true, + "ℏ": true, + "ĥ": true, + "♥": true, + "♥": true, + "…": true, + "⊹": true, + "𝔥": true, + "⤥": true, + "⤦": true, + "⇿": true, + "∻": true, + "↩": true, + "↪": true, + "𝕙": true, + "―": true, + "𝒽": true, + "ℏ": true, + "ħ": true, + "⁃": true, + "‐": true, + "í": true, + "í": true, + "⁣": true, + "î": true, + "î": true, + "и": true, + "е": true, + "¡": true, + "¡": true, + "⇔": true, + "𝔦": true, + "ì": true, + "ì": true, + "ⅈ": true, + "⨌": true, + "∭": true, + "⧜": true, + "℩": true, + "ij": true, + "ī": true, + "ℑ": true, + "ℐ": true, + "ℑ": true, + "ı": true, + "⊷": true, + "Ƶ": true, + "∈": true, + "℅": true, + "∞": true, + "⧝": true, + "ı": true, + "∫": true, + "⊺": true, + "ℤ": true, + "⊺": true, + "⨗": true, + "⨼": true, + "ё": true, + "į": true, + "𝕚": true, + "ι": true, + "⨼": true, + "¿": true, + "¿": true, + "𝒾": true, + "∈": true, + "⋹": true, + "⋵": true, + "⋴": true, + "⋳": true, + "∈": true, + "⁢": true, + "ĩ": true, + "і": true, + "ï": true, + "ï": true, + "ĵ": true, + "й": true, + "𝔧": true, + "ȷ": true, + "𝕛": true, + "𝒿": true, + "ј": true, + "є": true, + "κ": true, + "ϰ": true, + "ķ": true, + "к": true, + "𝔨": true, + "ĸ": true, + "х": true, + "ќ": true, + "𝕜": true, + "𝓀": true, + "⇚": true, + "⇐": true, + "⤛": true, + "⤎": true, + "≦": true, + "⪋": true, + "⥢": true, + "ĺ": true, + "⦴": true, + "ℒ": true, + "λ": true, + "⟨": true, + "⦑": true, + "⟨": true, + "⪅": true, + "«": true, + "«": true, + "←": true, + "⇤": true, + "⤟": true, + "⤝": true, + "↩": true, + "↫": true, + "⤹": true, + "⥳": true, + "↢": true, + "⪫": true, + "⤙": true, + "⪭": true, + "⪭︀": true, + "⤌": true, + "❲": true, + "{": true, + "[": true, + "⦋": true, + "⦏": true, + "⦍": true, + "ľ": true, + "ļ": true, + "⌈": true, + "{": true, + "л": true, + "⤶": true, + "“": true, + "„": true, + "⥧": true, + "⥋": true, + "↲": true, + "≤": true, + "←": true, + "↢": true, + "↽": true, + "↼": true, + "⇇": true, + "↔": true, + "⇆": true, + "⇋": true, + "↭": true, + "⋋": true, + "⋚": true, + "≤": true, + "≦": true, + "⩽": true, + "⩽": true, + "⪨": true, + "⩿": true, + "⪁": true, + "⪃": true, + "⋚︀": true, + "⪓": true, + "⪅": true, + "⋖": true, + "⋚": true, + "⪋": true, + "≶": true, + "≲": true, + "⥼": true, + "⌊": true, + "𝔩": true, + "≶": true, + "⪑": true, + "↽": true, + "↼": true, + "⥪": true, + "▄": true, + "љ": true, + "≪": true, + "⇇": true, + "⌞": true, + "⥫": true, + "◺": true, + "ŀ": true, + "⎰": true, + "⎰": true, + "≨": true, + "⪉": true, + "⪉": true, + "⪇": true, + "⪇": true, + "≨": true, + "⋦": true, + "⟬": true, + "⇽": true, + "⟦": true, + "⟵": true, + "⟷": true, + "⟼": true, + "⟶": true, + "↫": true, + "↬": true, + "⦅": true, + "𝕝": true, + "⨭": true, + "⨴": true, + "∗": true, + "_": true, + "◊": true, + "◊": true, + "⧫": true, + "(": true, + "⦓": true, + "⇆": true, + "⌟": true, + "⇋": true, + "⥭": true, + "‎": true, + "⊿": true, + "‹": true, + "𝓁": true, + "↰": true, + "≲": true, + "⪍": true, + "⪏": true, + "[": true, + "‘": true, + "‚": true, + "ł": true, + "<": true, + "<": true, + "⪦": true, + "⩹": true, + "⋖": true, + "⋋": true, + "⋉": true, + "⥶": true, + "⩻": true, + "⦖": true, + "◃": true, + "⊴": true, + "◂": true, + "⥊": true, + "⥦": true, + "≨︀": true, + "≨︀": true, + "∺": true, + "¯": true, + "¯": true, + "♂": true, + "✠": true, + "✠": true, + "↦": true, + "↦": true, + "↧": true, + "↤": true, + "↥": true, + "▮": true, + "⨩": true, + "м": true, + "—": true, + "∡": true, + "𝔪": true, + "℧": true, + "µ": true, + "µ": true, + "∣": true, + "*": true, + "⫰": true, + "·": true, + "·": true, + "−": true, + "⊟": true, + "∸": true, + "⨪": true, + "⫛": true, + "…": true, + "∓": true, + "⊧": true, + "𝕞": true, + "∓": true, + "𝓂": true, + "∾": true, + "μ": true, + "⊸": true, + "⊸": true, + "⋙̸": true, + "≫⃒": true, + "≫̸": true, + "⇍": true, + "⇎": true, + "⋘̸": true, + "≪⃒": true, + "≪̸": true, + "⇏": true, + "⊯": true, + "⊮": true, + "∇": true, + "ń": true, + "∠⃒": true, + "≉": true, + "⩰̸": true, + "≋̸": true, + "ʼn": true, + "≉": true, + "♮": true, + "♮": true, + "ℕ": true, + " ": true, + " ": true, + "≎̸": true, + "≏̸": true, + "⩃": true, + "ň": true, + "ņ": true, + "≇": true, + "⩭̸": true, + "⩂": true, + "н": true, + "–": true, + "≠": true, + "⇗": true, + "⤤": true, + "↗": true, + "↗": true, + "≐̸": true, + "≢": true, + "⤨": true, + "≂̸": true, + "∄": true, + "∄": true, + "𝔫": true, + "≧̸": true, + "≱": true, + "≱": true, + "≧̸": true, + "⩾̸": true, + "⩾̸": true, + "≵": true, + "≯": true, + "≯": true, + "⇎": true, + "↮": true, + "⫲": true, + "∋": true, + "⋼": true, + "⋺": true, + "∋": true, + "њ": true, + "⇍": true, + "≦̸": true, + "↚": true, + "‥": true, + "≰": true, + "↚": true, + "↮": true, + "≰": true, + "≦̸": true, + "⩽̸": true, + "⩽̸": true, + "≮": true, + "≴": true, + "≮": true, + "⋪": true, + "⋬": true, + "∤": true, + "𝕟": true, + "¬": true, + "¬": true, + "∉": true, + "⋹̸": true, + "⋵̸": true, + "∉": true, + "⋷": true, + "⋶": true, + "∌": true, + "∌": true, + "⋾": true, + "⋽": true, + "∦": true, + "∦": true, + "⫽⃥": true, + "∂̸": true, + "⨔": true, + "⊀": true, + "⋠": true, + "⪯̸": true, + "⊀": true, + "⪯̸": true, + "⇏": true, + "↛": true, + "⤳̸": true, + "↝̸": true, + "↛": true, + "⋫": true, + "⋭": true, + "⊁": true, + "⋡": true, + "⪰̸": true, + "𝓃": true, + "∤": true, + "∦": true, + "≁": true, + "≄": true, + "≄": true, + "∤": true, + "∦": true, + "⋢": true, + "⋣": true, + "⊄": true, + "⫅̸": true, + "⊈": true, + "⊂⃒": true, + "⊈": true, + "⫅̸": true, + "⊁": true, + "⪰̸": true, + "⊅": true, + "⫆̸": true, + "⊉": true, + "⊃⃒": true, + "⊉": true, + "⫆̸": true, + "≹": true, + "ñ": true, + "ñ": true, + "≸": true, + "⋪": true, + "⋬": true, + "⋫": true, + "⋭": true, + "ν": true, + "#": true, + "№": true, + " ": true, + "⊭": true, + "⤄": true, + "≍⃒": true, + "⊬": true, + "≥⃒": true, + ">⃒": true, + "⧞": true, + "⤂": true, + "≤⃒": true, + "<⃒": true, + "⊴⃒": true, + "⤃": true, + "⊵⃒": true, + "∼⃒": true, + "⇖": true, + "⤣": true, + "↖": true, + "↖": true, + "⤧": true, + "Ⓢ": true, + "ó": true, + "ó": true, + "⊛": true, + "⊚": true, + "ô": true, + "ô": true, + "о": true, + "⊝": true, + "ő": true, + "⨸": true, + "⊙": true, + "⦼": true, + "œ": true, + "⦿": true, + "𝔬": true, + "˛": true, + "ò": true, + "ò": true, + "⧁": true, + "⦵": true, + "Ω": true, + "∮": true, + "↺": true, + "⦾": true, + "⦻": true, + "‾": true, + "⧀": true, + "ō": true, + "ω": true, + "ο": true, + "⦶": true, + "⊖": true, + "𝕠": true, + "⦷": true, + "⦹": true, + "⊕": true, + "∨": true, + "↻": true, + "⩝": true, + "ℴ": true, + "ℴ": true, + "ª": true, + "ª": true, + "º": true, + "º": true, + "⊶": true, + "⩖": true, + "⩗": true, + "⩛": true, + "ℴ": true, + "ø": true, + "ø": true, + "⊘": true, + "õ": true, + "õ": true, + "⊗": true, + "⨶": true, + "ö": true, + "ö": true, + "⌽": true, + "∥": true, + "¶": true, + "¶": true, + "∥": true, + "⫳": true, + "⫽": true, + "∂": true, + "п": true, + "%": true, + ".": true, + "‰": true, + "⊥": true, + "‱": true, + "𝔭": true, + "φ": true, + "ϕ": true, + "ℳ": true, + "☎": true, + "π": true, + "⋔": true, + "ϖ": true, + "ℏ": true, + "ℎ": true, + "ℏ": true, + "+": true, + "⨣": true, + "⊞": true, + "⨢": true, + "∔": true, + "⨥": true, + "⩲": true, + "±": true, + "±": true, + "⨦": true, + "⨧": true, + "±": true, + "⨕": true, + "𝕡": true, + "£": true, + "£": true, + "≺": true, + "⪳": true, + "⪷": true, + "≼": true, + "⪯": true, + "≺": true, + "⪷": true, + "≼": true, + "⪯": true, + "⪹": true, + "⪵": true, + "⋨": true, + "≾": true, + "′": true, + "ℙ": true, + "⪵": true, + "⪹": true, + "⋨": true, + "∏": true, + "⌮": true, + "⌒": true, + "⌓": true, + "∝": true, + "∝": true, + "≾": true, + "⊰": true, + "𝓅": true, + "ψ": true, + " ": true, + "𝔮": true, + "⨌": true, + "𝕢": true, + "⁗": true, + "𝓆": true, + "ℍ": true, + "⨖": true, + "?": true, + "≟": true, + """: true, + """: true, + "⇛": true, + "⇒": true, + "⤜": true, + "⤏": true, + "⥤": true, + "∽̱": true, + "ŕ": true, + "√": true, + "⦳": true, + "⟩": true, + "⦒": true, + "⦥": true, + "⟩": true, + "»": true, + "»": true, + "→": true, + "⥵": true, + "⇥": true, + "⤠": true, + "⤳": true, + "⤞": true, + "↪": true, + "↬": true, + "⥅": true, + "⥴": true, + "↣": true, + "↝": true, + "⤚": true, + "∶": true, + "ℚ": true, + "⤍": true, + "❳": true, + "}": true, + "]": true, + "⦌": true, + "⦎": true, + "⦐": true, + "ř": true, + "ŗ": true, + "⌉": true, + "}": true, + "р": true, + "⤷": true, + "⥩": true, + "”": true, + "”": true, + "↳": true, + "ℜ": true, + "ℛ": true, + "ℜ": true, + "ℝ": true, + "▭": true, + "®": true, + "®": true, + "⥽": true, + "⌋": true, + "𝔯": true, + "⇁": true, + "⇀": true, + "⥬": true, + "ρ": true, + "ϱ": true, + "→": true, + "↣": true, + "⇁": true, + "⇀": true, + "⇄": true, + "⇌": true, + "⇉": true, + "↝": true, + "⋌": true, + "˚": true, + "≓": true, + "⇄": true, + "⇌": true, + "‏": true, + "⎱": true, + "⎱": true, + "⫮": true, + "⟭": true, + "⇾": true, + "⟧": true, + "⦆": true, + "𝕣": true, + "⨮": true, + "⨵": true, + ")": true, + "⦔": true, + "⨒": true, + "⇉": true, + "›": true, + "𝓇": true, + "↱": true, + "]": true, + "’": true, + "’": true, + "⋌": true, + "⋊": true, + "▹": true, + "⊵": true, + "▸": true, + "⧎": true, + "⥨": true, + "℞": true, + "ś": true, + "‚": true, + "≻": true, + "⪴": true, + "⪸": true, + "š": true, + "≽": true, + "⪰": true, + "ş": true, + "ŝ": true, + "⪶": true, + "⪺": true, + "⋩": true, + "⨓": true, + "≿": true, + "с": true, + "⋅": true, + "⊡": true, + "⩦": true, + "⇘": true, + "⤥": true, + "↘": true, + "↘": true, + "§": true, + "§": true, + ";": true, + "⤩": true, + "∖": true, + "∖": true, + "✶": true, + "𝔰": true, + "⌢": true, + "♯": true, + "щ": true, + "ш": true, + "∣": true, + "∥": true, + "­": true, + "­": true, + "σ": true, + "ς": true, + "ς": true, + "∼": true, + "⩪": true, + "≃": true, + "≃": true, + "⪞": true, + "⪠": true, + "⪝": true, + "⪟": true, + "≆": true, + "⨤": true, + "⥲": true, + "←": true, + "∖": true, + "⨳": true, + "⧤": true, + "∣": true, + "⌣": true, + "⪪": true, + "⪬": true, + "⪬︀": true, + "ь": true, + "/": true, + "⧄": true, + "⌿": true, + "𝕤": true, + "♠": true, + "♠": true, + "∥": true, + "⊓": true, + "⊓︀": true, + "⊔": true, + "⊔︀": true, + "⊏": true, + "⊑": true, + "⊏": true, + "⊑": true, + "⊐": true, + "⊒": true, + "⊐": true, + "⊒": true, + "□": true, + "□": true, + "▪": true, + "▪": true, + "→": true, + "𝓈": true, + "∖": true, + "⌣": true, + "⋆": true, + "☆": true, + "★": true, + "ϵ": true, + "ϕ": true, + "¯": true, + "⊂": true, + "⫅": true, + "⪽": true, + "⊆": true, + "⫃": true, + "⫁": true, + "⫋": true, + "⊊": true, + "⪿": true, + "⥹": true, + "⊂": true, + "⊆": true, + "⫅": true, + "⊊": true, + "⫋": true, + "⫇": true, + "⫕": true, + "⫓": true, + "≻": true, + "⪸": true, + "≽": true, + "⪰": true, + "⪺": true, + "⪶": true, + "⋩": true, + "≿": true, + "∑": true, + "♪": true, + "¹": true, + "¹": true, + "²": true, + "²": true, + "³": true, + "³": true, + "⊃": true, + "⫆": true, + "⪾": true, + "⫘": true, + "⊇": true, + "⫄": true, + "⟉": true, + "⫗": true, + "⥻": true, + "⫂": true, + "⫌": true, + "⊋": true, + "⫀": true, + "⊃": true, + "⊇": true, + "⫆": true, + "⊋": true, + "⫌": true, + "⫈": true, + "⫔": true, + "⫖": true, + "⇙": true, + "⤦": true, + "↙": true, + "↙": true, + "⤪": true, + "ß": true, + "ß": true, + "⌖": true, + "τ": true, + "⎴": true, + "ť": true, + "ţ": true, + "т": true, + "⃛": true, + "⌕": true, + "𝔱": true, + "∴": true, + "∴": true, + "θ": true, + "ϑ": true, + "ϑ": true, + "≈": true, + "∼": true, + " ": true, + "≈": true, + "∼": true, + "þ": true, + "þ": true, + "˜": true, + "×": true, + "×": true, + "⊠": true, + "⨱": true, + "⨰": true, + "∭": true, + "⤨": true, + "⊤": true, + "⌶": true, + "⫱": true, + "𝕥": true, + "⫚": true, + "⤩": true, + "‴": true, + "™": true, + "▵": true, + "▿": true, + "◃": true, + "⊴": true, + "≜": true, + "▹": true, + "⊵": true, + "◬": true, + "≜": true, + "⨺": true, + "⨹": true, + "⧍": true, + "⨻": true, + "⏢": true, + "𝓉": true, + "ц": true, + "ћ": true, + "ŧ": true, + "≬": true, + "↞": true, + "↠": true, + "⇑": true, + "⥣": true, + "ú": true, + "ú": true, + "↑": true, + "ў": true, + "ŭ": true, + "û": true, + "û": true, + "у": true, + "⇅": true, + "ű": true, + "⥮": true, + "⥾": true, + "𝔲": true, + "ù": true, + "ù": true, + "↿": true, + "↾": true, + "▀": true, + "⌜": true, + "⌜": true, + "⌏": true, + "◸": true, + "ū": true, + "¨": true, + "¨": true, + "ų": true, + "𝕦": true, + "↑": true, + "↕": true, + "↿": true, + "↾": true, + "⊎": true, + "υ": true, + "ϒ": true, + "υ": true, + "⇈": true, + "⌝": true, + "⌝": true, + "⌎": true, + "ů": true, + "◹": true, + "𝓊": true, + "⋰": true, + "ũ": true, + "▵": true, + "▴": true, + "⇈": true, + "ü": true, + "ü": true, + "⦧": true, + "⇕": true, + "⫨": true, + "⫩": true, + "⊨": true, + "⦜": true, + "ϵ": true, + "ϰ": true, + "∅": true, + "ϕ": true, + "ϖ": true, + "∝": true, + "↕": true, + "ϱ": true, + "ς": true, + "⊊︀": true, + "⫋︀": true, + "⊋︀": true, + "⫌︀": true, + "ϑ": true, + "⊲": true, + "⊳": true, + "в": true, + "⊢": true, + "∨": true, + "⊻": true, + "≚": true, + "⋮": true, + "|": true, + "|": true, + "𝔳": true, + "⊲": true, + "⊂⃒": true, + "⊃⃒": true, + "𝕧": true, + "∝": true, + "⊳": true, + "𝓋": true, + "⫋︀": true, + "⊊︀": true, + "⫌︀": true, + "⊋︀": true, + "⦚": true, + "ŵ": true, + "⩟": true, + "∧": true, + "≙": true, + "℘": true, + "𝔴": true, + "𝕨": true, + "℘": true, + "≀": true, + "≀": true, + "𝓌": true, + "⋂": true, + "◯": true, + "⋃": true, + "▽": true, + "𝔵": true, + "⟺": true, + "⟷": true, + "ξ": true, + "⟸": true, + "⟵": true, + "⟼": true, + "⋻": true, + "⨀": true, + "𝕩": true, + "⨁": true, + "⨂": true, + "⟹": true, + "⟶": true, + "𝓍": true, + "⨆": true, + "⨄": true, + "△": true, + "⋁": true, + "⋀": true, + "ý": true, + "ý": true, + "я": true, + "ŷ": true, + "ы": true, + "¥": true, + "¥": true, + "𝔶": true, + "ї": true, + "𝕪": true, + "𝓎": true, + "ю": true, + "ÿ": true, + "ÿ": true, + "ź": true, + "ž": true, + "з": true, + "ż": true, + "ℨ": true, + "ζ": true, + "𝔷": true, + "ж": true, + "⇝": true, + "𝕫": true, + "𝓏": true, + "‍": true, + "‌": true, +} diff --git a/vendor/github.com/russross/blackfriday/v2/esc.go b/vendor/github.com/russross/blackfriday/v2/esc.go index 6385f27cb6a4..6ab60102c9bf 100644 --- a/vendor/github.com/russross/blackfriday/v2/esc.go +++ b/vendor/github.com/russross/blackfriday/v2/esc.go @@ -13,13 +13,27 @@ var htmlEscaper = [256][]byte{ } func escapeHTML(w io.Writer, s []byte) { + escapeEntities(w, s, false) +} + +func escapeAllHTML(w io.Writer, s []byte) { + escapeEntities(w, s, true) +} + +func escapeEntities(w io.Writer, s []byte, escapeValidEntities bool) { var start, end int for end < len(s) { escSeq := htmlEscaper[s[end]] if escSeq != nil { - w.Write(s[start:end]) - w.Write(escSeq) - start = end + 1 + isEntity, entityEnd := nodeIsEntity(s, end) + if isEntity && !escapeValidEntities { + w.Write(s[start : entityEnd+1]) + start = entityEnd + 1 + } else { + w.Write(s[start:end]) + w.Write(escSeq) + start = end + 1 + } } end++ } @@ -28,6 +42,28 @@ func escapeHTML(w io.Writer, s []byte) { } } +func nodeIsEntity(s []byte, end int) (isEntity bool, endEntityPos int) { + isEntity = false + endEntityPos = end + 1 + + if s[end] == '&' { + for endEntityPos < len(s) { + if s[endEntityPos] == ';' { + if entities[string(s[end:endEntityPos+1])] { + isEntity = true + break + } + } + if !isalnum(s[endEntityPos]) && s[endEntityPos] != '&' && s[endEntityPos] != '#' { + break + } + endEntityPos++ + } + } + + return isEntity, endEntityPos +} + func escLink(w io.Writer, text []byte) { unesc := html.UnescapeString(string(text)) escapeHTML(w, []byte(unesc)) diff --git a/vendor/github.com/russross/blackfriday/v2/html.go b/vendor/github.com/russross/blackfriday/v2/html.go index 284c87184f77..cb4f26e30fd5 100644 --- a/vendor/github.com/russross/blackfriday/v2/html.go +++ b/vendor/github.com/russross/blackfriday/v2/html.go @@ -132,7 +132,10 @@ func NewHTMLRenderer(params HTMLRendererParameters) *HTMLRenderer { } if params.FootnoteReturnLinkContents == "" { - params.FootnoteReturnLinkContents = `[return]` + // U+FE0E is VARIATION SELECTOR-15. + // It suppresses automatic emoji presentation of the preceding + // U+21A9 LEFTWARDS ARROW WITH HOOK on iOS and iPadOS. + params.FootnoteReturnLinkContents = "↩\ufe0e" } return &HTMLRenderer{ @@ -616,7 +619,7 @@ func (r *HTMLRenderer) RenderNode(w io.Writer, node *Node, entering bool) WalkSt } case Code: r.out(w, codeTag) - escapeHTML(w, node.Literal) + escapeAllHTML(w, node.Literal) r.out(w, codeCloseTag) case Document: break @@ -762,7 +765,7 @@ func (r *HTMLRenderer) RenderNode(w io.Writer, node *Node, entering bool) WalkSt r.cr(w) r.out(w, preTag) r.tag(w, codeTag[:len(codeTag)-1], attrs) - escapeHTML(w, node.Literal) + escapeAllHTML(w, node.Literal) r.out(w, codeCloseTag) r.out(w, preCloseTag) if node.Parent.Type != Item { diff --git a/vendor/github.com/russross/blackfriday/v2/inline.go b/vendor/github.com/russross/blackfriday/v2/inline.go index 4ed2907921e0..d45bd941726e 100644 --- a/vendor/github.com/russross/blackfriday/v2/inline.go +++ b/vendor/github.com/russross/blackfriday/v2/inline.go @@ -278,7 +278,7 @@ func link(p *Markdown, data []byte, offset int) (int, *Node) { case data[i] == '\n': textHasNl = true - case data[i-1] == '\\': + case isBackslashEscaped(data, i): continue case data[i] == '[': diff --git a/vendor/github.com/russross/blackfriday/v2/node.go b/vendor/github.com/russross/blackfriday/v2/node.go index 51b9e8c1b538..04e6050ceeae 100644 --- a/vendor/github.com/russross/blackfriday/v2/node.go +++ b/vendor/github.com/russross/blackfriday/v2/node.go @@ -199,7 +199,8 @@ func (n *Node) InsertBefore(sibling *Node) { } } -func (n *Node) isContainer() bool { +// IsContainer returns true if 'n' can contain children. +func (n *Node) IsContainer() bool { switch n.Type { case Document: fallthrough @@ -238,6 +239,11 @@ func (n *Node) isContainer() bool { } } +// IsLeaf returns true if 'n' is a leaf node. +func (n *Node) IsLeaf() bool { + return !n.IsContainer() +} + func (n *Node) canContain(t NodeType) bool { if n.Type == List { return t == Item @@ -309,11 +315,11 @@ func newNodeWalker(root *Node) *nodeWalker { } func (nw *nodeWalker) next() { - if (!nw.current.isContainer() || !nw.entering) && nw.current == nw.root { + if (!nw.current.IsContainer() || !nw.entering) && nw.current == nw.root { nw.current = nil return } - if nw.entering && nw.current.isContainer() { + if nw.entering && nw.current.IsContainer() { if nw.current.FirstChild != nil { nw.current = nw.current.FirstChild nw.entering = true diff --git a/vendor/github.com/shurcooL/sanitized_anchor_name/LICENSE b/vendor/github.com/secure-systems-lab/go-securesystemslib/LICENSE similarity index 94% rename from vendor/github.com/shurcooL/sanitized_anchor_name/LICENSE rename to vendor/github.com/secure-systems-lab/go-securesystemslib/LICENSE index c35c17af9808..e51324f9b5b4 100644 --- a/vendor/github.com/shurcooL/sanitized_anchor_name/LICENSE +++ b/vendor/github.com/secure-systems-lab/go-securesystemslib/LICENSE @@ -1,6 +1,6 @@ -MIT License +The MIT License (MIT) -Copyright (c) 2015 Dmitri Shuralyov +Copyright (c) 2021 NYU Secure Systems Lab Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/in-toto/in-toto-golang/in_toto/canonicaljson.go b/vendor/github.com/secure-systems-lab/go-securesystemslib/cjson/canonicaljson.go similarity index 99% rename from vendor/github.com/in-toto/in-toto-golang/in_toto/canonicaljson.go rename to vendor/github.com/secure-systems-lab/go-securesystemslib/cjson/canonicaljson.go index 961756e5a09b..fb1d5918b282 100644 --- a/vendor/github.com/in-toto/in-toto-golang/in_toto/canonicaljson.go +++ b/vendor/github.com/secure-systems-lab/go-securesystemslib/cjson/canonicaljson.go @@ -1,4 +1,4 @@ -package in_toto +package cjson import ( "bytes" diff --git a/vendor/github.com/in-toto/in-toto-golang/pkg/ssl/sign.go b/vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/sign.go similarity index 70% rename from vendor/github.com/in-toto/in-toto-golang/pkg/ssl/sign.go rename to vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/sign.go index c40e90e8cae0..3dc05a4294e1 100644 --- a/vendor/github.com/in-toto/in-toto-golang/pkg/ssl/sign.go +++ b/vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/sign.go @@ -1,9 +1,8 @@ /* -Package ssl implements the Secure Systems Lab signing-spec (sometimes -abbreviated SSL Siging spec. -https://github.com/secure-systems-lab/signing-spec +Package dsse implements the Dead Simple Signing Envelope (DSSE) +https://github.com/secure-systems-lab/dsse */ -package ssl +package dsse import ( "encoding/base64" @@ -32,6 +31,16 @@ type Envelope struct { Signatures []Signature `json:"signatures"` } +/* +DecodeB64Payload returns the serialized body, decoded +from the envelope's payload field. A flexible +decoder is used, first trying standard base64, then +URL-encoded base64. +*/ +func (e *Envelope) DecodeB64Payload() ([]byte, error) { + return b64Decode(e.Payload) +} + /* Signature represents a generic in-toto signature that contains the identifier of the key which was used to create the signature. @@ -49,7 +58,7 @@ type Signature struct { PAE implementes the DSSE Pre-Authentic Encoding https://github.com/secure-systems-lab/dsse/blob/master/protocol.md#signature-definition */ -func PAE(payloadType, payload string) []byte { +func PAE(payloadType string, payload []byte) []byte { return []byte(fmt.Sprintf("DSSEv1 %d %s %d %s", len(payloadType), payloadType, len(payload), payload)) @@ -68,7 +77,8 @@ using the current algorithm, and the key used (if applicable). For an example see EcdsaSigner in sign_test.go. */ type Signer interface { - Sign(data []byte) ([]byte, string, error) + Sign(data []byte) ([]byte, error) + KeyID() (string, error) } // SignVerifer provides both the signing and verification interface. @@ -80,14 +90,25 @@ type SignVerifier interface { // EnvelopeSigner creates signed Envelopes. type EnvelopeSigner struct { providers []SignVerifier - ev EnvelopeVerifier + ev *EnvelopeVerifier } /* NewEnvelopeSigner creates an EnvelopeSigner that uses 1+ Signer algorithms to sign the data. +Creates a verifier with threshold=1, at least one of the providers must validate signitures successfully. */ func NewEnvelopeSigner(p ...SignVerifier) (*EnvelopeSigner, error) { + return NewMultiEnvelopeSigner(1, p...) +} + +/* +NewMultiEnvelopeSigner creates an EnvelopeSigner that uses 1+ Signer +algorithms to sign the data. +Creates a verifier with threshold. +threashold indicates the amount of providers that must validate the envelope. +*/ +func NewMultiEnvelopeSigner(threshold int, p ...SignVerifier) (*EnvelopeSigner, error) { var providers []SignVerifier for _, sv := range p { @@ -105,18 +126,21 @@ func NewEnvelopeSigner(p ...SignVerifier) (*EnvelopeSigner, error) { evps = append(evps, p.(Verifier)) } + ev, err := NewMultiEnvelopeVerifier(threshold, evps...) + if err != nil { + return nil, err + } + return &EnvelopeSigner{ providers: providers, - ev: EnvelopeVerifier{ - providers: evps, - }, + ev: ev, }, nil } /* -SignPayload signs a payload and payload type according to the SSL signing spec. +SignPayload signs a payload and payload type according to DSSE. Returned is an envelope as defined here: -https://github.com/secure-systems-lab/signing-spec/blob/master/envelope.md +https://github.com/secure-systems-lab/dsse/blob/master/envelope.md One signature will be added for each Signer in the EnvelopeSigner. */ func (es *EnvelopeSigner) SignPayload(payloadType string, body []byte) (*Envelope, error) { @@ -125,13 +149,17 @@ func (es *EnvelopeSigner) SignPayload(payloadType string, body []byte) (*Envelop PayloadType: payloadType, } - paeEnc := PAE(payloadType, string(body)) + paeEnc := PAE(payloadType, body) for _, signer := range es.providers { - sig, keyID, err := signer.Sign(paeEnc) + sig, err := signer.Sign(paeEnc) if err != nil { return nil, err } + keyID, err := signer.KeyID() + if err != nil { + keyID = "" + } e.Signatures = append(e.Signatures, Signature{ KeyID: keyID, @@ -146,14 +174,15 @@ func (es *EnvelopeSigner) SignPayload(payloadType string, body []byte) (*Envelop Verify decodes the payload and verifies the signature. Any domain specific validation such as parsing the decoded body and validating the payload type is left out to the caller. +Verify returns a list of accepted keys each including a keyid, public and signiture of the accepted provider keys. */ -func (es *EnvelopeSigner) Verify(e *Envelope) error { +func (es *EnvelopeSigner) Verify(e *Envelope) ([]AcceptedKey, error) { return es.ev.Verify(e) } /* Both standard and url encoding are allowed: -https://github.com/secure-systems-lab/signing-spec/blob/master/envelope.md +https://github.com/secure-systems-lab/dsse/blob/master/envelope.md */ func b64Decode(s string) ([]byte, error) { b, err := base64.StdEncoding.DecodeString(s) diff --git a/vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/verify.go b/vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/verify.go new file mode 100644 index 000000000000..ead1c32ca80b --- /dev/null +++ b/vendor/github.com/secure-systems-lab/go-securesystemslib/dsse/verify.go @@ -0,0 +1,146 @@ +package dsse + +import ( + "crypto" + "errors" + "fmt" + + "golang.org/x/crypto/ssh" +) + +/* +Verifier verifies a complete message against a signature and key. +If the message was hashed prior to signature generation, the verifier +must perform the same steps. +If KeyID returns successfully, only signature matching the key ID will be verified. +*/ +type Verifier interface { + Verify(data, sig []byte) error + KeyID() (string, error) + Public() crypto.PublicKey +} + +type EnvelopeVerifier struct { + providers []Verifier + threshold int +} + +type AcceptedKey struct { + Public crypto.PublicKey + KeyID string + Sig Signature +} + +func (ev *EnvelopeVerifier) Verify(e *Envelope) ([]AcceptedKey, error) { + if e == nil { + return nil, errors.New("cannot verify a nil envelope") + } + + if len(e.Signatures) == 0 { + return nil, ErrNoSignature + } + + // Decode payload (i.e serialized body) + body, err := e.DecodeB64Payload() + if err != nil { + return nil, err + } + // Generate PAE(payloadtype, serialized body) + paeEnc := PAE(e.PayloadType, body) + + // If *any* signature is found to be incorrect, it is skipped + var acceptedKeys []AcceptedKey + usedKeyids := make(map[string]string) + unverified_providers := ev.providers + for _, s := range e.Signatures { + sig, err := b64Decode(s.Sig) + if err != nil { + return nil, err + } + + // Loop over the providers. + // If provider and signature include key IDs but do not match skip. + // If a provider recognizes the key, we exit + // the loop and use the result. + providers := unverified_providers + for i, v := range providers { + keyID, err := v.KeyID() + + // Verifiers that do not provide a keyid will be generated one using public. + if err != nil || keyID == "" { + keyID, err = SHA256KeyID(v.Public()) + if err != nil { + keyID = "" + } + } + + if s.KeyID != "" && keyID != "" && err == nil && s.KeyID != keyID { + continue + } + + err = v.Verify(paeEnc, sig) + if err != nil { + continue + } + + acceptedKey := AcceptedKey{ + Public: v.Public(), + KeyID: keyID, + Sig: s, + } + unverified_providers = removeIndex(providers, i) + + // See https://github.com/in-toto/in-toto/pull/251 + if _, ok := usedKeyids[keyID]; ok { + fmt.Printf("Found envelope signed by different subkeys of the same main key, Only one of them is counted towards the step threshold, KeyID=%s\n", keyID) + continue + } + + usedKeyids[keyID] = "" + acceptedKeys = append(acceptedKeys, acceptedKey) + break + } + } + + // Sanity if with some reflect magic this happens. + if ev.threshold <= 0 || ev.threshold > len(ev.providers) { + return nil, errors.New("Invalid threshold") + } + + if len(usedKeyids) < ev.threshold { + return acceptedKeys, errors.New(fmt.Sprintf("Accepted signatures do not match threshold, Found: %d, Expected %d", len(acceptedKeys), ev.threshold)) + } + + return acceptedKeys, nil +} + +func NewEnvelopeVerifier(v ...Verifier) (*EnvelopeVerifier, error) { + return NewMultiEnvelopeVerifier(1, v...) +} + +func NewMultiEnvelopeVerifier(threshold int, p ...Verifier) (*EnvelopeVerifier, error) { + + if threshold <= 0 || threshold > len(p) { + return nil, errors.New("Invalid threshold") + } + + ev := EnvelopeVerifier{ + providers: p, + threshold: threshold, + } + return &ev, nil +} + +func SHA256KeyID(pub crypto.PublicKey) (string, error) { + // Generate public key fingerprint + sshpk, err := ssh.NewPublicKey(pub) + if err != nil { + return "", err + } + fingerprint := ssh.FingerprintSHA256(sshpk) + return fingerprint, nil +} + +func removeIndex(v []Verifier, index int) []Verifier { + return append(v[:index], v[index+1:]...) +} diff --git a/vendor/github.com/shibumi/go-pathspec/.gitignore b/vendor/github.com/shibumi/go-pathspec/.gitignore index 836562412fe8..3e32393f1238 100644 --- a/vendor/github.com/shibumi/go-pathspec/.gitignore +++ b/vendor/github.com/shibumi/go-pathspec/.gitignore @@ -21,3 +21,6 @@ _testmain.go *.exe *.test + +# ignore .idea +.idea diff --git a/vendor/github.com/shurcooL/sanitized_anchor_name/.travis.yml b/vendor/github.com/shurcooL/sanitized_anchor_name/.travis.yml deleted file mode 100644 index 93b1fcdb31a2..000000000000 --- a/vendor/github.com/shurcooL/sanitized_anchor_name/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -sudo: false -language: go -go: - - 1.x - - master -matrix: - allow_failures: - - go: master - fast_finish: true -install: - - # Do nothing. This is needed to prevent default install action "go get -t -v ./..." from happening here (we want it to happen inside script step). -script: - - go get -t -v ./... - - diff -u <(echo -n) <(gofmt -d -s .) - - go tool vet . - - go test -v -race ./... diff --git a/vendor/github.com/shurcooL/sanitized_anchor_name/README.md b/vendor/github.com/shurcooL/sanitized_anchor_name/README.md deleted file mode 100644 index 670bf0fe6c79..000000000000 --- a/vendor/github.com/shurcooL/sanitized_anchor_name/README.md +++ /dev/null @@ -1,36 +0,0 @@ -sanitized_anchor_name -===================== - -[![Build Status](https://travis-ci.org/shurcooL/sanitized_anchor_name.svg?branch=master)](https://travis-ci.org/shurcooL/sanitized_anchor_name) [![GoDoc](https://godoc.org/github.com/shurcooL/sanitized_anchor_name?status.svg)](https://godoc.org/github.com/shurcooL/sanitized_anchor_name) - -Package sanitized_anchor_name provides a func to create sanitized anchor names. - -Its logic can be reused by multiple packages to create interoperable anchor names -and links to those anchors. - -At this time, it does not try to ensure that generated anchor names -are unique, that responsibility falls on the caller. - -Installation ------------- - -```bash -go get -u github.com/shurcooL/sanitized_anchor_name -``` - -Example -------- - -```Go -anchorName := sanitized_anchor_name.Create("This is a header") - -fmt.Println(anchorName) - -// Output: -// this-is-a-header -``` - -License -------- - -- [MIT License](LICENSE) diff --git a/vendor/github.com/shurcooL/sanitized_anchor_name/main.go b/vendor/github.com/shurcooL/sanitized_anchor_name/main.go deleted file mode 100644 index 6a77d1243173..000000000000 --- a/vendor/github.com/shurcooL/sanitized_anchor_name/main.go +++ /dev/null @@ -1,29 +0,0 @@ -// Package sanitized_anchor_name provides a func to create sanitized anchor names. -// -// Its logic can be reused by multiple packages to create interoperable anchor names -// and links to those anchors. -// -// At this time, it does not try to ensure that generated anchor names -// are unique, that responsibility falls on the caller. -package sanitized_anchor_name // import "github.com/shurcooL/sanitized_anchor_name" - -import "unicode" - -// Create returns a sanitized anchor name for the given text. -func Create(text string) string { - var anchorName []rune - var futureDash = false - for _, r := range text { - switch { - case unicode.IsLetter(r) || unicode.IsNumber(r): - if futureDash && len(anchorName) > 0 { - anchorName = append(anchorName, '-') - } - futureDash = false - anchorName = append(anchorName, unicode.ToLower(r)) - default: - futureDash = true - } - } - return string(anchorName) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 920fa16066c1..25e8d7a79682 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -363,8 +363,8 @@ github.com/containernetworking/cni/pkg/version ## explicit; go 1.12 github.com/coreos/go-systemd/v22/activation github.com/coreos/go-systemd/v22/daemon -# github.com/cpuguy83/go-md2man/v2 v2.0.0 -## explicit; go 1.12 +# github.com/cpuguy83/go-md2man/v2 v2.0.2 +## explicit; go 1.11 github.com/cpuguy83/go-md2man/v2/md2man # github.com/davecgh/go-spew v1.1.1 ## explicit @@ -518,10 +518,10 @@ github.com/hashicorp/go-retryablehttp # github.com/hashicorp/golang-lru v0.5.4 ## explicit; go 1.12 github.com/hashicorp/golang-lru/simplelru -# github.com/in-toto/in-toto-golang v0.3.3 +# github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add ## explicit; go 1.17 github.com/in-toto/in-toto-golang/in_toto -github.com/in-toto/in-toto-golang/pkg/ssl +github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2 # github.com/jmespath/go-jmespath v0.4.0 ## explicit; go 1.14 github.com/jmespath/go-jmespath @@ -623,18 +623,19 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/russross/blackfriday/v2 v2.0.1 +# github.com/russross/blackfriday/v2 v2.1.0 ## explicit github.com/russross/blackfriday/v2 +# github.com/secure-systems-lab/go-securesystemslib v0.4.0 +## explicit; go 1.17 +github.com/secure-systems-lab/go-securesystemslib/cjson +github.com/secure-systems-lab/go-securesystemslib/dsse # github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 ## explicit github.com/serialx/hashring -# github.com/shibumi/go-pathspec v1.2.0 -## explicit; go 1.14 +# github.com/shibumi/go-pathspec v1.3.0 +## explicit; go 1.17 github.com/shibumi/go-pathspec -# github.com/shurcooL/sanitized_anchor_name v1.0.0 -## explicit -github.com/shurcooL/sanitized_anchor_name # github.com/sirupsen/logrus v1.9.0 ## explicit; go 1.13 github.com/sirupsen/logrus From 5d5a6b93e09c466aece20f9835b07138a95660fe Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 25 Aug 2022 21:20:15 -0700 Subject: [PATCH 2/9] Add slsa provenance attestation unless disabled Signed-off-by: Tonis Tiigi --- cmd/buildctl/build.go | 15 ++- control/control.go | 4 + exporter/containerimage/writer.go | 72 ++++++++----- solver/jobs.go | 18 ++-- solver/llbsolver/proc/provenance.go | 118 ++++++++++++++++++++ solver/llbsolver/solver.go | 16 +-- solver/result/attestation.go | 3 +- solver/result/result.go | 2 +- util/buildinfo/buildinfo.go | 2 +- util/provenance/buildinfo.go | 161 ++++++++++++++++++++++++++++ 10 files changed, 366 insertions(+), 45 deletions(-) create mode 100644 solver/llbsolver/proc/provenance.go create mode 100644 util/provenance/buildinfo.go diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index bd78f8ff7b3d..223bc3076673 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -277,13 +277,26 @@ func buildAction(clicontext *cli.Context) error { close(w.Status()) } }() + + solveAttr := map[string]string{} + frontendAttr := map[string]string{} + for k, v := range solveOpt.FrontendAttrs { + if strings.HasPrefix(k, "attest:") || strings.HasPrefix(k, "build-arg:BUILDKIT_ATTEST_") { + frontendAttr[k] = v + } else { + solveAttr[k] = v + } + } + sreq := gateway.SolveRequest{ Frontend: solveOpt.Frontend, - FrontendOpt: solveOpt.FrontendAttrs, + FrontendOpt: solveAttr, } if def != nil { sreq.Definition = def.ToPB() } + solveOpt.Frontend = "" + solveOpt.FrontendAttrs = frontendAttr resp, err := c.Build(ctx, solveOpt, "buildctl", func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { _, isSubRequest := sreq.FrontendOpt["requestid"] diff --git a/control/control.go b/control/control.go index 718dfd282f74..1fb35eb5a909 100644 --- a/control/control.go +++ b/control/control.go @@ -343,6 +343,10 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.ForceRefsProcessor, proc.SBOMProcessor(ref.String())) } + if attrs, ok := attests["provenance"]; ok { + procs = append(procs, proc.ForceRefsProcessor, proc.ProvenanceProcessor(attrs)) + } + resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{ Frontend: req.Frontend, Definition: req.Definition, diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index d78a982aa148..ebf930ebbd97 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -131,7 +131,11 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session attestCount := 0 for _, attests := range inp.Attestations { - attestCount += len(attests) + for _, attest := range attests { + if attest.ContentFunc == nil { + attestCount++ + } + } } if count := attestCount + len(p.Platforms); count != len(inp.Refs) { return nil, errors.Errorf("number of required refs (%d) does not match number of references (%d)", count, len(inp.Refs)) @@ -306,35 +310,53 @@ func (ic *ImageWriter) extractAttestations(ctx context.Context, opts *ImageCommi for i, att := range attestations { i, att := i, att eg.Go(func() error { - ref, ok := refs[att.Ref] - if !ok { - return errors.Errorf("key %s not found in refs map", att.Ref) - } - mount, err := ref.Mount(ctx, true, s) - if err != nil { - return err - } - lm := snapshot.LocalMounter(mount) - src, err := lm.Mount() - if err != nil { - return err - } - defer lm.Unmount() - - switch att.Kind { - case gatewaypb.AttestationKindInToto: - p, err := fs.RootPath(src, att.Path) + var data []byte + var err error + var dir string + if att.ContentFunc != nil { + data, err = att.ContentFunc() if err != nil { return err } - data, err := os.ReadFile(p) + } else { + ref, ok := refs[att.Ref] + if !ok { + return errors.Errorf("key %s not found in refs map", att.Ref) + } + mount, err := ref.Mount(ctx, true, s) if err != nil { - return errors.Wrap(err, "cannot read in-toto attestation") + return err } - if len(data) == 0 { - data = nil + lm := snapshot.LocalMounter(mount) + src, err := lm.Mount() + if err != nil { + return err } + defer lm.Unmount() + switch att.Kind { + case gatewaypb.AttestationKindInToto: + p, err := fs.RootPath(src, att.Path) + if err != nil { + return err + } + data, err = os.ReadFile(p) + if err != nil { + return errors.Wrap(err, "cannot read in-toto attestation") + } + if len(data) == 0 { + data = nil + } + case gatewaypb.AttestationKindBundle: + dir, err = fs.RootPath(src, att.Path) + if err != nil { + return err + } + } + } + + switch att.Kind { + case gatewaypb.AttestationKindInToto: var subjects []intoto.Subject if len(att.InToto.Subjects) == 0 { att.InToto.Subjects = []result.InTotoSubject{{ @@ -380,10 +402,6 @@ func (ic *ImageWriter) extractAttestations(ctx context.Context, opts *ImageCommi } statements[i] = append(statements[i], stmt) case gatewaypb.AttestationKindBundle: - dir, err := fs.RootPath(src, att.Path) - if err != nil { - return err - } entries, err := os.ReadDir(dir) if err != nil { return err diff --git a/solver/jobs.go b/solver/jobs.go index d2a52c39e61b..4eb89d2af72c 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -230,12 +230,13 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(interfa } type Job struct { - list *Solver - pr *progress.MultiReader - pw progress.Writer - span trace.Span - values sync.Map - id string + list *Solver + pr *progress.MultiReader + pw progress.Writer + span trace.Span + values sync.Map + id string + startedTime time.Time progressCloser func() SessionID string @@ -448,6 +449,7 @@ func (jl *Solver) NewJob(id string) (*Job, error) { progressCloser: progressCloser, span: span, id: id, + startedTime: time.Now(), } jl.jobs[id] = j @@ -564,6 +566,10 @@ func (j *Job) Discard() error { return nil } +func (j *Job) StartedTime() time.Time { + return j.startedTime +} + func (j *Job) InContext(ctx context.Context, f func(context.Context, session.Group) error) error { return f(progress.WithProgress(ctx, j.pw), session.NewGroup(j.SessionID)) } diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go new file mode 100644 index 000000000000..9d0d36abd73a --- /dev/null +++ b/solver/llbsolver/proc/provenance.go @@ -0,0 +1,118 @@ +package proc + +import ( + "context" + "encoding/json" + "strconv" + "strings" + "time" + + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + gatewaypb "github.com/moby/buildkit/frontend/gateway/pb" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/result" + binfotypes "github.com/moby/buildkit/util/buildinfo/types" + provenance "github.com/moby/buildkit/util/provenance" + "github.com/pkg/errors" +) + +var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" + +func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { + return func(ctx context.Context, res *frontend.Result, s *llbsolver.Solver, j *solver.Job) (*frontend.Result, error) { + if len(res.Refs) == 0 { + return nil, errors.New("provided result has no refs") + } + + platformsBytes, ok := res.Metadata[exptypes.ExporterPlatformsKey] + if !ok { + return nil, errors.Errorf("unable to collect multiple refs, missing platforms mapping") + } + + var ps exptypes.Platforms + if len(platformsBytes) > 0 { + if err := json.Unmarshal(platformsBytes, &ps); err != nil { + return nil, errors.Wrapf(err, "failed to parse platforms passed to sbom processor") + } + } + + buildID := identity.NewID() + + var reproducible bool + if v, ok := attrs["reproducible"]; ok { + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse reproducible flag %q", v) + } + reproducible = b + } + + var mode string + if v, ok := attrs["mode"]; ok { + switch mode { + case "disabled", "none": + return res, nil + case "full": + mode = "max" + case "max", "min": + mode = v + default: + return nil, errors.Errorf("invalid mode %q", v) + } + } + + for _, p := range ps.Platforms { + dt, ok := res.Metadata[exptypes.ExporterBuildInfo+"/"+p.ID] + if !ok { + return nil, errors.New("no build info found for provenance") + } + + var bi binfotypes.BuildInfo + if err := json.Unmarshal(dt, &bi); err != nil { + return nil, errors.Wrap(err, "failed to parse build info") + } + + pr, err := provenance.FromBuildInfo(bi) + if err != nil { + return nil, err + } + + st := j.StartedTime() + + pr.Metadata.BuildStartedOn = &st + pr.Metadata.Reproducible = reproducible + pr.Metadata.BuildInvocationID = buildID + + if mode != "max" { + param := make(map[string]*string) + for k, v := range pr.Invocation.Parameters.(map[string]*string) { + if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") { + pr.Metadata.Completeness.Parameters = false + continue + } + param[k] = v + } + pr.Invocation.Parameters = param + } + + res.AddAttestation(p.ID, result.Attestation{ + Kind: gatewaypb.AttestationKindInToto, + InToto: result.InTotoAttestation{ + PredicateType: slsa.PredicateSLSAProvenance, + }, + ContentFunc: func() ([]byte, error) { + end := time.Now() + pr.Metadata.BuildFinishedOn = &end + // TODO: pass indent to json.Marshal + return json.MarshalIndent(pr, "", " ") + }, + }, nil) + } + + return res, nil + } +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index dcc4d1475577..e4d9ff4a56eb 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -155,14 +155,6 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } } - for _, post := range post { - res2, err := post(ctx, res, s, j) - if err != nil { - return nil, err - } - res = res2 - } - if res == nil { res = &frontend.Result{} } @@ -208,6 +200,14 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } } + for _, post := range post { + res2, err := post(ctx, res, s, j) + if err != nil { + return nil, err + } + res = res2 + } + cached, err := result.ConvertResult(res, func(res solver.ResultProxy) (solver.CachedResult, error) { return res.Result(ctx) }) diff --git a/solver/result/attestation.go b/solver/result/attestation.go index 51ec1c0bf257..ceb6e535c8e1 100644 --- a/solver/result/attestation.go +++ b/solver/result/attestation.go @@ -11,7 +11,8 @@ type Attestation struct { Ref string Path string - InToto InTotoAttestation + InToto InTotoAttestation + ContentFunc func() ([]byte, error) } type InTotoAttestation struct { diff --git a/solver/result/result.go b/solver/result/result.go index 54d79fa167d1..eb0319f7bd2e 100644 --- a/solver/result/result.go +++ b/solver/result/result.go @@ -47,7 +47,7 @@ func (r *Result[T]) AddAttestation(k string, v Attestation, ref T) { if r.Attestations == nil { r.Attestations = map[string][]Attestation{} } - if !strings.HasPrefix(v.Ref, attestationRefPrefix) { + if v.ContentFunc == nil && !strings.HasPrefix(v.Ref, attestationRefPrefix) { v.Ref = "attestation:" + identity.NewID() r.Refs[v.Ref] = ref } diff --git a/util/buildinfo/buildinfo.go b/util/buildinfo/buildinfo.go index 9771d9d3484e..a36b767ac154 100644 --- a/util/buildinfo/buildinfo.go +++ b/util/buildinfo/buildinfo.go @@ -323,7 +323,7 @@ func filterAttrs(key string, attrs map[string]*string) map[string]*string { continue } // always include - if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") { + if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") || strings.HasPrefix(k, "vcs:") { filtered[k] = v continue } diff --git a/util/provenance/buildinfo.go b/util/provenance/buildinfo.go new file mode 100644 index 000000000000..1dcdfab7fd35 --- /dev/null +++ b/util/provenance/buildinfo.go @@ -0,0 +1,161 @@ +package proc + +import ( + "encoding/hex" + "strings" + + distreference "github.com/docker/distribution/reference" + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + binfotypes "github.com/moby/buildkit/util/buildinfo/types" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" + +type ProvenancePredicate struct { + slsa.ProvenancePredicate + Metadata *ProvenanceMetadata `json:"metadata,omitempty"` +} + +type ProvenanceMetadata struct { + slsa.ProvenanceMetadata + VCS map[string]string `json:"vcs,omitempty"` +} + +func convertMaterial(s binfotypes.Source) (*slsa.ProvenanceMaterial, error) { + // https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst + switch s.Type { + case binfotypes.SourceTypeDockerImage: + dgst, err := digest.Parse(s.Pin) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse digest %q for %s", s.Pin, s.Ref) + } + named, err := distreference.ParseNamed(s.Ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse ref %q", s.Ref) + } + version := "" + if tagged, ok := named.(distreference.Tagged); ok { + version = tagged.Tag() + } else { + if canonical, ok := named.(distreference.Canonical); ok { + version = canonical.Digest().String() + } + } + uri := "pkg:docker/" + distreference.FamiliarName(named) + if version != "" { + uri += "@" + version + } + return &slsa.ProvenanceMaterial{ + URI: uri, + Digest: slsa.DigestSet{ + dgst.Algorithm().String(): dgst.Hex(), + }, + }, nil + case binfotypes.SourceTypeGit: + if _, err := hex.DecodeString(s.Pin); err != nil { + return nil, errors.Wrapf(err, "failed to parse commit %q for %s", s.Pin, s.Ref) + } + return &slsa.ProvenanceMaterial{ + URI: s.Ref, + Digest: slsa.DigestSet{ + "sha1": s.Pin, // TODO: check length? + }, + }, nil + case binfotypes.SourceTypeHTTP: + dgst, err := digest.Parse(s.Pin) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse digest %q for %s", s.Pin, s.Ref) + } + return &slsa.ProvenanceMaterial{ + URI: s.Ref, + Digest: slsa.DigestSet{ + dgst.Algorithm().String(): dgst.Hex(), + }, + }, nil + default: + return nil, errors.Errorf("unsupported source type %q", s.Type) + } +} + +func findMaterial(srcs []binfotypes.Source, uri string) (*slsa.ProvenanceMaterial, bool) { + for _, s := range srcs { + if s.Ref == uri { + m, err := convertMaterial(s) + if err != nil { + continue + } + return m, true + } + } + return nil, false +} + +func FromBuildInfo(bi binfotypes.BuildInfo) (*ProvenancePredicate, error) { + materials := make([]slsa.ProvenanceMaterial, len(bi.Sources)) + for i, s := range bi.Sources { + m, err := convertMaterial(s) + if err != nil { + return nil, err + } + materials[i] = *m + } + + inv := slsa.ProvenanceInvocation{} + + contextKey := "context" + if v, ok := bi.Attrs["contextkey"]; ok && v != nil { + contextKey = *v + } + + if v, ok := bi.Attrs[contextKey]; ok && v != nil { + if m, ok := findMaterial(bi.Sources, *v); ok { + inv.ConfigSource.URI = m.URI + inv.ConfigSource.Digest = m.Digest + } else { + inv.ConfigSource.URI = *v + } + delete(bi.Attrs, contextKey) + } + + if v, ok := bi.Attrs["filename"]; ok && v != nil { + inv.ConfigSource.EntryPoint = *v + delete(bi.Attrs, "filename") + } + + vcs := make(map[string]string) + for k, v := range bi.Attrs { + if strings.HasPrefix(k, "vcs:") { + delete(bi.Attrs, k) + if v != nil { + vcs[strings.TrimPrefix(k, "vcs:")] = *v + } + } + } + + inv.Parameters = bi.Attrs + + pr := &ProvenancePredicate{ + ProvenancePredicate: slsa.ProvenancePredicate{ + BuildType: BuildKitBuildType, + Invocation: inv, + Materials: materials, + }, + Metadata: &ProvenanceMetadata{ + ProvenanceMetadata: slsa.ProvenanceMetadata{ + Completeness: slsa.ProvenanceComplete{ + Parameters: true, + Environment: true, + Materials: true, // TODO: check that there were no local sources + }, + }, + }, + } + + if len(vcs) > 0 { + pr.Metadata.VCS = vcs + } + + return pr, nil +} From eb20ff5a47f273a3d97eacb67532bb1aa456b313 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 31 Aug 2022 15:54:18 -0700 Subject: [PATCH 3/9] containerimage: correct attestation count detection Signed-off-by: Tonis Tiigi --- exporter/containerimage/writer.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index ebf930ebbd97..3b5cdaa9fb9c 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -129,8 +129,10 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return mfstDesc, nil } + hasAttestations := false attestCount := 0 for _, attests := range inp.Attestations { + hasAttestations = true for _, attest := range attests { if attest.ContentFunc == nil { attestCount++ @@ -141,7 +143,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return nil, errors.Errorf("number of required refs (%d) does not match number of references (%d)", count, len(inp.Refs)) } - if attestCount > 0 { + if hasAttestations { opts.EnableOCITypes("attestations") } From 915d7dcf7a5deb4d5a6980f5fb1ed779ded24631 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 20 Sep 2022 19:13:34 -0700 Subject: [PATCH 4/9] provenance: build definition support Signed-off-by: Tonis Tiigi --- solver/llbsolver/proc/provenance.go | 6 +- util/provenance/buildconfig.go | 169 ++++++++++++++++++++++++++++ util/provenance/buildinfo.go | 5 +- 3 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 util/provenance/buildconfig.go diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 9d0d36abd73a..4777d61574b9 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -53,7 +53,7 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { var mode string if v, ok := attrs["mode"]; ok { - switch mode { + switch v { case "disabled", "none": return res, nil case "full": @@ -97,6 +97,10 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { param[k] = v } pr.Invocation.Parameters = param + } else { + if err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID]); err != nil { + return nil, err + } } res.AddAttestation(p.ID, result.Attestation{ diff --git a/util/provenance/buildconfig.go b/util/provenance/buildconfig.go new file mode 100644 index 000000000000..140eaf84e355 --- /dev/null +++ b/util/provenance/buildconfig.go @@ -0,0 +1,169 @@ +package provenance + +import ( + "context" + "fmt" + + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/pb" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type BuildConfig struct { + Definition []BuildStep `json:"llbDefinition,omitempty"` +} + +type BuildStep struct { + ID string `json:"ID,omitempty"` + Op interface{} `json:"op,omitempty"` + Inputs []string `json:"inputs,omitempty"` +} + +type Source struct { + Locations map[string]*pb.Locations `json:"locations,omitempty"` + Infos []SourceInfo `json:"infos,omitempty"` +} + +type SourceInfo struct { + Filename string `json:"filename,omitempty"` + Data []byte `json:"data,omitempty"` + Definition []BuildStep `json:"llbDefinition,omitempty"` +} + +func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.ResultProxy) error { + def := rp.Definition() + steps, indexes, err := toBuildSteps(def) + if err != nil { + return err + } + + bc := &BuildConfig{ + Definition: steps, + } + + p.BuildConfig = bc + + if def.Source != nil { + sis := make([]SourceInfo, len(def.Source.Infos)) + for i, si := range def.Source.Infos { + steps, _, err := toBuildSteps(si.Definition) + if err != nil { + return err + } + s := SourceInfo{ + Filename: si.Filename, + Data: si.Data, + Definition: steps, + } + sis[i] = s + } + + if len(def.Source.Infos) != 0 { + locs := map[string]*pb.Locations{} + for k, l := range def.Source.Locations { + idx, ok := indexes[digest.Digest(k)] + if !ok { + continue + } + locs[fmt.Sprintf("step%d", idx)] = l + } + + p.Source = &Source{ + Infos: sis, + } + } + } + + return nil +} + +func toBuildSteps(def *pb.Definition) ([]BuildStep, map[digest.Digest]int, error) { + if def == nil || len(def.Def) == 0 { + return nil, nil, nil + } + + ops := make(map[digest.Digest]*pb.Op) + defs := make(map[digest.Digest][]byte) + + var dgst digest.Digest + for _, dt := range def.Def { + var op pb.Op + if err := (&op).Unmarshal(dt); err != nil { + return nil, nil, errors.Wrap(err, "failed to parse llb proto op") + } + dgst = digest.FromBytes(dt) + ops[dgst] = &op + defs[dgst] = dt + } + + if dgst == "" { + return nil, nil, nil + } + + // depth first backwards + dgsts := make([]digest.Digest, 0, len(def.Def)) + op := ops[dgst] + + if op.Op != nil { + return nil, nil, errors.Errorf("invalid last vertex: %T", op.Op) + } + + if len(op.Inputs) != 1 { + return nil, nil, errors.Errorf("invalid last vertex inputs: %v", len(op.Inputs)) + } + + visited := map[digest.Digest]struct{}{} + dgsts, err := walkDigests(dgsts, ops, dgst, visited) + if err != nil { + return nil, nil, err + } + for i := 0; i < len(dgsts)/2; i++ { + j := len(dgsts) - 1 - i + dgsts[i], dgsts[j] = dgsts[j], dgsts[i] + } + + indexes := map[digest.Digest]int{} + for i, dgst := range dgsts { + indexes[dgst] = i + } + + out := make([]BuildStep, 0, len(dgsts)) + for i, dgst := range dgsts { + op := *ops[dgst] + inputs := make([]string, len(op.Inputs)) + for i, inp := range op.Inputs { + inputs[i] = fmt.Sprintf("step%d:%d", indexes[inp.Digest], inp.Index) + } + op.Inputs = nil + out = append(out, BuildStep{ + ID: fmt.Sprintf("step%d", i), + Inputs: inputs, + Op: op, + }) + } + return out, indexes, nil +} + +func walkDigests(dgsts []digest.Digest, ops map[digest.Digest]*pb.Op, dgst digest.Digest, visited map[digest.Digest]struct{}) ([]digest.Digest, error) { + if _, ok := visited[dgst]; ok { + return dgsts, nil + } + op, ok := ops[dgst] + if !ok { + return nil, errors.Errorf("failed to find input %v", dgst) + } + if op == nil { + return nil, errors.Errorf("invalid nil input %v", dgst) + } + dgsts = append(dgsts, dgst) + visited[dgst] = struct{}{} + for _, inp := range op.Inputs { + var err error + dgsts, err = walkDigests(dgsts, ops, inp.Digest, visited) + if err != nil { + return nil, err + } + } + return dgsts, nil +} diff --git a/util/provenance/buildinfo.go b/util/provenance/buildinfo.go index 1dcdfab7fd35..05fedfe67af3 100644 --- a/util/provenance/buildinfo.go +++ b/util/provenance/buildinfo.go @@ -1,4 +1,4 @@ -package proc +package provenance import ( "encoding/hex" @@ -7,7 +7,7 @@ import ( distreference "github.com/docker/distribution/reference" slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" binfotypes "github.com/moby/buildkit/util/buildinfo/types" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -16,6 +16,7 @@ var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" type ProvenancePredicate struct { slsa.ProvenancePredicate Metadata *ProvenanceMetadata `json:"metadata,omitempty"` + Source *Source `json:"buildSource,omitempty"` } type ProvenanceMetadata struct { From a20e48f36d2314ec1df65cb5186d7178399a13ed Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 25 Sep 2022 23:02:38 -0700 Subject: [PATCH 5/9] provenance: add layers support Signed-off-by: Tonis Tiigi --- cache/remotecache/v1/chains.go | 4 +- cache/remotecache/v1/chains_test.go | 2 +- cache/remotecache/v1/parse.go | 4 +- exporter/containerimage/writer.go | 6 +- solver/cache_test.go | 30 +++---- solver/cachekey.go | 5 +- solver/edge.go | 10 ++- solver/exporter.go | 15 ++-- solver/index_test.go | 74 ++++++++--------- solver/llbsolver/proc/provenance.go | 124 +++++++++++++++++++++++++++- solver/scheduler_test.go | 2 +- solver/types.go | 4 +- util/provenance/buildconfig.go | 11 +-- util/provenance/buildinfo.go | 6 +- 14 files changed, 217 insertions(+), 80 deletions(-) diff --git a/cache/remotecache/v1/chains.go b/cache/remotecache/v1/chains.go index 306e037f7f1c..8c8bbde5dc76 100644 --- a/cache/remotecache/v1/chains.go +++ b/cache/remotecache/v1/chains.go @@ -146,7 +146,7 @@ func (c *item) removeLink(src *item) bool { return found } -func (c *item) AddResult(createdAt time.Time, result *solver.Remote) { +func (c *item) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *solver.Remote) { c.resultTime = createdAt c.result = result } @@ -214,7 +214,7 @@ func (c *item) walkAllResults(fn func(i *item) error, visited map[*item]struct{} type nopRecord struct { } -func (c *nopRecord) AddResult(createdAt time.Time, result *solver.Remote) { +func (c *nopRecord) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *solver.Remote) { } func (c *nopRecord) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) { diff --git a/cache/remotecache/v1/chains_test.go b/cache/remotecache/v1/chains_test.go index 7e2a2f525769..5e7bcd0691c7 100644 --- a/cache/remotecache/v1/chains_test.go +++ b/cache/remotecache/v1/chains_test.go @@ -29,7 +29,7 @@ func TestSimpleMarshal(t *testing.T) { Digest: dgst("d1"), }}, } - baz.AddResult(time.Now(), r0) + baz.AddResult("", 0, time.Now(), r0) } addRecords() diff --git a/cache/remotecache/v1/parse.go b/cache/remotecache/v1/parse.go index 65a6e441f575..3c8294a602c0 100644 --- a/cache/remotecache/v1/parse.go +++ b/cache/remotecache/v1/parse.go @@ -61,7 +61,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver. return nil, err } if remote != nil { - r.AddResult(res.CreatedAt, remote) + r.AddResult("", 0, res.CreatedAt, remote) } } @@ -86,7 +86,7 @@ func parseRecord(cc CacheConfig, idx int, provider DescriptorProvider, t solver. } if remote != nil { remote.Provider = mp - r.AddResult(res.CreatedAt, remote) + r.AddResult("", 0, res.CreatedAt, remote) } } diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 3b5cdaa9fb9c..d1504fff53d6 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -519,7 +519,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima } for i, desc := range remote.Descriptors { - removeInternalLayerAnnotations(&desc, opts.OCITypes) + RemoveInternalLayerAnnotations(&desc, opts.OCITypes) mfst.Layers = append(mfst.Layers, desc) labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = desc.Digest.String() } @@ -631,7 +631,7 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima "containerd.io/gc.ref.content.0": configDigest.String(), } for i, desc := range layers { - removeInternalLayerAnnotations(&desc, opts.OCITypes) + RemoveInternalLayerAnnotations(&desc, opts.OCITypes) mfst.Layers = append(mfst.Layers, desc) labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = desc.Digest.String() } @@ -889,7 +889,7 @@ func normalizeLayersAndHistory(ctx context.Context, remote *solver.Remote, histo return remote, history } -func removeInternalLayerAnnotations(desc *ocispecs.Descriptor, oci bool) { +func RemoveInternalLayerAnnotations(desc *ocispecs.Descriptor, oci bool) { if oci { // oci supports annotations but don't export internal annotations delete(desc.Annotations, "containerd.io/uncompressed") diff --git a/solver/cache_test.go b/solver/cache_test.go index 5830d62cef21..8e20145f9e42 100644 --- a/solver/cache_test.go +++ b/solver/cache_test.go @@ -19,7 +19,7 @@ func depKeys(cks ...ExportableCacheKey) []CacheKeyWithSelector { } func testCacheKey(dgst digest.Digest, output Index, deps ...ExportableCacheKey) *CacheKey { - k := NewCacheKey(dgst, output) + k := NewCacheKey(dgst, "", output) k.deps = make([][]CacheKeyWithSelector, len(deps)) for i, dep := range deps { k.deps[i] = depKeys(dep) @@ -28,7 +28,7 @@ func testCacheKey(dgst digest.Digest, output Index, deps ...ExportableCacheKey) } func testCacheKeyWithDeps(dgst digest.Digest, output Index, deps [][]CacheKeyWithSelector) *CacheKey { - k := NewCacheKey(dgst, output) + k := NewCacheKey(dgst, "", output) k.deps = deps return k } @@ -42,7 +42,7 @@ func TestInMemoryCache(t *testing.T) { m := NewInMemoryCacheManager() - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now()) require.NoError(t, err) keys, err := m.Query(nil, 0, dgst("foo"), 0) @@ -58,7 +58,7 @@ func TestInMemoryCache(t *testing.T) { require.Equal(t, "result0", unwrap(res)) // another record - cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("result1"), time.Now()) + cacheBar, err := m.Save(NewCacheKey(dgst("bar"), "", 0), testResult("result1"), time.Now()) require.NoError(t, err) keys, err = m.Query(nil, 0, dgst("bar"), 0) @@ -155,7 +155,7 @@ func TestInMemoryCacheSelector(t *testing.T) { m := NewInMemoryCacheManager() - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now()) require.NoError(t, err) _, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ @@ -189,11 +189,11 @@ func TestInMemoryCacheSelectorNested(t *testing.T) { m := NewInMemoryCacheManager() - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("result0"), time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("result0"), time.Now()) require.NoError(t, err) _, err = m.Save(testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("second"), 0))}}, + {{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("second"), "", 0))}}, }), testResult("result1"), time.Now()) require.NoError(t, err) @@ -219,7 +219,7 @@ func TestInMemoryCacheSelectorNested(t *testing.T) { require.NoError(t, err) require.Equal(t, len(keys), 0) - keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), 0))), 0, dgst("bar"), 0) + keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), "", 0))), 0, dgst("bar"), 0) require.NoError(t, err) require.Equal(t, len(keys), 1) @@ -231,7 +231,7 @@ func TestInMemoryCacheSelectorNested(t *testing.T) { require.NoError(t, err) require.Equal(t, "result1", unwrap(res)) - keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), 0))), 0, dgst("bar"), 0) + keys, err = m.Query(depKeys(expKey(NewCacheKey(dgst("second"), "", 0))), 0, dgst("bar"), 0) require.NoError(t, err) require.Equal(t, len(keys), 1) } @@ -242,7 +242,7 @@ func TestInMemoryCacheReleaseParent(t *testing.T) { m := NewCacheManager(context.TODO(), identity.NewID(), storage, results) res0 := testResult("result0") - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), res0, time.Now()) require.NoError(t, err) res1 := testResult("result1") @@ -294,7 +294,7 @@ func TestInMemoryCacheRestoreOfflineDeletion(t *testing.T) { m := NewCacheManager(context.TODO(), identity.NewID(), storage, results) res0 := testResult("result0") - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), res0, time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), res0, time.Now()) require.NoError(t, err) res1 := testResult("result1") @@ -329,20 +329,20 @@ func TestCarryOverFromSublink(t *testing.T) { results := NewInMemoryResultStorage() m := NewCacheManager(context.TODO(), identity.NewID(), storage, results) - cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), 0), testResult("resultFoo"), time.Now()) + cacheFoo, err := m.Save(NewCacheKey(dgst("foo"), "", 0), testResult("resultFoo"), time.Now()) require.NoError(t, err) _, err = m.Save(testCacheKeyWithDeps(dgst("res"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("content0"), 0))}}, + {{CacheKey: *cacheFoo, Selector: dgst("sel0")}, {CacheKey: expKey(NewCacheKey(dgst("content0"), "", 0))}}, }), testResult("result0"), time.Now()) require.NoError(t, err) - cacheBar, err := m.Save(NewCacheKey(dgst("bar"), 0), testResult("resultBar"), time.Now()) + cacheBar, err := m.Save(NewCacheKey(dgst("bar"), "", 0), testResult("resultBar"), time.Now()) require.NoError(t, err) keys, err := m.Query([]CacheKeyWithSelector{ {CacheKey: *cacheBar, Selector: dgst("sel0")}, - {CacheKey: expKey(NewCacheKey(dgst("content0"), 0))}, + {CacheKey: expKey(NewCacheKey(dgst("content0"), "", 0))}, }, 0, dgst("res"), 0) require.NoError(t, err) require.Equal(t, len(keys), 1) diff --git a/solver/cachekey.go b/solver/cachekey.go index 3749af0ab3ac..398368716ad2 100644 --- a/solver/cachekey.go +++ b/solver/cachekey.go @@ -7,10 +7,11 @@ import ( ) // NewCacheKey creates a new cache key for a specific output index -func NewCacheKey(dgst digest.Digest, output Index) *CacheKey { +func NewCacheKey(dgst, vtx digest.Digest, output Index) *CacheKey { return &CacheKey{ ID: rootKey(dgst, output).String(), digest: dgst, + vtx: vtx, output: output, ids: map[*cacheManager]string{}, } @@ -29,6 +30,7 @@ type CacheKey struct { ID string deps [][]CacheKeyWithSelector // only [][]*inMemoryCacheKey digest digest.Digest + vtx digest.Digest output Index ids map[*cacheManager]string @@ -56,6 +58,7 @@ func (ck *CacheKey) clone() *CacheKey { nk := &CacheKey{ ID: ck.ID, digest: ck.digest, + vtx: ck.vtx, output: ck.output, ids: map[*cacheManager]string{}, } diff --git a/solver/edge.go b/solver/edge.go index 3493054d3e97..5e3068010f80 100644 --- a/solver/edge.go +++ b/solver/edge.go @@ -136,11 +136,11 @@ func (e *edge) release() { // commitOptions returns parameters for the op execution func (e *edge) commitOptions() ([]*CacheKey, []CachedResult) { - k := NewCacheKey(e.cacheMap.Digest, e.edge.Index) + k := NewCacheKey(e.cacheMap.Digest, e.edge.Vertex.Digest(), e.edge.Index) if len(e.deps) == 0 { keys := make([]*CacheKey, 0, len(e.cacheMapDigests)) for _, dgst := range e.cacheMapDigests { - keys = append(keys, NewCacheKey(dgst, e.edge.Index)) + keys = append(keys, NewCacheKey(dgst, e.edge.Vertex.Digest(), e.edge.Index)) } return keys, nil } @@ -201,6 +201,7 @@ func (e *edge) probeCache(d *dep, depKeys []CacheKeyWithSelector) bool { } found := false for _, k := range keys { + k.vtx = e.edge.Vertex.Digest() if _, ok := d.keyMap[k.ID]; !ok { d.keyMap[k.ID] = k found = true @@ -275,7 +276,7 @@ func (e *edge) currentIndexKey() *CacheKey { } } - k := NewCacheKey(e.cacheMap.Digest, e.edge.Index) + k := NewCacheKey(e.cacheMap.Digest, e.edge.Vertex.Digest(), e.edge.Index) k.deps = keys return k @@ -403,6 +404,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) { bklog.G(context.TODO()).Error(errors.Wrap(err, "invalid query response")) // make the build fail for this error } else { for _, k := range keys { + k.vtx = e.edge.Vertex.Digest() records, err := e.op.Cache().Records(k) if err != nil { bklog.G(context.TODO()).Errorf("error receiving cache records: %v", err) @@ -508,7 +510,7 @@ func (e *edge) processUpdate(upt pipe.Receiver) (depChanged bool) { } else if !dep.slowCacheComplete { dgst := upt.Status().Value.(digest.Digest) if e.cacheMap.Deps[int(dep.index)].ComputeDigestFunc != nil && dgst != "" { - k := NewCacheKey(dgst, -1) + k := NewCacheKey(dgst, "", -1) dep.slowCacheKey = &ExportableCacheKey{CacheKey: k, Exporter: &exporter{k: k}} slowKeyExp := CacheKeyWithSelector{CacheKey: *dep.slowCacheKey} defKeys := make([]CacheKeyWithSelector, 0, len(dep.result.CacheKeys())) diff --git a/solver/exporter.go b/solver/exporter.go index 67ede422239b..78ce77c2d2f5 100644 --- a/solver/exporter.go +++ b/solver/exporter.go @@ -96,12 +96,17 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach addRecord = *e.override } - if e.record == nil && len(e.k.Deps()) > 0 { + exportRecord := opt.ExportRoots + if len(e.k.Deps()) > 0 { + exportRecord = true + } + + if e.record == nil && exportRecord { e.record = getBestResult(e.records) } var remote *Remote - if v := e.record; v != nil && len(e.k.Deps()) > 0 && addRecord { + if v := e.record; v != nil && exportRecord && addRecord { var variants []CacheExporterRecord cm := v.cacheManager @@ -121,7 +126,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach if opt.CompressionOpt != nil { for _, r := range remotes { // record all remaining remotes as well rec := t.Add(recKey) - rec.AddResult(v.CreatedAt, r) + rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, r) variants = append(variants, rec) } } @@ -142,7 +147,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach if opt.CompressionOpt != nil { for _, r := range remotes { // record all remaining remotes as well rec := t.Add(recKey) - rec.AddResult(v.CreatedAt, r) + rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, r) variants = append(variants, rec) } } @@ -150,7 +155,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach if remote != nil { for _, rec := range allRec { - rec.AddResult(v.CreatedAt, remote) + rec.AddResult(e.k.vtx, int(e.k.output), v.CreatedAt, remote) } } allRec = append(allRec, variants...) diff --git a/solver/index_test.go b/solver/index_test.go index c77ff3cbce94..da7dd818951d 100644 --- a/solver/index_test.go +++ b/solver/index_test.go @@ -18,21 +18,21 @@ func TestIndexSimple(t *testing.T) { e2 := &edge{} e3 := &edge{} - k1 := NewCacheKey(dgst("foo"), 0) + k1 := NewCacheKey(dgst("foo"), "", 0) v := idx.LoadOrStore(k1, e1) require.Nil(t, v) - k2 := NewCacheKey(dgst("bar"), 0) + k2 := NewCacheKey(dgst("bar"), "", 0) v = idx.LoadOrStore(k2, e2) require.Nil(t, v) - v = idx.LoadOrStore(NewCacheKey(dgst("bar"), 0), e3) + v = idx.LoadOrStore(NewCacheKey(dgst("bar"), "", 0), e3) require.Equal(t, v, e2) - v = idx.LoadOrStore(NewCacheKey(dgst("bar"), 0), e3) + v = idx.LoadOrStore(NewCacheKey(dgst("bar"), "", 0), e3) require.Equal(t, v, e2) - v = idx.LoadOrStore(NewCacheKey(dgst("foo"), 0), e3) + v = idx.LoadOrStore(NewCacheKey(dgst("foo"), "", 0), e3) require.Equal(t, v, e1) idx.Release(e1) @@ -48,16 +48,16 @@ func TestIndexMultiLevelSimple(t *testing.T) { e3 := &edge{} k1 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}}, }) v := idx.LoadOrStore(k1, e1) require.Nil(t, v) k2 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}}, }) v = idx.LoadOrStore(k2, e2) @@ -72,18 +72,18 @@ func TestIndexMultiLevelSimple(t *testing.T) { // update selector k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0))}}, }) v = idx.LoadOrStore(k2, e2) require.Nil(t, v) // add one dep to e1 k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, { - {CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}, - {CacheKey: expKey(NewCacheKey("s1", 1))}, + {CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}, + {CacheKey: expKey(NewCacheKey("s1", "", 1))}, }, }) v = idx.LoadOrStore(k2, e2) @@ -91,9 +91,9 @@ func TestIndexMultiLevelSimple(t *testing.T) { // recheck with only the new dep key k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, { - {CacheKey: expKey(NewCacheKey("s1", 1))}, + {CacheKey: expKey(NewCacheKey("s1", "", 1))}, }, }) v = idx.LoadOrStore(k2, e2) @@ -101,10 +101,10 @@ func TestIndexMultiLevelSimple(t *testing.T) { // combine e1 and e2 k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, { - {CacheKey: expKey(NewCacheKey("s0", 0))}, - {CacheKey: expKey(NewCacheKey("s1", 1))}, + {CacheKey: expKey(NewCacheKey("s0", "", 0))}, + {CacheKey: expKey(NewCacheKey("s1", "", 1))}, }, }) v = idx.LoadOrStore(k2, e2) @@ -112,8 +112,8 @@ func TestIndexMultiLevelSimple(t *testing.T) { // initial e2 now points to e1 k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0))}}, }) v = idx.LoadOrStore(k2, e2) require.Equal(t, v, e1) @@ -122,8 +122,8 @@ func TestIndexMultiLevelSimple(t *testing.T) { // e2 still remains after e1 is gone k2 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0))}}, }) v = idx.LoadOrStore(k2, e3) require.Equal(t, v, e2) @@ -140,8 +140,8 @@ func TestIndexThreeLevels(t *testing.T) { e3 := &edge{} k1 := testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, - {{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}}, }) v := idx.LoadOrStore(k1, e1) @@ -151,26 +151,26 @@ func TestIndexThreeLevels(t *testing.T) { require.Equal(t, v, e1) k2 := testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, {{CacheKey: expKey(k1)}}, }) v = idx.LoadOrStore(k2, e2) require.Nil(t, v) k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, { {CacheKey: expKey(k1)}, - {CacheKey: expKey(NewCacheKey("alt", 0))}, + {CacheKey: expKey(NewCacheKey("alt", "", 0))}, }, }) v = idx.LoadOrStore(k2, e2) require.Nil(t, v) k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, { - {CacheKey: expKey(NewCacheKey("alt", 0))}, + {CacheKey: expKey(NewCacheKey("alt", "", 0))}, }, }) v = idx.LoadOrStore(k2, e3) @@ -179,13 +179,13 @@ func TestIndexThreeLevels(t *testing.T) { // change dep in a low key k1 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ { - {CacheKey: expKey(NewCacheKey("f0", 0))}, - {CacheKey: expKey(NewCacheKey("f0_", 0))}, + {CacheKey: expKey(NewCacheKey("f0", "", 0))}, + {CacheKey: expKey(NewCacheKey("f0_", "", 0))}, }, - {{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}}, }) k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, {{CacheKey: expKey(k1)}}, }) v = idx.LoadOrStore(k2, e3) @@ -194,12 +194,12 @@ func TestIndexThreeLevels(t *testing.T) { // reload with only f0_ still matches k1 = testCacheKeyWithDeps(dgst("foo"), 1, [][]CacheKeyWithSelector{ { - {CacheKey: expKey(NewCacheKey("f0_", 0))}, + {CacheKey: expKey(NewCacheKey("f0_", "", 0))}, }, - {{CacheKey: expKey(NewCacheKey("s0", 0)), Selector: dgst("s0")}}, + {{CacheKey: expKey(NewCacheKey("s0", "", 0)), Selector: dgst("s0")}}, }) k2 = testCacheKeyWithDeps(dgst("bar"), 0, [][]CacheKeyWithSelector{ - {{CacheKey: expKey(NewCacheKey("f0", 0))}}, + {{CacheKey: expKey(NewCacheKey("f0", "", 0))}}, {{CacheKey: expKey(k1)}}, }) v = idx.LoadOrStore(k2, e3) diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 4777d61574b9..3a1d19f31af3 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -3,11 +3,15 @@ package proc import ( "context" "encoding/json" + "fmt" "strconv" "strings" "time" slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/exporter/containerimage" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend" gatewaypb "github.com/moby/buildkit/frontend/gateway/pb" @@ -17,6 +21,9 @@ import ( "github.com/moby/buildkit/solver/result" binfotypes "github.com/moby/buildkit/util/buildinfo/types" provenance "github.com/moby/buildkit/util/provenance" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -87,6 +94,8 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { pr.Metadata.Reproducible = reproducible pr.Metadata.BuildInvocationID = buildID + var addLayers func() error + if mode != "max" { param := make(map[string]*string) for k, v := range pr.Invocation.Parameters.(map[string]*string) { @@ -98,9 +107,43 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { } pr.Invocation.Parameters = param } else { - if err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID]); err != nil { + dgsts, err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID]) + if err != nil { + return nil, err + } + + r, err := res.Refs[p.ID].Result(ctx) + if err != nil { return nil, err } + + addLayers = func() error { + e := newCacheExporter() + if _, err := r.CacheKeys()[0].Exporter.ExportTo(ctx, e, solver.CacheExportOpt{ + ResolveRemotes: resolveRemotes, + Mode: solver.CacheExportModeRemoteOnly, + ExportRoots: true, + }); err != nil { + return err + } + + m := map[string][][]ocispecs.Descriptor{} + + for l, descs := range e.layers { + idx, ok := dgsts[l.digest] + if !ok { + continue + } + + m[fmt.Sprintf("step%d:%d", idx, l.index)] = descs + } + + if len(m) != 0 { + pr.Layers = m + } + + return nil + } } res.AddAttestation(p.ID, result.Attestation{ @@ -111,6 +154,13 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { ContentFunc: func() ([]byte, error) { end := time.Now() pr.Metadata.BuildFinishedOn = &end + + if addLayers != nil { + if err := addLayers(); err != nil { + return nil, err + } + } + // TODO: pass indent to json.Marshal return json.MarshalIndent(pr, "", " ") }, @@ -120,3 +170,75 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { return res, nil } } + +func resolveRemotes(ctx context.Context, res solver.Result) ([]*solver.Remote, error) { + ref, ok := res.Sys().(*worker.WorkerRef) + if !ok { + return nil, errors.Errorf("invalid result: %T", res.Sys()) + } + + remotes, err := ref.GetRemotes(ctx, false, config.RefConfig{}, true, nil) + if err != nil { + if errors.Is(err, cache.ErrNoBlobs) { + return nil, nil + } + return nil, err + } + return remotes, nil +} + +type edge struct { + digest digest.Digest + index int +} + +func newCacheExporter() *cacheExporter { + return &cacheExporter{ + m: map[interface{}]struct{}{}, + layers: map[edge][][]ocispecs.Descriptor{}, + } +} + +type cacheExporter struct { + layers map[edge][][]ocispecs.Descriptor + m map[interface{}]struct{} +} + +func (ce *cacheExporter) Add(dgst digest.Digest) solver.CacheExporterRecord { + return &cacheRecord{ + ce: ce, + } +} + +func (ce *cacheExporter) Visit(v interface{}) { + ce.m[v] = struct{}{} +} + +func (ce *cacheExporter) Visited(v interface{}) bool { + _, ok := ce.m[v] + return ok +} + +type cacheRecord struct { + ce *cacheExporter +} + +func (c *cacheRecord) AddResult(dgst digest.Digest, idx int, createdAt time.Time, result *solver.Remote) { + if result == nil || dgst == "" { + return + } + e := edge{ + digest: dgst, + index: idx, + } + descs := make([]ocispecs.Descriptor, len(result.Descriptors)) + for i, desc := range result.Descriptors { + d := desc + containerimage.RemoveInternalLayerAnnotations(&d, true) + descs[i] = d + } + c.ce.layers[e] = append(c.ce.layers[e], descs) +} + +func (c *cacheRecord) LinkFrom(rec solver.CacheExporterRecord, index int, selector string) { +} diff --git a/solver/scheduler_test.go b/solver/scheduler_test.go index 7b17b88892c5..6917baa81f7d 100644 --- a/solver/scheduler_test.go +++ b/solver/scheduler_test.go @@ -3844,7 +3844,7 @@ type testExporterRecord struct { linkMap map[digest.Digest]struct{} } -func (r *testExporterRecord) AddResult(createdAt time.Time, result *Remote) { +func (r *testExporterRecord) AddResult(_ digest.Digest, _ int, createdAt time.Time, result *Remote) { r.results++ } diff --git a/solver/types.go b/solver/types.go index a20c1020f21e..b62da7680bb3 100644 --- a/solver/types.go +++ b/solver/types.go @@ -104,6 +104,8 @@ type CacheExportOpt struct { // CompressionOpt is an option to specify the compression of the object to load. // If specified, all objects that meet the option will be cached. CompressionOpt *compression.Config + // ExportRoots defines if records for root vertexes should be exported. + ExportRoots bool } // CacheExporter can export the artifacts of the build chain @@ -120,7 +122,7 @@ type CacheExporterTarget interface { // CacheExporterRecord is a single object being exported type CacheExporterRecord interface { - AddResult(createdAt time.Time, result *Remote) + AddResult(vtx digest.Digest, index int, createdAt time.Time, result *Remote) LinkFrom(src CacheExporterRecord, index int, selector string) } diff --git a/util/provenance/buildconfig.go b/util/provenance/buildconfig.go index 140eaf84e355..90e7cd05279a 100644 --- a/util/provenance/buildconfig.go +++ b/util/provenance/buildconfig.go @@ -31,11 +31,11 @@ type SourceInfo struct { Definition []BuildStep `json:"llbDefinition,omitempty"` } -func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.ResultProxy) error { +func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.ResultProxy) (map[digest.Digest]int, error) { def := rp.Definition() steps, indexes, err := toBuildSteps(def) if err != nil { - return err + return nil, err } bc := &BuildConfig{ @@ -49,7 +49,7 @@ func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.Resul for i, si := range def.Source.Infos { steps, _, err := toBuildSteps(si.Definition) if err != nil { - return err + return nil, err } s := SourceInfo{ Filename: si.Filename, @@ -70,12 +70,13 @@ func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.Resul } p.Source = &Source{ - Infos: sis, + Infos: sis, + Locations: locs, } } } - return nil + return indexes, nil } func toBuildSteps(def *pb.Definition) ([]BuildStep, map[digest.Digest]int, error) { diff --git a/util/provenance/buildinfo.go b/util/provenance/buildinfo.go index 05fedfe67af3..90dc8028777d 100644 --- a/util/provenance/buildinfo.go +++ b/util/provenance/buildinfo.go @@ -8,6 +8,7 @@ import ( slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" binfotypes "github.com/moby/buildkit/util/buildinfo/types" digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -15,8 +16,9 @@ var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" type ProvenancePredicate struct { slsa.ProvenancePredicate - Metadata *ProvenanceMetadata `json:"metadata,omitempty"` - Source *Source `json:"buildSource,omitempty"` + Metadata *ProvenanceMetadata `json:"metadata,omitempty"` + Source *Source `json:"buildSource,omitempty"` + Layers map[string][][]ocispecs.Descriptor `json:"buildLayers,omitempty"` } type ProvenanceMetadata struct { From 5df7d75964770c3ddffaf22df9eebb1a37865f41 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 30 Oct 2022 20:19:00 -0700 Subject: [PATCH 6/9] dockerfile: add provenance attestation tests Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile_test.go | 150 +++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index e13855fb8088..9eec347f062d 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/containerd/continuity/fs/fstest" intoto "github.com/in-toto/in-toto-golang/in_toto" + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/builder" @@ -39,6 +40,7 @@ import ( "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" + "github.com/moby/buildkit/util/provenance" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" "github.com/moby/buildkit/util/testutil/integration" @@ -147,6 +149,7 @@ var allTests = integration.TestFuncs( testDockerfileAddChownExpand, testSourceDateEpochWithoutExporter, testSBOMScannerImage, + testProvenanceAttestation, ) // Tests that depend on the `security.*` entitlements @@ -6153,6 +6156,153 @@ EOF require.Equal(t, map[string]interface{}{"success": true}, attest.Predicate) } +func testProvenanceAttestation(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +RUN echo "ok" > /foo +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + for _, mode := range []string{"min", "max"} { + t.Run(mode, func(t *testing.T) { + target := registry + "/buildkit/testwithprovenance:" + mode + provReq := "" + if mode == "max" { + provReq = "mode=max" + } + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + FrontendAttrs: map[string]string{ + "attest:provenance": provReq, + "build-arg:FOO": "bar", + "label:lbl": "abc", + "vcs:source": "https://example.invalid/repo.git", + "vcs:revision": "123456", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + require.NotNil(t, img) + require.Equal(t, []byte("ok\n"), img.Layers[1]["foo"].Data) + + att := imgs.Find("unknown/unknown") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + + require.Equal(t, slsa.ConfigSource{}, pred.Invocation.ConfigSource) + + switch f.(type) { + case *clientFrontend, *gatewayFrontend: + // TODO: buildinfo broken + default: + params, ok := pred.Invocation.Parameters.(map[string]interface{}) + require.True(t, ok, "%T", pred.Invocation.Parameters) + if mode == "max" { + require.Equal(t, 2, len(params)) + require.True(t, pred.Metadata.Completeness.Parameters) + + require.Equal(t, "bar", params["build-arg:FOO"]) + require.Equal(t, "abc", params["label:lbl"]) + } else { + require.False(t, pred.Metadata.Completeness.Parameters) + require.Equal(t, 0, len(params), "%v", params) + } + + require.Equal(t, "https://example.invalid/repo.git", pred.Metadata.VCS["source"]) + require.Equal(t, "123456", pred.Metadata.VCS["revision"]) + } + + require.Equal(t, 1, len(pred.Materials)) + require.Equal(t, "pkg:docker/busybox@latest", pred.Materials[0].URI) + require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) + + require.NotEmpty(t, pred.Metadata.BuildInvocationID) + + require.NotNil(t, pred.Metadata.BuildFinishedOn) + require.True(t, time.Since(*pred.Metadata.BuildFinishedOn) < 5*time.Minute) + require.NotNil(t, pred.Metadata.BuildStartedOn) + require.True(t, time.Since(*pred.Metadata.BuildStartedOn) < 5*time.Minute) + require.True(t, pred.Metadata.BuildStartedOn.Before(*pred.Metadata.BuildFinishedOn)) + + require.True(t, pred.Metadata.Completeness.Environment) + require.True(t, pred.Metadata.Completeness.Materials) + require.False(t, pred.Metadata.Reproducible) + + if mode == "max" { + require.Equal(t, 2, len(pred.Layers)) + require.NotNil(t, pred.Source) + require.Equal(t, "Dockerfile", pred.Source.Infos[0].Filename) + require.Equal(t, dockerfile, pred.Source.Infos[0].Data) + require.NotNil(t, pred.BuildConfig) + + bc, ok := pred.BuildConfig.(map[string]interface{}) + require.True(t, ok, "wrong type %T", pred.BuildConfig) + + llb, ok := bc["llbDefinition"].([]interface{}) + require.True(t, ok, "wrong buildconfig %+v", bc) + + require.Equal(t, 3, len(llb)) + } else { + require.Equal(t, 0, len(pred.Layers)) + require.Nil(t, pred.Source) + require.Nil(t, pred.BuildConfig) + } + }) + } +} + func runShell(dir string, cmds ...string) error { for _, args := range cmds { var cmd *exec.Cmd From 4e8afd52188f06ffa993cd7c7b50b58ca78f16f4 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 30 Oct 2022 23:19:14 -0700 Subject: [PATCH 7/9] dockerfile: add test for provenance attestation from git Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile_test.go | 108 +++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 9eec347f062d..f83513e353be 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -150,6 +150,7 @@ var allTests = integration.TestFuncs( testSourceDateEpochWithoutExporter, testSBOMScannerImage, testProvenanceAttestation, + testGitProvenanceAttestation, ) // Tests that depend on the `security.*` entitlements @@ -6303,6 +6304,113 @@ RUN echo "ok" > /foo } } +func testGitProvenanceAttestation(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +RUN echo "git" > /foo +COPY myapp.Dockerfile / +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("myapp.Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + err = runShell(dir, + "git init", + "git config --local user.email test", + "git config --local user.name test", + "git add myapp.Dockerfile", + "git commit -m initial", + "git branch v1", + "git update-server-info", + ) + require.NoError(t, err) + + cmd := exec.Command("git", "rev-parse", "v1") + cmd.Dir = dir + expectedGitSHA, err := cmd.Output() + require.NoError(t, err) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(dir)))) + defer server.Close() + + target := registry + "/buildkit/testwithprovenance:git" + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context": server.URL + "/.git#v1", + "attest:provenance": "", + "filename": "myapp.Dockerfile", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + require.NotNil(t, img) + require.Equal(t, []byte("git\n"), img.Layers[1]["foo"].Data) + + att := imgs.Find("unknown/unknown") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + switch f.(type) { + case *clientFrontend: + // TODO: buildinfo broken + default: + require.Equal(t, server.URL+"/.git#v1", pred.Invocation.ConfigSource.URI) + require.Equal(t, "myapp.Dockerfile", pred.Invocation.ConfigSource.EntryPoint) + } + + require.Equal(t, 2, len(pred.Materials)) + require.Equal(t, "pkg:docker/busybox@latest", pred.Materials[0].URI) + require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) + + require.Equal(t, strings.Replace(server.URL+"/.git#v1", "http://", "https://", 1), pred.Materials[1].URI) // TODO: buildinfo broken + require.Equal(t, strings.TrimSpace(string(expectedGitSHA)), pred.Materials[1].Digest["sha1"]) +} + func runShell(dir string, cmds ...string) error { for _, args := range cmds { var cmd *exec.Cmd From 9acc6d30eb339d0fa6d3df94db450b693800f4a2 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 6 Sep 2022 23:30:58 -0700 Subject: [PATCH 8/9] refactor buildinfo into provenance capture Change how provenance information is captured from builds. While previously frontend passed the buildinfo sources with metadata, now all information is captured through buildkit. A frontend does not need to implement buildinfo and can't set incorrect/incomplete buildinfo for a build result. All LLB operations can now collect as much provenance info as they like that will be used when making the attestation. Previously this was limited to a single Pin value. For example now we also detect secrets and SSH IDs that the build uses, or if it accesses network, if local sources are used etc.. The new design makes sure this can be easily extended in the future. Provenance capture can now detect builds that do multiple separate subsolves in sequence. For example, first subsolve gathers the sources for the build and second one builds from immutable sources without a network connection. If first solve does not participate in final build result it does not end up in provenance. Signed-off-by: Tonis Tiigi --- client/build_test.go | 63 -- client/client_test.go | 90 +- control/control.go | 9 +- examples/dockerfile2llb/main.go | 7 +- frontend/attestations/parse.go | 12 +- frontend/dockerfile/builder/build.go | 96 +- frontend/dockerfile/dockerfile2llb/convert.go | 107 +-- .../dockerfile/dockerfile2llb/convert_test.go | 51 +- .../dockerfile/dockerfile_buildinfo_test.go | 51 +- .../dockerfile/dockerfile_provenance_test.go | 852 ++++++++++++++++++ frontend/dockerfile/dockerfile_test.go | 260 +----- frontend/gateway/gateway.go | 21 - go.mod | 2 +- go.sum | 4 +- solver/jobs.go | 61 +- solver/llbsolver/bridge.go | 105 +-- solver/llbsolver/ops/build.go | 20 +- solver/llbsolver/ops/diff.go | 4 +- solver/llbsolver/ops/exec.go | 39 +- solver/llbsolver/ops/file.go | 35 +- solver/llbsolver/ops/merge.go | 4 +- solver/llbsolver/ops/opsutils/contenthash.go | 71 ++ solver/llbsolver/ops/opsutils/validate.go | 63 ++ solver/llbsolver/ops/source.go | 43 +- solver/llbsolver/proc/provenance.go | 35 +- solver/llbsolver/proc/refs.go | 9 +- solver/llbsolver/proc/sbom.go | 6 +- solver/llbsolver/provenance.go | 342 +++++++ .../llbsolver}/provenance/buildconfig.go | 7 +- solver/llbsolver/provenance/capture.go | 250 +++++ solver/llbsolver/provenance/predicate.go | 248 +++++ solver/llbsolver/result.go | 75 +- solver/llbsolver/solver.go | 160 +++- solver/llbsolver/vertex.go | 60 +- solver/scheduler_test.go | 241 ++--- solver/types.go | 19 +- source/manager.go | 2 +- util/buildinfo/buildinfo.go | 123 ++- util/provenance/buildinfo.go | 164 ---- util/testutil/imageinfo.go | 13 + .../package-url/packageurl-go/.gitignore | 2 - .../package-url/packageurl-go/.golangci.yaml | 17 + .../package-url/packageurl-go/.travis.yml | 19 - .../packageurl-go/{mit.LICENSE => LICENSE} | 0 .../package-url/packageurl-go/README.md | 6 +- .../package-url/packageurl-go/packageurl.go | 86 +- vendor/modules.txt | 4 +- 47 files changed, 2610 insertions(+), 1348 deletions(-) create mode 100644 frontend/dockerfile/dockerfile_provenance_test.go create mode 100644 solver/llbsolver/ops/opsutils/contenthash.go create mode 100644 solver/llbsolver/ops/opsutils/validate.go create mode 100644 solver/llbsolver/provenance.go rename {util => solver/llbsolver}/provenance/buildconfig.go (95%) create mode 100644 solver/llbsolver/provenance/capture.go create mode 100644 solver/llbsolver/provenance/predicate.go delete mode 100644 util/provenance/buildinfo.go create mode 100644 vendor/github.com/package-url/packageurl-go/.golangci.yaml delete mode 100644 vendor/github.com/package-url/packageurl-go/.travis.yml rename vendor/github.com/package-url/packageurl-go/{mit.LICENSE => LICENSE} (100%) diff --git a/client/build_test.go b/client/build_test.go index 4ce5523533a7..3e1109603df3 100644 --- a/client/build_test.go +++ b/client/build_test.go @@ -3,8 +3,6 @@ package client import ( "bytes" "context" - "encoding/base64" - "encoding/json" "fmt" "io" "os" @@ -16,7 +14,6 @@ import ( "time" "github.com/moby/buildkit/client/llb" - "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/gateway/client" gatewayapi "github.com/moby/buildkit/frontend/gateway/pb" "github.com/moby/buildkit/identity" @@ -25,7 +22,6 @@ import ( "github.com/moby/buildkit/session/sshforward/sshprovider" "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" - binfotypes "github.com/moby/buildkit/util/buildinfo/types" "github.com/moby/buildkit/util/entitlements" utilsystem "github.com/moby/buildkit/util/system" "github.com/moby/buildkit/util/testutil/echoserver" @@ -58,7 +54,6 @@ func TestClientGatewayIntegration(t *testing.T) { testClientGatewayContainerExtraHosts, testClientGatewayContainerSignal, testWarnings, - testClientGatewayFrontendAttrs, testClientGatewayNilResult, testClientGatewayEmptyImageExec, ), integration.WithMirroredImages(integration.OfficialImages("busybox:latest"))) @@ -1995,64 +1990,6 @@ func testClientGatewayContainerSignal(t *testing.T, sb integration.Sandbox) { checkAllReleasable(t, c, sb, true) } -// moby/buildkit#2476 -func testClientGatewayFrontendAttrs(t *testing.T, sb integration.Sandbox) { - requiresLinux(t) - c, err := New(sb.Context(), sb.Address()) - require.NoError(t, err) - defer c.Close() - - fooattrval := "bar" - bazattrval := "fuu" - - b := func(ctx context.Context, c client.Client) (*client.Result, error) { - st := llb.Image("busybox:latest").Run( - llb.ReadonlyRootFS(), - llb.Args([]string{"/bin/sh", "-c", `echo hello`}), - ) - def, err := st.Marshal(sb.Context()) - if err != nil { - return nil, err - } - res, err := c.Solve(ctx, client.SolveRequest{ - Definition: def.ToPB(), - FrontendOpt: map[string]string{ - "build-arg:foo": fooattrval, - }, - }) - require.NoError(t, err) - require.Contains(t, res.Metadata, exptypes.ExporterBuildInfo) - - var bi binfotypes.BuildInfo - require.NoError(t, json.Unmarshal(res.Metadata[exptypes.ExporterBuildInfo], &bi)) - require.Contains(t, bi.Attrs, "build-arg:foo") - bi.Attrs["build-arg:baz"] = &bazattrval - - bmbi, err := json.Marshal(bi) - require.NoError(t, err) - - res.AddMeta(exptypes.ExporterBuildInfo, bmbi) - return res, err - } - - res, err := c.Build(sb.Context(), SolveOpt{}, "", b, nil) - require.NoError(t, err) - - require.Contains(t, res.ExporterResponse, exptypes.ExporterBuildInfo) - decbi, err := base64.StdEncoding.DecodeString(res.ExporterResponse[exptypes.ExporterBuildInfo]) - require.NoError(t, err) - - var bi binfotypes.BuildInfo - require.NoError(t, json.Unmarshal(decbi, &bi)) - - require.Contains(t, bi.Attrs, "build-arg:foo") - require.Equal(t, &fooattrval, bi.Attrs["build-arg:foo"]) - require.Contains(t, bi.Attrs, "build-arg:baz") - require.Equal(t, &bazattrval, bi.Attrs["build-arg:baz"]) - - checkAllReleasable(t, c, sb, true) -} - func testClientGatewayNilResult(t *testing.T, sb integration.Sandbox) { requiresLinux(t) c, err := New(sb.Context(), sb.Address()) diff --git a/client/client_test.go b/client/client_test.go index 7cb205c716d3..137c20544cd8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -6199,8 +6199,7 @@ func testBuildInfoExporter(t *testing.T, sb integration.Sandbox) { return nil, err } return c.Solve(ctx, gateway.SolveRequest{ - Definition: def.ToPB(), - FrontendOpt: map[string]string{"build-arg:foo": "bar"}, + Definition: def.ToPB(), }) } @@ -6233,8 +6232,6 @@ func testBuildInfoExporter(t *testing.T, sb integration.Sandbox) { err = json.Unmarshal(decbi, &exbi) require.NoError(t, err) - attrval := "bar" - require.Equal(t, exbi.Attrs, map[string]*string{"build-arg:foo": &attrval}) require.Equal(t, len(exbi.Sources), 1) require.Equal(t, exbi.Sources[0].Type, binfotypes.SourceTypeDockerImage) require.Equal(t, exbi.Sources[0].Ref, "docker.io/library/busybox:latest") @@ -6271,66 +6268,42 @@ func testBuildInfoInline(t *testing.T, sb integration.Sandbox) { ctx := namespaces.WithNamespace(sb.Context(), "buildkit") - for _, tt := range []struct { - name string - buildAttrs bool - }{{ - "attrsEnabled", - true, - }, { - "attrsDisabled", - false, - }} { - t.Run(tt.name, func(t *testing.T) { - target := registry + "/buildkit/test-buildinfo:latest" - - _, err = c.Solve(sb.Context(), def, SolveOpt{ - Exports: []ExportEntry{ - { - Type: ExporterImage, - Attrs: map[string]string{ - "name": target, - "push": "true", - "buildinfo-attrs": strconv.FormatBool(tt.buildAttrs), - }, - }, - }, - FrontendAttrs: map[string]string{ - "build-arg:foo": "bar", + target := registry + "/buildkit/test-buildinfo:latest" + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", }, - }, nil) - require.NoError(t, err) + }, + }, + }, nil) + require.NoError(t, err) - img, err := client.GetImage(ctx, target) - require.NoError(t, err) + img, err := client.GetImage(ctx, target) + require.NoError(t, err) - desc, err := img.Config(ctx) - require.NoError(t, err) + desc, err := img.Config(ctx) + require.NoError(t, err) - dt, err := content.ReadBlob(ctx, img.ContentStore(), desc) - require.NoError(t, err) + dt, err := content.ReadBlob(ctx, img.ContentStore(), desc) + require.NoError(t, err) - var config binfotypes.ImageConfig - require.NoError(t, json.Unmarshal(dt, &config)) + var config binfotypes.ImageConfig + require.NoError(t, json.Unmarshal(dt, &config)) - dec, err := base64.StdEncoding.DecodeString(config.BuildInfo) - require.NoError(t, err) + dec, err := base64.StdEncoding.DecodeString(config.BuildInfo) + require.NoError(t, err) - var bi binfotypes.BuildInfo - require.NoError(t, json.Unmarshal(dec, &bi)) + var bi binfotypes.BuildInfo + require.NoError(t, json.Unmarshal(dec, &bi)) - if tt.buildAttrs { - attrval := "bar" - require.Contains(t, bi.Attrs, "build-arg:foo") - require.Equal(t, bi.Attrs["build-arg:foo"], &attrval) - } else { - require.NotContains(t, bi.Attrs, "build-arg:foo") - } - require.Equal(t, len(bi.Sources), 1) - require.Equal(t, bi.Sources[0].Type, binfotypes.SourceTypeDockerImage) - require.Equal(t, bi.Sources[0].Ref, "docker.io/library/busybox:latest") - }) - } + require.Equal(t, len(bi.Sources), 1) + require.Equal(t, bi.Sources[0].Type, binfotypes.SourceTypeDockerImage) + require.Equal(t, bi.Sources[0].Ref, "docker.io/library/busybox:latest") } func testBuildInfoNoExport(t *testing.T, sb integration.Sandbox) { @@ -6348,8 +6321,7 @@ func testBuildInfoNoExport(t *testing.T, sb integration.Sandbox) { return nil, err } return c.Solve(ctx, gateway.SolveRequest{ - Definition: def.ToPB(), - FrontendOpt: map[string]string{"build-arg:foo": "bar"}, + Definition: def.ToPB(), }) } @@ -6364,8 +6336,6 @@ func testBuildInfoNoExport(t *testing.T, sb integration.Sandbox) { err = json.Unmarshal(decbi, &exbi) require.NoError(t, err) - attrval := "bar" - require.Equal(t, exbi.Attrs, map[string]*string{"build-arg:foo": &attrval}) require.Equal(t, len(exbi.Sources), 1) require.Equal(t, exbi.Sources[0].Type, binfotypes.SourceTypeDockerImage) require.Equal(t, exbi.Sources[0].Ref, "docker.io/library/busybox:latest") diff --git a/control/control.go b/control/control.go index 1fb35eb5a909..591dac90b6d6 100644 --- a/control/control.go +++ b/control/control.go @@ -330,6 +330,11 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* } var procs []llbsolver.Processor + + if len(attests) > 0 { + procs = append(procs, proc.ForceRefsProcessor) + } + if attrs, ok := attests["sbom"]; ok { src := attrs["generator"] if src == "" { @@ -340,11 +345,11 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* return nil, errors.Wrapf(err, "failed to parse sbom generator %s", src) } ref = reference.TagNameOnly(ref) - procs = append(procs, proc.ForceRefsProcessor, proc.SBOMProcessor(ref.String())) + procs = append(procs, proc.SBOMProcessor(ref.String())) } if attrs, ok := attests["provenance"]; ok { - procs = append(procs, proc.ForceRefsProcessor, proc.ProvenanceProcessor(attrs)) + procs = append(procs, proc.ProvenanceProcessor(attrs)) } resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{ diff --git a/examples/dockerfile2llb/main.go b/examples/dockerfile2llb/main.go index 224aee47f898..b575765a41f0 100644 --- a/examples/dockerfile2llb/main.go +++ b/examples/dockerfile2llb/main.go @@ -41,7 +41,7 @@ func xmain() error { caps := pb.Caps.CapSet(pb.Caps.All()) - state, img, bi, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{ + state, img, err := dockerfile2llb.Dockerfile2LLB(appcontext.Context(), df, dockerfile2llb.ConvertOpt{ MetaResolver: imagemetaresolver.Default(), Target: opt.target, LLBCaps: &caps, @@ -62,11 +62,6 @@ func xmain() error { return err } } - if opt.partialMetadataFile != "" { - if err := writeJSON(opt.partialMetadataFile, bi); err != nil { - return err - } - } return nil } diff --git a/frontend/attestations/parse.go b/frontend/attestations/parse.go index b3eb35c07616..b5740a964aaa 100644 --- a/frontend/attestations/parse.go +++ b/frontend/attestations/parse.go @@ -32,6 +32,15 @@ func Filter(v map[string]string) map[string]string { return attests } +func Validate(values map[string]map[string]string) (map[string]map[string]string, error) { + for k := range values { + if k != KeyTypeSbom && k != KeyTypeProvenance { + return nil, errors.Errorf("unknown attestation type %q", k) + } + } + return values, nil +} + func Parse(values map[string]string) (map[string]map[string]string, error) { attests := make(map[string]string) for k, v := range values { @@ -68,5 +77,6 @@ func Parse(values map[string]string) (map[string]map[string]string, error) { attrs[parts[0]] = parts[1] } } - return out, nil + + return Validate(out) } diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index dd318136e2f3..81c690d0aa20 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -32,7 +32,6 @@ import ( "github.com/moby/buildkit/frontend/subrequests/targets" "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" - binfotypes "github.com/moby/buildkit/util/buildinfo/types" "github.com/moby/buildkit/util/gitutil" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -524,7 +523,7 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { opt.Warn = nil } opt.ContextByName = contextByNameFunc(c, c.BuildOpts().SessionID) - st, img, bi, err := dockerfile2llb.Dockerfile2LLB(ctx2, dtDockerfile, opt) + st, img, err := dockerfile2llb.Dockerfile2LLB(ctx2, dtDockerfile, opt) if err != nil { return err @@ -579,11 +578,6 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { return err } - buildinfo, err := json.Marshal(bi) - if err != nil { - return errors.Wrapf(err, "failed to marshal build info") - } - p := platforms.DefaultSpec() if tp != nil { p = *tp @@ -593,7 +587,6 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { if !exportMap { res.AddMeta(exptypes.ExporterImageConfigKey, config) - res.AddMeta(exptypes.ExporterBuildInfo, buildinfo) res.SetRef(ref) expPlatforms.Platforms[i] = exptypes.Platform{ @@ -602,7 +595,6 @@ func Build(ctx context.Context, c client.Client) (_ *client.Result, err error) { } } else { res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, k), config) - res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k), buildinfo) res.AddRef(k, ref) expPlatforms.Platforms[i] = exptypes.Platform{ ID: k, @@ -871,11 +863,11 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } -func contextByNameFunc(c client.Client, sessionID string) func(context.Context, string, string, *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { - return func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { +func contextByNameFunc(c client.Client, sessionID string) func(context.Context, string, string, *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) { + return func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) { named, err := reference.ParseNormalizedNamed(name) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name) + return nil, nil, errors.Wrapf(err, "invalid context name %s", name) } name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") @@ -885,28 +877,28 @@ func contextByNameFunc(c client.Client, sessionID string) func(context.Context, } if p != nil { name := name + "::" + platforms.Format(platforms.Normalize(*p)) - st, img, bi, err := contextByName(ctx, c, sessionID, name, p, resolveMode) + st, img, err := contextByName(ctx, c, sessionID, name, p, resolveMode) if err != nil { - return nil, nil, nil, err + return nil, nil, err } if st != nil { - return st, img, bi, nil + return st, img, nil } } return contextByName(ctx, c, sessionID, name, p, resolveMode) } } -func contextByName(ctx context.Context, c client.Client, sessionID, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) { +func contextByName(ctx context.Context, c client.Client, sessionID, name string, platform *ocispecs.Platform, resolveMode string) (*llb.State, *dockerfile2llb.Image, error) { opts := c.BuildOpts().Opts v, ok := opts[contextPrefix+name] if !ok { - return nil, nil, nil, nil + return nil, nil, nil } vv := strings.SplitN(v, ":", 2) if len(vv) != 2 { - return nil, nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) } // allow git@ without protocol for SSH URLs for backwards compatibility if strings.HasPrefix(vv[0], "git@") { @@ -917,7 +909,7 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, ref := strings.TrimPrefix(vv[1], "//") if ref == "scratch" { st := llb.Scratch() - return &st, nil, nil, nil + return &st, nil, nil } imgOpt := []llb.ImageOption{ @@ -929,7 +921,7 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, named, err := reference.ParseNormalizedNamed(ref) if err != nil { - return nil, nil, nil, err + return nil, nil, err } named = reference.TagNameOnly(named) @@ -942,45 +934,45 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, SessionID: sessionID, }) if err != nil { - return nil, nil, nil, err + return nil, nil, err } var img dockerfile2llb.Image if err := json.Unmarshal(data, &img); err != nil { - return nil, nil, nil, err + return nil, nil, err } img.Created = nil st := llb.Image(ref, imgOpt...) st, err = st.WithImageConfig(data) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return &st, &img, nil, nil + return &st, &img, nil case "git": st, ok := detectGitContext(v, true) if !ok { - return nil, nil, nil, errors.Errorf("invalid git context %s", v) + return nil, nil, errors.Errorf("invalid git context %s", v) } - return st, nil, nil, nil + return st, nil, nil case "http", "https": st, ok := detectGitContext(v, true) if !ok { httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) st = &httpst } - return st, nil, nil, nil + return st, nil, nil case "oci-layout": ref := strings.TrimPrefix(vv[1], "//") // expected format is storeID@hash parts := strings.SplitN(ref, "@", 2) if len(parts) != 2 { - return nil, nil, nil, errors.Errorf("invalid oci-layout format '%s', must be oci-layout:///content-store@sha256:digest", vv[1]) + return nil, nil, errors.Errorf("invalid oci-layout format '%s', must be oci-layout:///content-store@sha256:digest", vv[1]) } storeID := parts[0] dig, err := digest.Parse(parts[1]) if err != nil { - return nil, nil, nil, errors.Errorf("invalid digest format '%s', must be oci-layout:///content-store@sha256:digest", vv[1]) + return nil, nil, errors.Errorf("invalid digest format '%s', must be oci-layout:///content-store@sha256:digest", vv[1]) } // the ref now is "content-store@sha256:digest" @@ -1001,12 +993,12 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, ResolverType: llb.ResolverTypeOCILayout, }) if err != nil { - return nil, nil, nil, err + return nil, nil, err } var img dockerfile2llb.Image if err := json.Unmarshal(data, &img); err != nil { - return nil, nil, nil, err + return nil, nil, err } st := llb.OCILayout(storeID, dig, @@ -1015,9 +1007,9 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, ) st, err = st.WithImageConfig(data) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return &st, &img, nil, nil + return &st, &img, nil case "local": st := llb.Local(vv[1], llb.SessionID(c.BuildOpts().SessionID), @@ -1028,18 +1020,18 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, ) def, err := st.Marshal(ctx) if err != nil { - return nil, nil, nil, err + return nil, nil, err } res, err := c.Solve(ctx, client.SolveRequest{ Evaluate: true, Definition: def.ToPB(), }) if err != nil { - return nil, nil, nil, err + return nil, nil, err } ref, err := res.SingleRef() if err != nil { - return nil, nil, nil, err + return nil, nil, err } dt, _ := ref.ReadFile(ctx, client.ReadRequest{ Filename: dockerignoreFilename, @@ -1048,7 +1040,7 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, if len(dt) != 0 { excludes, err = dockerignore.ReadAll(bytes.NewBuffer(dt)) if err != nil { - return nil, nil, nil, err + return nil, nil, err } } st = llb.Local(vv[1], @@ -1057,49 +1049,37 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string, llb.SharedKeyHint("context:"+name), llb.ExcludePatterns(excludes), ) - return &st, nil, nil, nil + return &st, nil, nil case "input": inputs, err := c.Inputs(ctx) if err != nil { - return nil, nil, nil, err + return nil, nil, err } st, ok := inputs[vv[1]] if !ok { - return nil, nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) } md, ok := opts[inputMetadataPrefix+vv[1]] if ok { m := make(map[string][]byte) if err := json.Unmarshal([]byte(md), &m); err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) - } - var bi *binfotypes.BuildInfo - if dtbi, ok := m[exptypes.ExporterBuildInfo]; ok { - var depbi binfotypes.BuildInfo - if err := json.Unmarshal(dtbi, &depbi); err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to parse buildinfo for %s", name) - } - bi = &binfotypes.BuildInfo{ - Deps: map[string]binfotypes.BuildInfo{ - strings.SplitN(vv[1], "::", 2)[0]: depbi, - }, - } + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) } var img *dockerfile2llb.Image if dtic, ok := m[exptypes.ExporterImageConfigKey]; ok { st, err = st.WithImageConfig(dtic) if err != nil { - return nil, nil, nil, err + return nil, nil, err } if err := json.Unmarshal(dtic, &img); err != nil { - return nil, nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) } } - return &st, img, bi, nil + return &st, img, nil } - return &st, nil, nil, nil + return &st, nil, nil default: - return nil, nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) } } diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 427d13341569..e41a89c05dd2 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -13,7 +13,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/containerd/containerd/platforms" @@ -75,19 +74,19 @@ type ConvertOpt struct { Hostname string SourceDateEpoch *time.Time Warn func(short, url string, detail [][]byte, location *parser.Range) - ContextByName func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error) + ContextByName func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, error) } -func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) { - ds, bi, err := toDispatchState(ctx, dt, opt) +func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { + ds, err := toDispatchState(ctx, dt, opt) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return &ds.state, &ds.image, bi, nil + return &ds.state, &ds.image, nil } func Dockefile2Outline(ctx context.Context, dt []byte, opt ConvertOpt) (*outline.Outline, error) { - ds, _, err := toDispatchState(ctx, dt, opt) + ds, err := toDispatchState(ctx, dt, opt) if err != nil { return nil, err } @@ -123,38 +122,26 @@ func ListTargets(ctx context.Context, dt []byte) (*targets.List, error) { return l, nil } -func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchState, *binfotypes.BuildInfo, error) { - buildInfo := &binfotypes.BuildInfo{} - buildInfoDepsMu := sync.Mutex{} +func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchState, error) { contextByName := opt.ContextByName - opt.ContextByName = func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error) { + opt.ContextByName = func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, error) { if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") { if contextByName != nil { if p == nil { p = opt.TargetPlatform } - st, img, bi, err := contextByName(ctx, name, resolveMode, p) + st, img, err := contextByName(ctx, name, resolveMode, p) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - if bi != nil && bi.Deps != nil { - buildInfoDepsMu.Lock() - if buildInfo.Deps == nil { - buildInfo.Deps = make(map[string]binfotypes.BuildInfo) - } - for k := range bi.Deps { - buildInfo.Deps[k] = bi.Deps[k] - } - buildInfoDepsMu.Unlock() - } - return st, img, bi, nil + return st, img, nil } } - return nil, nil, nil, nil + return nil, nil, nil } if len(dt) == 0 { - return nil, nil, errors.Errorf("the Dockerfile cannot be empty") + return nil, errors.Errorf("the Dockerfile cannot be empty") } if opt.ContextLocalName == "" { @@ -170,7 +157,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS dockerfile, err := parser.Parse(bytes.NewReader(dt)) if err != nil { - return nil, nil, err + return nil, err } for _, w := range dockerfile.Warnings { @@ -181,7 +168,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS stages, metaArgs, err := instructions.Parse(dockerfile.AST) if err != nil { - return nil, nil, err + return nil, err } shlex := shell.NewLex(dockerfile.EscapeToken) @@ -216,10 +203,10 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS for i, st := range stages { name, used, err := shlex.ProcessWordWithMatches(st.BaseName, metaArgsToMap(optMetaArgs)) if err != nil { - return nil, nil, parser.WithLocation(err, st.Location) + return nil, parser.WithLocation(err, st.Location) } if name == "" { - return nil, nil, parser.WithLocation(errors.Errorf("base name (%s) should not be blank", st.BaseName), st.Location) + return nil, parser.WithLocation(errors.Errorf("base name (%s) should not be blank", st.BaseName), st.Location) } st.BaseName = name @@ -236,12 +223,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS if v := st.Platform; v != "" { v, u, err := shlex.ProcessWordWithMatches(v, metaArgsToMap(optMetaArgs)) if err != nil { - return nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location) + return nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location) } p, err := platforms.Parse(v) if err != nil { - return nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location) + return nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location) } for k := range u { used[k] = struct{}{} @@ -250,9 +237,9 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if st.Name != "" { - s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String(), ds.platform) + s, img, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String(), ds.platform) if err != nil { - return nil, nil, err + return nil, err } if s != nil { ds.noinit = true @@ -267,9 +254,6 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } } } - if bi != nil { - ds.buildInfo = *bi - } allDispatchStates.addState(ds) continue } @@ -319,7 +303,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS var ok bool target, ok = allDispatchStates.findStateByName(opt.Target) if !ok { - return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target) + return nil, errors.Errorf("target stage %s could not be found", opt.Target) } } @@ -329,7 +313,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS for i, cmd := range d.stage.Commands { newCmd, err := toCommand(cmd, allDispatchStates) if err != nil { - return nil, nil, err + return nil, err } d.commands[i] = newCmd for _, src := range newCmd.sources { @@ -344,7 +328,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if has, state := hasCircularDependency(allDispatchStates.states); has { - return nil, nil, errors.Errorf("circular dependency detected on stage: %s", state.stageName) + return nil, errors.Errorf("circular dependency detected on stage: %s", state.stageName) } if len(allDispatchStates.states) == 1 { @@ -387,7 +371,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS d.stage.BaseName = reference.TagNameOnly(ref).String() var isScratch bool - st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String(), platform) + st, img, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String(), platform) if err != nil { return err } @@ -397,9 +381,6 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } else { d.image = emptyImage(platformOpt.targetPlatform) } - if bi != nil { - d.buildInfo = *bi - } d.state = st.Platform(*platform) d.platform = platform return nil @@ -477,7 +458,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if err := eg.Wait(); err != nil { - return nil, nil, err + return nil, err } buildContext := &mutableOutput{} @@ -488,19 +469,6 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS continue } - // collect build sources and dependencies - if len(d.buildInfo.Sources) > 0 { - buildInfo.Sources = append(buildInfo.Sources, d.buildInfo.Sources...) - } - if d.buildInfo.Deps != nil { - for name, bi := range d.buildInfo.Deps { - if buildInfo.Deps == nil { - buildInfo.Deps = make(map[string]binfotypes.BuildInfo) - } - buildInfo.Deps[name] = bi - } - } - if d.base != nil { d.state = d.base.state d.platform = d.base.platform @@ -509,11 +477,11 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS // make sure that PATH is always set if _, ok := shell.BuildEnvs(d.image.Config.Env)["PATH"]; !ok { - var os string + var pathOS string if d.platform != nil { - os = d.platform.OS + pathOS = d.platform.OS } - d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(os)) + d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(pathOS)) } // initialize base metadata from image conf @@ -526,12 +494,12 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if d.image.Config.WorkingDir != "" { if err = dispatchWorkdir(d, &instructions.WorkdirCommand{Path: d.image.Config.WorkingDir}, false, nil); err != nil { - return nil, nil, parser.WithLocation(err, d.stage.Location) + return nil, parser.WithLocation(err, d.stage.Location) } } if d.image.Config.User != "" { if err = dispatchUser(d, &instructions.UserCommand{User: d.image.Config.User}, false); err != nil { - return nil, nil, parser.WithLocation(err, d.stage.Location) + return nil, parser.WithLocation(err, d.stage.Location) } } d.state = d.state.Network(opt.ForceNetMode) @@ -556,13 +524,13 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if err = dispatchOnBuildTriggers(d, d.image.Config.OnBuild, opt); err != nil { - return nil, nil, parser.WithLocation(err, d.stage.Location) + return nil, parser.WithLocation(err, d.stage.Location) } d.image.Config.OnBuild = nil for _, cmd := range d.commands { if err := dispatch(d, cmd, opt); err != nil { - return nil, nil, parser.WithLocation(err, cmd.Location()) + return nil, parser.WithLocation(err, cmd.Location()) } } @@ -571,13 +539,6 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } } - // sort build sources - if len(buildInfo.Sources) > 0 { - sort.Slice(buildInfo.Sources, func(i, j int) bool { - return buildInfo.Sources[i].Ref < buildInfo.Sources[j].Ref - }) - } - if len(opt.Labels) != 0 && target.image.Config.Labels == nil { target.image.Config.Labels = make(map[string]string, len(opt.Labels)) } @@ -615,7 +576,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS target.image.Variant = platformOpt.targetPlatform.Variant } - return target, buildInfo, nil + return target, nil } func metaArgsToMap(metaArgs []instructions.KeyValuePairOptional) map[string]string { diff --git a/frontend/dockerfile/dockerfile2llb/convert_test.go b/frontend/dockerfile/dockerfile2llb/convert_test.go index 5c1817addf91..8fe1aa6e2ad8 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_test.go +++ b/frontend/dockerfile/dockerfile2llb/convert_test.go @@ -1,16 +1,12 @@ package dockerfile2llb import ( - "strings" "testing" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/buildkit/util/appcontext" - binfotypes "github.com/moby/buildkit/util/buildinfo/types" - ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func toEnvMap(args []instructions.KeyValuePairOptional, env []string) map[string]string { @@ -35,7 +31,7 @@ ENV FOO bar COPY f1 f2 /sub/ RUN ls -l ` - _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.NoError(t, err) df = `FROM scratch AS foo @@ -44,7 +40,7 @@ FROM foo COPY --from=foo f1 / COPY --from=0 f2 / ` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.NoError(t, err) df = `FROM scratch AS foo @@ -53,12 +49,12 @@ FROM foo COPY --from=foo f1 / COPY --from=0 f2 / ` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ Target: "Foo", }) assert.NoError(t, err) - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ Target: "nosuch", }) assert.Error(t, err) @@ -66,21 +62,21 @@ COPY --from=0 f2 / df = `FROM scratch ADD http://github.com/moby/buildkit/blob/master/README.md / ` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.NoError(t, err) df = `FROM scratch COPY http://github.com/moby/buildkit/blob/master/README.md / ` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.EqualError(t, err, "source can't be a URL for COPY") df = `FROM "" AS foo` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.Error(t, err) df = `FROM ${BLANK} AS foo` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.Error(t, err) } @@ -178,7 +174,7 @@ func TestDockerfileCircularDependencies(t *testing.T) { df := `FROM busybox AS stage0 COPY --from=stage0 f1 /sub/ ` - _, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.EqualError(t, err, "circular dependency detected on stage: stage0") // multiple stages with circular dependency @@ -189,33 +185,6 @@ COPY --from=stage0 f2 /sub/ FROM busybox AS stage2 COPY --from=stage1 f2 /sub/ ` - _, _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) + _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.EqualError(t, err, "circular dependency detected on stage: stage0") } - -// moby/buildkit#2311 -func TestTargetBuildInfo(t *testing.T) { - df := ` -FROM busybox -ADD https://raw.githubusercontent.com/moby/buildkit/master/README.md / -` - _, _, bi, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ - TargetPlatform: &ocispecs.Platform{ - Architecture: "amd64", - OS: "linux", - }, - BuildPlatforms: []ocispecs.Platform{ - { - Architecture: "amd64", - OS: "linux", - }, - }, - }) - require.NoError(t, err) - - require.Equal(t, 1, len(bi.Sources)) - assert.Equal(t, binfotypes.SourceTypeDockerImage, bi.Sources[0].Type) - assert.Equal(t, "busybox", bi.Sources[0].Ref) - assert.True(t, strings.HasPrefix(bi.Sources[0].Alias, "docker.io/library/busybox@")) - assert.NotEmpty(t, bi.Sources[0].Pin) -} diff --git a/frontend/dockerfile/dockerfile_buildinfo_test.go b/frontend/dockerfile/dockerfile_buildinfo_test.go index 4d68462a0f62..ac9460d2b66e 100644 --- a/frontend/dockerfile/dockerfile_buildinfo_test.go +++ b/frontend/dockerfile/dockerfile_buildinfo_test.go @@ -112,9 +112,16 @@ COPY --from=alpine /bin/busybox /alpine-busybox require.Contains(t, bi.Attrs, "context") require.Equal(t, server.URL+"/.git#buildinfo", *bi.Attrs["context"]) - sources := bi.Sources - require.Equal(t, 3, len(sources)) + _, isGateway := f.(*gatewayFrontend) + sources := bi.Sources + if isGateway { + require.Equal(t, 5, len(sources), "%+v", sources) + assert.Equal(t, binfotypes.SourceTypeDockerImage, sources[0].Type) + assert.Contains(t, sources[0].Ref, "buildkit_test") + sources = sources[1:] + } + require.Equal(t, 4, len(sources), "%+v", sources) assert.Equal(t, binfotypes.SourceTypeDockerImage, sources[0].Type) assert.Equal(t, "docker.io/library/alpine:latest@sha256:21a3deaa0d32a8057914f36584b5288d2e5ecc984380bc0118285c70fa8c9300", sources[0].Ref) assert.Equal(t, "sha256:21a3deaa0d32a8057914f36584b5288d2e5ecc984380bc0118285c70fa8c9300", sources[0].Pin) @@ -123,9 +130,13 @@ COPY --from=alpine /bin/busybox /alpine-busybox assert.Equal(t, "docker.io/library/busybox:latest", sources[1].Ref) assert.NotEmpty(t, sources[1].Pin) - assert.Equal(t, binfotypes.SourceTypeHTTP, sources[2].Type) - assert.Equal(t, "https://raw.githubusercontent.com/moby/moby/v20.10.21/README.md", sources[2].Ref) - assert.Equal(t, "sha256:419455202b0ef97e480d7f8199b26a721a417818bc0e2d106975f74323f25e6c", sources[2].Pin) + assert.Equal(t, binfotypes.SourceTypeGit, sources[2].Type) + assert.Equal(t, server.URL+"/.git#buildinfo", sources[2].Ref) + assert.NotEmpty(t, sources[2].Pin) + + assert.Equal(t, binfotypes.SourceTypeHTTP, sources[3].Type) + assert.Equal(t, "https://raw.githubusercontent.com/moby/moby/v20.10.21/README.md", sources[3].Ref) + assert.Equal(t, "sha256:419455202b0ef97e480d7f8199b26a721a417818bc0e2d106975f74323f25e6c", sources[3].Pin) } func testBuildInfoSourcesNoop(t *testing.T, sb integration.Sandbox) { @@ -180,8 +191,12 @@ FROM busybox:latest require.NoError(t, err) sources := bi.Sources - require.Equal(t, 1, len(sources)) + if _, isGateway := f.(*gatewayFrontend); isGateway { + require.Equal(t, 2, len(sources), "%+v", sources) + sources = sources[1:] + } + require.Equal(t, 1, len(sources)) assert.Equal(t, binfotypes.SourceTypeDockerImage, sources[0].Type) assert.Equal(t, "docker.io/library/busybox:latest", sources[0].Ref) assert.NotEmpty(t, sources[0].Pin) @@ -303,8 +318,14 @@ ADD https://raw.githubusercontent.com/moby/moby/v20.10.21/README.md / require.Contains(t, bi.Attrs, "build-arg:foo") require.Equal(t, "bar", *bi.Attrs["build-arg:foo"]) + _, isGateway := f.(*gatewayFrontend) + sources := bi.Sources - require.Equal(t, 2, len(sources)) + if isGateway { + require.Equal(t, 3, len(sources), "%+v", sources) + sources = sources[1:] + } + require.Equal(t, 2, len(sources), "%+v", sources) assert.Equal(t, binfotypes.SourceTypeDockerImage, sources[0].Type) assert.Equal(t, "docker.io/library/busybox:latest", sources[0].Ref) @@ -379,8 +400,15 @@ COPY --from=base /out / require.Contains(t, bi.Attrs, "build-arg:foo") require.Equal(t, "bar", *bi.Attrs["build-arg:foo"]) + _, isGateway := f.(*gatewayFrontend) + sources := bi.Sources - require.Equal(t, 1, len(sources)) + if isGateway { + require.Equal(t, 2, len(sources), "%+v", sources) + sources = sources[1:] + } else { + require.Equal(t, 1, len(sources)) + } assert.Equal(t, binfotypes.SourceTypeDockerImage, sources[0].Type) assert.Equal(t, "docker.io/library/alpine:latest", sources[0].Ref) assert.NotEmpty(t, sources[0].Pin) @@ -460,7 +488,12 @@ COPY --from=base /o* / require.Contains(t, bi.Attrs, "build-arg:foo") require.Equal(t, "bar", *bi.Attrs["build-arg:foo"]) - require.Equal(t, 0, len(bi.Sources)) + _, isGateway := f.(*gatewayFrontend) + if isGateway { + require.Equal(t, 1, len(bi.Sources)) + } else { + require.Equal(t, 0, len(bi.Sources)) + } } func testBuildInfoDeps(t *testing.T, sb integration.Sandbox) { diff --git a/frontend/dockerfile/dockerfile_provenance_test.go b/frontend/dockerfile/dockerfile_provenance_test.go new file mode 100644 index 000000000000..6729f7c1fcc6 --- /dev/null +++ b/frontend/dockerfile/dockerfile_provenance_test.go @@ -0,0 +1,852 @@ +package dockerfile + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/containerd/containerd/platforms" + "github.com/containerd/continuity/fs/fstest" + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend/dockerfile/builder" + gateway "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/util/contentutil" + "github.com/moby/buildkit/util/testutil" + "github.com/moby/buildkit/util/testutil/integration" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func testProvenanceAttestation(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +RUN echo "ok" > /foo +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + for _, mode := range []string{"min", "max"} { + t.Run(mode, func(t *testing.T) { + target := registry + "/buildkit/testwithprovenance:" + mode + provReq := "" + if mode == "max" { + provReq = "mode=max" + } + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + FrontendAttrs: map[string]string{ + "attest:provenance": provReq, + "build-arg:FOO": "bar", + "label:lbl": "abc", + "vcs:source": "https://example.invalid/repo.git", + "vcs:revision": "123456", + "filename": "Dockerfile", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + require.NotNil(t, img) + require.Equal(t, []byte("ok\n"), img.Layers[1]["foo"].Data) + + att := imgs.Find("unknown/unknown") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + + _, isClient := f.(*clientFrontend) + _, isGateway := f.(*gatewayFrontend) + + args := pred.Invocation.Parameters.Args + if isClient { + require.Equal(t, "", pred.Invocation.Parameters.Frontend) + require.Equal(t, 0, len(args), "%v", args) + require.False(t, pred.Metadata.Completeness.Parameters) + require.Equal(t, "", pred.Invocation.ConfigSource.EntryPoint) + } else if isGateway { + require.Equal(t, "gateway.v0", pred.Invocation.Parameters.Frontend) + + if mode == "max" { + require.Equal(t, 3, len(args), "%v", args) + require.True(t, pred.Metadata.Completeness.Parameters) + + require.Equal(t, "bar", args["build-arg:FOO"]) + require.Equal(t, "abc", args["label:lbl"]) + require.Contains(t, args["source"], "buildkit_test/") + } else { + require.False(t, pred.Metadata.Completeness.Parameters) + require.Equal(t, 1, len(args), "%v", args) + require.Contains(t, args["source"], "buildkit_test/") + } + } else { + require.Equal(t, "dockerfile.v0", pred.Invocation.Parameters.Frontend) + + if mode == "max" { + require.Equal(t, 2, len(args)) + require.True(t, pred.Metadata.Completeness.Parameters) + + require.Equal(t, "bar", args["build-arg:FOO"]) + require.Equal(t, "abc", args["label:lbl"]) + } else { + require.False(t, pred.Metadata.Completeness.Parameters) + require.Equal(t, 0, len(args), "%v", args) + } + } + + expectedBase := "pkg:docker/busybox@latest?platform=" + url.PathEscape(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + if isGateway { + require.Equal(t, 2, len(pred.Materials), "%+v", pred.Materials) + require.Contains(t, pred.Materials[0].URI, "docker/buildkit_test") + require.Equal(t, expectedBase, pred.Materials[1].URI) + require.NotEmpty(t, pred.Materials[1].Digest["sha256"]) + } else { + require.Equal(t, 1, len(pred.Materials), "%+v", pred.Materials) + require.Equal(t, expectedBase, pred.Materials[0].URI) + require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) + } + + if !isClient { + require.Equal(t, "Dockerfile", pred.Invocation.ConfigSource.EntryPoint) + require.Equal(t, "https://example.invalid/repo.git", pred.Metadata.BuildKitMetadata.VCS["source"]) + require.Equal(t, "123456", pred.Metadata.BuildKitMetadata.VCS["revision"]) + } + + require.NotEmpty(t, pred.Metadata.BuildInvocationID) + + require.Equal(t, 2, len(pred.Invocation.Parameters.Locals), "%+v", pred.Invocation.Parameters.Locals) + require.Equal(t, "context", pred.Invocation.Parameters.Locals[0].Name) + require.Equal(t, "dockerfile", pred.Invocation.Parameters.Locals[1].Name) + + require.NotNil(t, pred.Metadata.BuildFinishedOn) + require.True(t, time.Since(*pred.Metadata.BuildFinishedOn) < 5*time.Minute) + require.NotNil(t, pred.Metadata.BuildStartedOn) + require.True(t, time.Since(*pred.Metadata.BuildStartedOn) < 5*time.Minute) + require.True(t, pred.Metadata.BuildStartedOn.Before(*pred.Metadata.BuildFinishedOn)) + + require.True(t, pred.Metadata.Completeness.Environment) + require.Equal(t, platforms.Format(platforms.Normalize(platforms.DefaultSpec())), pred.Invocation.Environment.Platform) + + require.False(t, pred.Metadata.Completeness.Materials) + require.False(t, pred.Metadata.Reproducible) + require.False(t, pred.Metadata.Completeness.Hermetic) + + if mode == "max" { + require.Equal(t, 2, len(pred.Metadata.BuildKitMetadata.Layers)) + require.NotNil(t, pred.Metadata.BuildKitMetadata.Source) + require.Equal(t, "Dockerfile", pred.Metadata.BuildKitMetadata.Source.Infos[0].Filename) + require.Equal(t, dockerfile, pred.Metadata.BuildKitMetadata.Source.Infos[0].Data) + require.NotNil(t, pred.BuildConfig) + + require.Equal(t, 3, len(pred.BuildConfig.Definition)) + } else { + require.Equal(t, 0, len(pred.Metadata.BuildKitMetadata.Layers)) + require.Nil(t, pred.Metadata.BuildKitMetadata.Source) + require.Nil(t, pred.BuildConfig) + } + }) + } +} + +func testGitProvenanceAttestation(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +RUN --network=none echo "git" > /foo +COPY myapp.Dockerfile / +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("myapp.Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + err = runShell(dir, + "git init", + "git config --local user.email test", + "git config --local user.name test", + "git add myapp.Dockerfile", + "git commit -m initial", + "git branch v1", + "git update-server-info", + ) + require.NoError(t, err) + + cmd := exec.Command("git", "rev-parse", "v1") + cmd.Dir = dir + expectedGitSHA, err := cmd.Output() + require.NoError(t, err) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(dir)))) + defer server.Close() + + target := registry + "/buildkit/testwithprovenance:git" + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context": server.URL + "/.git#v1", + "attest:provenance": "", + "filename": "myapp.Dockerfile", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + require.NotNil(t, img) + require.Equal(t, []byte("git\n"), img.Layers[1]["foo"].Data) + + att := imgs.Find("unknown/unknown") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + _, isClient := f.(*clientFrontend) + _, isGateway := f.(*gatewayFrontend) + + if isClient { + require.Empty(t, pred.Invocation.Parameters.Frontend) + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + require.Equal(t, "", pred.Invocation.ConfigSource.EntryPoint) + } else { + require.NotEmpty(t, pred.Invocation.Parameters.Frontend) + require.Equal(t, server.URL+"/.git#v1", pred.Invocation.ConfigSource.URI) + require.Equal(t, "myapp.Dockerfile", pred.Invocation.ConfigSource.EntryPoint) + } + + expBase := "pkg:docker/busybox@latest?platform=" + url.PathEscape(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) + if isGateway { + require.Equal(t, 3, len(pred.Materials), "%+v", pred.Materials) + + require.Contains(t, pred.Materials[0].URI, "pkg:docker/buildkit_test/") + require.NotEmpty(t, pred.Materials[0].Digest) + + require.Equal(t, expBase, pred.Materials[1].URI) + require.NotEmpty(t, pred.Materials[1].Digest["sha256"]) + + require.Equal(t, server.URL+"/.git#v1", pred.Materials[2].URI) + require.Equal(t, strings.TrimSpace(string(expectedGitSHA)), pred.Materials[2].Digest["sha1"]) + } else { + require.Equal(t, 2, len(pred.Materials), "%+v", pred.Materials) + + require.Equal(t, expBase, pred.Materials[0].URI) + require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) + + require.Equal(t, server.URL+"/.git#v1", pred.Materials[1].URI) + require.Equal(t, strings.TrimSpace(string(expectedGitSHA)), pred.Materials[1].Digest["sha1"]) + } + + require.Equal(t, 0, len(pred.Invocation.Parameters.Locals)) + + require.True(t, pred.Metadata.Completeness.Materials) + require.True(t, pred.Metadata.Completeness.Hermetic) + require.True(t, pred.Metadata.Completeness.Environment) + + if isClient { + require.False(t, pred.Metadata.Completeness.Parameters) + } else { + require.True(t, pred.Metadata.Completeness.Parameters) + } + require.False(t, pred.Metadata.Reproducible) + + require.Equal(t, 0, len(pred.Metadata.BuildKitMetadata.VCS), "%+v", pred.Metadata.BuildKitMetadata.VCS) +} + +func testMultiPlatformProvenance(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +ARG TARGETARCH +RUN echo "ok-$TARGETARCH" > /foo +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + target := registry + "/buildkit/testmultiprovenance:latest" + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max", + "build-arg:FOO": "bar", + "label:lbl": "abc", + "platform": "linux/amd64,linux/arm64", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 4, len(imgs.Images)) + + _, isClient := f.(*clientFrontend) + _, isGateway := f.(*gatewayFrontend) + + for _, p := range []string{"linux/amd64", "linux/arm64"} { + img := imgs.Find(p) + require.NotNil(t, img) + if p == "linux/amd64" { + require.Equal(t, []byte("ok-amd64\n"), img.Layers[1]["foo"].Data) + } else { + require.Equal(t, []byte("ok-arm64\n"), img.Layers[1]["foo"].Data) + } + + att := imgs.FindAttestation(p) + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + + if isGateway { + require.Equal(t, 2, len(pred.Materials), "%+v", pred.Materials) + require.Contains(t, pred.Materials[0].URI, "buildkit_test") + require.Contains(t, pred.Materials[1].URI, "pkg:docker/busybox@latest") + require.Contains(t, pred.Materials[1].URI, url.PathEscape(p)) + } else { + require.Equal(t, 1, len(pred.Materials), "%+v", pred.Materials) + require.Contains(t, pred.Materials[0].URI, "pkg:docker/busybox@latest") + require.Contains(t, pred.Materials[0].URI, url.PathEscape(p)) + } + + args := pred.Invocation.Parameters.Args + if isClient { + require.Equal(t, 0, len(args), "%+v", args) + } else if isGateway { + require.Equal(t, 3, len(args), "%+v", args) + require.Equal(t, "bar", args["build-arg:FOO"]) + require.Equal(t, "abc", args["label:lbl"]) + require.Contains(t, args["source"], "buildkit_test/") + } else { + require.Equal(t, 2, len(args), "%+v", args) + require.Equal(t, "bar", args["build-arg:FOO"]) + require.Equal(t, "abc", args["label:lbl"]) + } + } +} + +func testClientFrontendProvenance(t *testing.T, sb integration.Sandbox) { + // Building with client frontend does not capture frontend provenance + // because frontend runs in client, not in BuildKit. + // This test builds Dockerfile inside a client frontend ensuring that + // in that case frontend provenance is captured. + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/clientprovenance:latest" + + f := getFrontend(t, sb) + + _, isClient := f.(*clientFrontend) + if !isClient { + t.Skip("not a client frontend") + } + + dockerfile := []byte(` + FROM alpine as x86target + RUN echo "alpine" > /foo + + FROM busybox:latest AS armtarget + RUN --network=none echo "bbox" > /foo + `) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := llb.HTTP("https://raw.githubusercontent.com/moby/moby/v20.10.21/README.md") + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + // This does not show up in provenance + res0, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + dt, err := res0.Ref.ReadFile(ctx, gateway.ReadRequest{ + Filename: "README.md", + }) + if err != nil { + return nil, err + } + + res1, err := c.Solve(ctx, gateway.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendOpt: map[string]string{ + "build-arg:FOO": string(dt[:3]), + "target": "armtarget", + }, + }) + if err != nil { + return nil, err + } + + res2, err := c.Solve(ctx, gateway.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendOpt: map[string]string{ + "build-arg:FOO": string(dt[4:8]), + "target": "x86target", + }, + }) + if err != nil { + return nil, err + } + + res := gateway.NewResult() + res.AddRef("linux/arm64", res1.Ref) + res.AddRef("linux/amd64", res2.Ref) + + pl, err := json.Marshal(exptypes.Platforms{ + Platforms: []exptypes.Platform{ + { + ID: "linux/arm64", + Platform: ocispecs.Platform{OS: "linux", Architecture: "arm64"}, + }, + { + ID: "linux/amd64", + Platform: ocispecs.Platform{OS: "linux", Architecture: "amd64"}, + }, + }, + }) + if err != nil { + return nil, err + } + res.AddMeta(exptypes.ExporterPlatformsKey, pl) + + return res, nil + } + + _, err = c.Build(sb.Context(), client.SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=full", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, "", frontend, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 4, len(imgs.Images)) + + img := imgs.Find("linux/arm64") + require.NotNil(t, img) + require.Equal(t, []byte("bbox\n"), img.Layers[1]["foo"].Data) + + att := imgs.FindAttestation("linux/arm64") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + + args := pred.Invocation.Parameters.Args + require.Equal(t, 2, len(args), "%+v", args) + require.Equal(t, "The", args["build-arg:FOO"]) + require.Equal(t, "armtarget", args["target"]) + + require.Equal(t, 2, len(pred.Invocation.Parameters.Locals)) + require.Equal(t, 1, len(pred.Materials)) + require.Contains(t, pred.Materials[0].URI, "docker/busybox") + + // amd64 + img = imgs.Find("linux/amd64") + require.NotNil(t, img) + require.Equal(t, []byte("alpine\n"), img.Layers[1]["foo"].Data) + + att = imgs.FindAttestation("linux/amd64") + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + attest = intoto.Statement{} + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + stmt = stmtT{} + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred = stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + + args = pred.Invocation.Parameters.Args + require.Equal(t, 2, len(args), "%+v", args) + require.Equal(t, "Moby", args["build-arg:FOO"]) + require.Equal(t, "x86target", args["target"]) + + require.Equal(t, 2, len(pred.Invocation.Parameters.Locals)) + require.Equal(t, 1, len(pred.Materials)) + require.Contains(t, pred.Materials[0].URI, "docker/alpine") +} + +func testClientLLBProvenance(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/clientprovenance:llb" + + f := getFrontend(t, sb) + + _, isClient := f.(*clientFrontend) + if !isClient { + t.Skip("not a client frontend") + } + + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := llb.HTTP("https://raw.githubusercontent.com/moby/moby/v20.10.21/README.md") + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + // this also shows up in the provenance + res0, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + dt, err := res0.Ref.ReadFile(ctx, gateway.ReadRequest{ + Filename: "README.md", + }) + if err != nil { + return nil, err + } + + st = llb.Image("alpine").File(llb.Mkfile("/foo", 0600, dt)) + def, err = st.Marshal(ctx) + if err != nil { + return nil, err + } + res1, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + return res1, nil + } + + _, err = c.Build(sb.Context(), client.SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=full", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + LocalDirs: map[string]string{}, + }, "", frontend, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + nativePlatform := platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + + img := imgs.Find(nativePlatform) + require.NotNil(t, img) + require.Contains(t, string(img.Layers[1]["foo"].Data), "The Moby Project") + + att := imgs.FindAttestation(nativePlatform) + require.NotNil(t, att) + require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") + var attest intoto.Statement + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const + + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) + require.Equal(t, "", pred.Builder.ID) + require.Equal(t, "", pred.Invocation.ConfigSource.URI) + + args := pred.Invocation.Parameters.Args + require.Equal(t, 0, len(args), "%+v", args) + require.Equal(t, 0, len(pred.Invocation.Parameters.Locals)) + + require.Equal(t, 2, len(pred.Materials), "%+v", pred.Materials) + require.Contains(t, pred.Materials[0].URI, "docker/alpine") + require.Contains(t, pred.Materials[1].URI, "README.md") +} + +func testSecretSSHProvenance(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM busybox:latest +RUN --mount=type=secret,id=mysecret --mount=type=secret,id=othersecret --mount=type=ssh echo "ok" > /foo +`) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + + target := registry + "/buildkit/testsecretprovenance:latest" + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max", + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + expPlatform := platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + + img := imgs.Find(expPlatform) + require.NotNil(t, img) + require.Equal(t, []byte("ok\n"), img.Layers[1]["foo"].Data) + + att := imgs.FindAttestation(expPlatform) + type stmtT struct { + Predicate provenance.ProvenancePredicate `json:"predicate"` + } + var stmt stmtT + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) + pred := stmt.Predicate + + require.Equal(t, 2, len(pred.Invocation.Parameters.Secrets), "%+v", pred.Invocation.Parameters.Secrets) + require.Equal(t, "mysecret", pred.Invocation.Parameters.Secrets[0].ID) + require.True(t, pred.Invocation.Parameters.Secrets[0].Optional) + require.Equal(t, "othersecret", pred.Invocation.Parameters.Secrets[1].ID) + require.True(t, pred.Invocation.Parameters.Secrets[1].Optional) + + require.Equal(t, 1, len(pred.Invocation.Parameters.SSH), "%+v", pred.Invocation.Parameters.SSH) + require.Equal(t, "default", pred.Invocation.Parameters.SSH[0].ID) + require.True(t, pred.Invocation.Parameters.SSH[0].Optional) +} diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index f83513e353be..4fd465ddc55a 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -28,7 +28,6 @@ import ( "github.com/containerd/containerd/snapshots" "github.com/containerd/continuity/fs/fstest" intoto "github.com/in-toto/in-toto-golang/in_toto" - slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/builder" @@ -40,7 +39,6 @@ import ( "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" - "github.com/moby/buildkit/util/provenance" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" "github.com/moby/buildkit/util/testutil/integration" @@ -151,6 +149,10 @@ var allTests = integration.TestFuncs( testSBOMScannerImage, testProvenanceAttestation, testGitProvenanceAttestation, + testMultiPlatformProvenance, + testClientFrontendProvenance, + testClientLLBProvenance, + testSecretSSHProvenance, ) // Tests that depend on the `security.*` entitlements @@ -6157,260 +6159,6 @@ EOF require.Equal(t, map[string]interface{}{"success": true}, attest.Predicate) } -func testProvenanceAttestation(t *testing.T, sb integration.Sandbox) { - ctx := sb.Context() - - c, err := client.New(ctx, sb.Address()) - require.NoError(t, err) - defer c.Close() - - registry, err := sb.NewRegistry() - if errors.Is(err, integration.ErrRequirements) { - t.Skip(err.Error()) - } - require.NoError(t, err) - - f := getFrontend(t, sb) - - dockerfile := []byte(` -FROM busybox:latest -RUN echo "ok" > /foo -`) - dir, err := integration.Tmpdir( - t, - fstest.CreateFile("Dockerfile", dockerfile, 0600), - ) - require.NoError(t, err) - - for _, mode := range []string{"min", "max"} { - t.Run(mode, func(t *testing.T) { - target := registry + "/buildkit/testwithprovenance:" + mode - provReq := "" - if mode == "max" { - provReq = "mode=max" - } - _, err = f.Solve(sb.Context(), c, client.SolveOpt{ - LocalDirs: map[string]string{ - builder.DefaultLocalNameDockerfile: dir, - builder.DefaultLocalNameContext: dir, - }, - FrontendAttrs: map[string]string{ - "attest:provenance": provReq, - "build-arg:FOO": "bar", - "label:lbl": "abc", - "vcs:source": "https://example.invalid/repo.git", - "vcs:revision": "123456", - }, - Exports: []client.ExportEntry{ - { - Type: client.ExporterImage, - Attrs: map[string]string{ - "name": target, - "push": "true", - }, - }, - }, - }, nil) - require.NoError(t, err) - - desc, provider, err := contentutil.ProviderFromRef(target) - require.NoError(t, err) - imgs, err := testutil.ReadImages(sb.Context(), provider, desc) - require.NoError(t, err) - require.Equal(t, 2, len(imgs.Images)) - - img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) - require.NotNil(t, img) - require.Equal(t, []byte("ok\n"), img.Layers[1]["foo"].Data) - - att := imgs.Find("unknown/unknown") - require.NotNil(t, att) - require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) - require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") - var attest intoto.Statement - require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) - require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) - require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const - - type stmtT struct { - Predicate provenance.ProvenancePredicate `json:"predicate"` - } - var stmt stmtT - require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) - pred := stmt.Predicate - - require.Equal(t, "https://mobyproject.org/buildkit@v1", pred.BuildType) - require.Equal(t, "", pred.Builder.ID) - - require.Equal(t, slsa.ConfigSource{}, pred.Invocation.ConfigSource) - - switch f.(type) { - case *clientFrontend, *gatewayFrontend: - // TODO: buildinfo broken - default: - params, ok := pred.Invocation.Parameters.(map[string]interface{}) - require.True(t, ok, "%T", pred.Invocation.Parameters) - if mode == "max" { - require.Equal(t, 2, len(params)) - require.True(t, pred.Metadata.Completeness.Parameters) - - require.Equal(t, "bar", params["build-arg:FOO"]) - require.Equal(t, "abc", params["label:lbl"]) - } else { - require.False(t, pred.Metadata.Completeness.Parameters) - require.Equal(t, 0, len(params), "%v", params) - } - - require.Equal(t, "https://example.invalid/repo.git", pred.Metadata.VCS["source"]) - require.Equal(t, "123456", pred.Metadata.VCS["revision"]) - } - - require.Equal(t, 1, len(pred.Materials)) - require.Equal(t, "pkg:docker/busybox@latest", pred.Materials[0].URI) - require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) - - require.NotEmpty(t, pred.Metadata.BuildInvocationID) - - require.NotNil(t, pred.Metadata.BuildFinishedOn) - require.True(t, time.Since(*pred.Metadata.BuildFinishedOn) < 5*time.Minute) - require.NotNil(t, pred.Metadata.BuildStartedOn) - require.True(t, time.Since(*pred.Metadata.BuildStartedOn) < 5*time.Minute) - require.True(t, pred.Metadata.BuildStartedOn.Before(*pred.Metadata.BuildFinishedOn)) - - require.True(t, pred.Metadata.Completeness.Environment) - require.True(t, pred.Metadata.Completeness.Materials) - require.False(t, pred.Metadata.Reproducible) - - if mode == "max" { - require.Equal(t, 2, len(pred.Layers)) - require.NotNil(t, pred.Source) - require.Equal(t, "Dockerfile", pred.Source.Infos[0].Filename) - require.Equal(t, dockerfile, pred.Source.Infos[0].Data) - require.NotNil(t, pred.BuildConfig) - - bc, ok := pred.BuildConfig.(map[string]interface{}) - require.True(t, ok, "wrong type %T", pred.BuildConfig) - - llb, ok := bc["llbDefinition"].([]interface{}) - require.True(t, ok, "wrong buildconfig %+v", bc) - - require.Equal(t, 3, len(llb)) - } else { - require.Equal(t, 0, len(pred.Layers)) - require.Nil(t, pred.Source) - require.Nil(t, pred.BuildConfig) - } - }) - } -} - -func testGitProvenanceAttestation(t *testing.T, sb integration.Sandbox) { - ctx := sb.Context() - - c, err := client.New(ctx, sb.Address()) - require.NoError(t, err) - defer c.Close() - - registry, err := sb.NewRegistry() - if errors.Is(err, integration.ErrRequirements) { - t.Skip(err.Error()) - } - require.NoError(t, err) - - f := getFrontend(t, sb) - - dockerfile := []byte(` -FROM busybox:latest -RUN echo "git" > /foo -COPY myapp.Dockerfile / -`) - dir, err := integration.Tmpdir( - t, - fstest.CreateFile("myapp.Dockerfile", dockerfile, 0600), - ) - require.NoError(t, err) - - err = runShell(dir, - "git init", - "git config --local user.email test", - "git config --local user.name test", - "git add myapp.Dockerfile", - "git commit -m initial", - "git branch v1", - "git update-server-info", - ) - require.NoError(t, err) - - cmd := exec.Command("git", "rev-parse", "v1") - cmd.Dir = dir - expectedGitSHA, err := cmd.Output() - require.NoError(t, err) - - server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(dir)))) - defer server.Close() - - target := registry + "/buildkit/testwithprovenance:git" - - _, err = f.Solve(sb.Context(), c, client.SolveOpt{ - FrontendAttrs: map[string]string{ - "context": server.URL + "/.git#v1", - "attest:provenance": "", - "filename": "myapp.Dockerfile", - }, - Exports: []client.ExportEntry{ - { - Type: client.ExporterImage, - Attrs: map[string]string{ - "name": target, - "push": "true", - }, - }, - }, - }, nil) - require.NoError(t, err) - - desc, provider, err := contentutil.ProviderFromRef(target) - require.NoError(t, err) - imgs, err := testutil.ReadImages(sb.Context(), provider, desc) - require.NoError(t, err) - require.Equal(t, 2, len(imgs.Images)) - - img := imgs.Find(platforms.Format(platforms.Normalize(platforms.DefaultSpec()))) - require.NotNil(t, img) - require.Equal(t, []byte("git\n"), img.Layers[1]["foo"].Data) - - att := imgs.Find("unknown/unknown") - require.NotNil(t, att) - require.Equal(t, att.Desc.Annotations["vnd.docker.reference.digest"], string(img.Desc.Digest)) - require.Equal(t, att.Desc.Annotations["vnd.docker.reference.type"], "attestation-manifest") - var attest intoto.Statement - require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) - require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) - require.Equal(t, "https://slsa.dev/provenance/v0.2", attest.PredicateType) // intentionally not const - - type stmtT struct { - Predicate provenance.ProvenancePredicate `json:"predicate"` - } - var stmt stmtT - require.NoError(t, json.Unmarshal(att.LayersRaw[0], &stmt)) - pred := stmt.Predicate - - switch f.(type) { - case *clientFrontend: - // TODO: buildinfo broken - default: - require.Equal(t, server.URL+"/.git#v1", pred.Invocation.ConfigSource.URI) - require.Equal(t, "myapp.Dockerfile", pred.Invocation.ConfigSource.EntryPoint) - } - - require.Equal(t, 2, len(pred.Materials)) - require.Equal(t, "pkg:docker/busybox@latest", pred.Materials[0].URI) - require.NotEmpty(t, pred.Materials[0].Digest["sha256"]) - - require.Equal(t, strings.Replace(server.URL+"/.git#v1", "http://", "https://", 1), pred.Materials[1].URI) // TODO: buildinfo broken - require.Equal(t, strings.TrimSpace(string(expectedGitSHA)), pred.Materials[1].Digest["sha1"]) -} - func runShell(dir string, cmds ...string) error { for _, args := range cmds { var cmd *exec.Cmd diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index 5c14fa2940ff..dfd7e0dd9bb6 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -39,7 +39,6 @@ import ( opspb "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/stack" "github.com/moby/buildkit/util/tracing" @@ -650,16 +649,6 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest) if ref == nil { id = "" } else { - dtbi, err := buildinfo.Encode(ctx, pbRes.Metadata, fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k), ref.BuildSources()) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - if pbRes.Metadata == nil { - pbRes.Metadata = make(map[string][]byte) - } - pbRes.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k)] = dtbi - } lbf.refs[id] = ref } ids[k] = id @@ -683,16 +672,6 @@ func (lbf *llbBridgeForwarder) Solve(ctx context.Context, req *pb.SolveRequest) if ref == nil { id = "" } else { - dtbi, err := buildinfo.Encode(ctx, pbRes.Metadata, exptypes.ExporterBuildInfo, ref.BuildSources()) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - if pbRes.Metadata == nil { - pbRes.Metadata = make(map[string][]byte) - } - pbRes.Metadata[exptypes.ExporterBuildInfo] = dtbi - } def = ref.Definition() lbf.refs[id] = ref } diff --git a/go.mod b/go.mod index b2483f9cb780..74bad2a99e9b 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/opencontainers/runc v1.1.3 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/selinux v1.10.2 - github.com/package-url/packageurl-go v0.1.0 + github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 github.com/pelletier/go-toml v1.9.4 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.5.0 diff --git a/go.sum b/go.sum index 7d9488728fcb..5bbf08454b33 100644 --- a/go.sum +++ b/go.sum @@ -1161,8 +1161,8 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/package-url/packageurl-go v0.1.0 h1:efWBc98O/dBZRg1pw2xiDzovnlMjCa9NPnfaiBduh8I= -github.com/package-url/packageurl-go v0.1.0/go.mod h1:C/ApiuWpmbpni4DIOECf6WCjFUZV7O1Fx7VAzrZHgBw= +github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 h1:DiLBVp4DAcZlBVBEtJpNWZpZVq0AEeCY7Hqk8URVs4o= +github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= diff --git a/solver/jobs.go b/solver/jobs.go index 4eb89d2af72c..070b4020b775 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -23,7 +23,7 @@ import ( type ResolveOpFunc func(Vertex, Builder) (Op, error) type Builder interface { - Build(ctx context.Context, e Edge) (CachedResult, BuildSources, error) + Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) InContext(ctx context.Context, f func(ctx context.Context, g session.Group) error) error EachValue(ctx context.Context, key string, fn func(interface{}) error) error } @@ -198,16 +198,16 @@ type subBuilder struct { exporters []ExportableCacheKey } -func (sb *subBuilder) Build(ctx context.Context, e Edge) (CachedResult, BuildSources, error) { +func (sb *subBuilder) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { // TODO(@crazy-max): Handle BuildInfo from subbuild res, err := sb.solver.subBuild(ctx, e, sb.vtx) if err != nil { - return nil, nil, err + return nil, err } sb.mu.Lock() sb.exporters = append(sb.exporters, res.CacheKeys()[0]) // all keys already have full export chain sb.mu.Unlock() - return res, nil, nil + return &withProvenance{CachedResult: res}, nil } func (sb *subBuilder) InContext(ctx context.Context, f func(context.Context, session.Group) error) error { @@ -499,43 +499,62 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } } -func (j *Job) Build(ctx context.Context, e Edge) (CachedResult, BuildSources, error) { +func (j *Job) Build(ctx context.Context, e Edge) (CachedResultWithProvenance, error) { if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { j.span = span } v, err := j.list.load(e.Vertex, nil, j) if err != nil { - return nil, nil, err + return nil, err } e.Vertex = v res, err := j.list.s.build(ctx, e) if err != nil { - return nil, nil, err + return nil, err } j.list.mu.Lock() defer j.list.mu.Unlock() - return res, j.walkBuildSources(ctx, e, make(BuildSources)), nil + return &withProvenance{CachedResult: res, j: j, e: e}, nil } -func (j *Job) walkBuildSources(ctx context.Context, e Edge, bsrc BuildSources) BuildSources { - for _, inp := range e.Vertex.Inputs() { - if st, ok := j.list.actives[inp.Vertex.Digest()]; ok { - st.mu.Lock() - for _, cacheRes := range st.op.cacheRes { - for key, val := range cacheRes.BuildSources { - if _, ok := bsrc[key]; !ok { - bsrc[key] = val - } - } +type withProvenance struct { + CachedResult + j *Job + e Edge +} + +func (wp *withProvenance) WalkProvenance(ctx context.Context, f func(ProvenanceProvider) error) error { + if wp.j == nil { + return nil + } + m := map[digest.Digest]struct{}{} + return wp.j.walkProvenance(ctx, wp.e, f, m) +} + +func (j *Job) walkProvenance(ctx context.Context, e Edge, f func(ProvenanceProvider) error, visited map[digest.Digest]struct{}) error { + if _, ok := visited[e.Vertex.Digest()]; ok { + return nil + } + visited[e.Vertex.Digest()] = struct{}{} + if st, ok := j.list.actives[e.Vertex.Digest()]; ok { + st.mu.Lock() + if wp, ok := st.op.op.(ProvenanceProvider); ok { + if err := f(wp); err != nil { + st.mu.Unlock() + return err } - st.mu.Unlock() - bsrc = j.walkBuildSources(ctx, inp, bsrc) } + st.mu.Unlock() } - return bsrc + for _, inp := range e.Vertex.Inputs() { + if err := j.walkProvenance(ctx, inp, f, visited); err != nil { + return err + } + } + return nil } func (j *Job) Discard() error { diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index b0ec157b46ea..a7c4ab4efa4a 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -11,7 +11,6 @@ import ( "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" - "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend" gw "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" @@ -19,9 +18,9 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/worker" @@ -63,20 +62,20 @@ func (b *llbBridge) Warn(ctx context.Context, dgst digest.Digest, msg string, op }) } -func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImports []gw.CacheOptionsEntry) (solver.CachedResult, solver.BuildSources, error) { +func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImports []gw.CacheOptionsEntry) (solver.CachedResultWithProvenance, error) { w, err := b.resolveWorker() if err != nil { - return nil, nil, err + return nil, err } ent, err := loadEntitlements(b.builder) if err != nil { - return nil, nil, err + return nil, err } var cms []solver.CacheManager for _, im := range cacheImports { cmID, err := cmKey(im) if err != nil { - return nil, nil, err + return nil, err } b.cmsMu.Lock() var cm solver.CacheManager @@ -113,7 +112,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { - return nil, nil, errors.Wrap(err, "failed to load LLB") + return nil, errors.Wrap(err, "failed to load LLB") } if len(dpc.ids) > 0 { @@ -124,88 +123,44 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp if err := b.eachWorker(func(w worker.Worker) error { return w.PruneCacheMounts(ctx, ids) }); err != nil { - return nil, nil, err - } - } - - res, bi, err := b.builder.Build(ctx, edge) - if err != nil { - return nil, nil, err - } - return res, bi, nil -} - -func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) { - if req.Definition != nil && req.Definition.Def != nil && req.Frontend != "" { - return nil, errors.New("cannot solve with both Definition and Frontend specified") - } - - if req.Definition != nil && req.Definition.Def != nil { - res = &frontend.Result{Ref: newResultProxy(b, req)} - } else if req.Frontend != "" { - f, ok := b.frontends[req.Frontend] - if !ok { - return nil, errors.Errorf("invalid frontend: %s", req.Frontend) - } - res, err = f.Solve(ctx, b, req.FrontendOpt, req.FrontendInputs, sid, b.sm) - if err != nil { return nil, err } - } else { - return &frontend.Result{}, nil - } - if req.Evaluate { - err = res.EachRef(func(ref solver.ResultProxy) error { - _, err := res.Ref.Result(ctx) - return err - }) } - if len(res.Refs) > 0 { - for p := range res.Refs { - dtbi, err := buildinfo.GetMetadata(res.Metadata, fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, p), req.Frontend, req.FrontendOpt) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, p), dtbi) - } - } - } else { - dtbi, err := buildinfo.GetMetadata(res.Metadata, exptypes.ExporterBuildInfo, req.Frontend, req.FrontendOpt) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - res.AddMeta(exptypes.ExporterBuildInfo, dtbi) - } + res, err := b.builder.Build(ctx, edge) + if err != nil { + return nil, err } - - return + return res, nil } type resultProxy struct { - b *llbBridge + id string + b *provenanceBridge req frontend.SolveRequest g flightcontrol.Group mu sync.Mutex released bool v solver.CachedResult - bsrc solver.BuildSources err error errResults []solver.Result + provenance *provenance.Capture +} + +func newResultProxy(b *provenanceBridge, req frontend.SolveRequest) *resultProxy { + return &resultProxy{req: req, b: b, id: identity.NewID()} } -func newResultProxy(b *llbBridge, req frontend.SolveRequest) *resultProxy { - return &resultProxy{req: req, b: b} +func (rp *resultProxy) ID() string { + return rp.id } func (rp *resultProxy) Definition() *pb.Definition { return rp.req.Definition } -func (rp *resultProxy) BuildSources() solver.BuildSources { - return rp.bsrc +func (rp *resultProxy) Provenance() interface{} { + return rp.provenance } func (rp *resultProxy) Release(ctx context.Context) (err error) { @@ -251,8 +206,8 @@ func (rp *resultProxy) wrapError(err error) error { return err } -func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResult, solver.BuildSources, error) { - res, bsrc, err := rp.b.loadResult(ctx, rp.req.Definition, rp.req.CacheImports) +func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResultWithProvenance, error) { + res, err := rp.b.loadResult(ctx, rp.req.Definition, rp.req.CacheImports) var ee *llberrdefs.ExecError if errors.As(err, &ee) { ee.EachRef(func(res solver.Result) error { @@ -262,7 +217,7 @@ func (rp *resultProxy) loadResult(ctx context.Context) (solver.CachedResult, sol // acquire ownership so ExecError finalizer doesn't attempt to release as well ee.OwnerBorrowed = true } - return res, bsrc, err + return res, err } func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err error) { @@ -280,7 +235,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err return rp.v, rp.err } rp.mu.Unlock() - v, bsrc, err := rp.loadResult(ctx) + v, err := rp.loadResult(ctx) if err != nil { select { case <-ctx.Done(): @@ -299,8 +254,16 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err return nil, errors.Errorf("evaluating released result") } rp.v = v - rp.bsrc = bsrc rp.err = err + if err == nil { + capture, err := captureProvenance(ctx, v) + if err != nil && rp.err != nil { + rp.err = errors.Wrapf(rp.err, "failed to capture provenance: %v", err) + v.Release(context.TODO()) + rp.v = nil + } + rp.provenance = capture + } rp.mu.Unlock() return v, err }) diff --git a/solver/llbsolver/ops/build.go b/solver/llbsolver/ops/build.go index 4ada980dce88..fd47df3ae311 100644 --- a/solver/llbsolver/ops/build.go +++ b/solver/llbsolver/ops/build.go @@ -11,7 +11,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" @@ -20,24 +20,26 @@ import ( const buildCacheType = "buildkit.build.v0" -type buildOp struct { +type BuildOp struct { op *pb.BuildOp b frontend.FrontendLLBBridge v solver.Vertex } -func NewBuildOp(v solver.Vertex, op *pb.Op_Build, b frontend.FrontendLLBBridge, _ worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { +var _ solver.Op = &BuildOp{} + +func NewBuildOp(v solver.Vertex, op *pb.Op_Build, b frontend.FrontendLLBBridge, _ worker.Worker) (*BuildOp, error) { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } - return &buildOp{ + return &BuildOp{ op: op.Build, b: b, v: v, }, nil } -func (b *buildOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { +func (b *BuildOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { dt, err := json.Marshal(struct { Type string Exec *pb.BuildOp @@ -59,7 +61,7 @@ func (b *buildOp) CacheMap(ctx context.Context, g session.Group, index int) (*so }, true, nil } -func (b *buildOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (outputs []solver.Result, retErr error) { +func (b *BuildOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (outputs []solver.Result, retErr error) { if b.op.Builder != pb.LLBBuilder { return nil, errors.Errorf("only LLB builder is currently allowed") } @@ -145,7 +147,9 @@ func (b *buildOp) Exec(ctx context.Context, g session.Group, inputs []solver.Res return []solver.Result{r}, err } -func (b *buildOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { +func (b *BuildOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { // buildOp itself does not count towards parallelism budget. return func() {}, nil } + +func (b *BuildOp) IsProvenanceProvider() {} diff --git a/solver/llbsolver/ops/diff.go b/solver/llbsolver/ops/diff.go index 82234e7d834c..338a8748e8c6 100644 --- a/solver/llbsolver/ops/diff.go +++ b/solver/llbsolver/ops/diff.go @@ -10,7 +10,7 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" ) @@ -24,7 +24,7 @@ type diffOp struct { } func NewDiffOp(v solver.Vertex, op *pb.Op_Diff, w worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } return &diffOp{ diff --git a/solver/llbsolver/ops/exec.go b/solver/llbsolver/ops/exec.go index 51df9ff375c7..56df35e8a9d6 100644 --- a/solver/llbsolver/ops/exec.go +++ b/solver/llbsolver/ops/exec.go @@ -17,9 +17,9 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/secrets" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/llbsolver/errdefs" "github.com/moby/buildkit/solver/llbsolver/mounts" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/progress/logs" utilsystem "github.com/moby/buildkit/util/system" @@ -33,7 +33,7 @@ import ( const execCacheType = "buildkit.exec.v0" -type execOp struct { +type ExecOp struct { op *pb.ExecOp cm cache.Manager mm *mounts.MountManager @@ -45,12 +45,14 @@ type execOp struct { parallelism *semaphore.Weighted } -func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, parallelism *semaphore.Weighted, sm *session.Manager, exec executor.Executor, w worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { +var _ solver.Op = &ExecOp{} + +func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache.Manager, parallelism *semaphore.Weighted, sm *session.Manager, exec executor.Executor, w worker.Worker) (*ExecOp, error) { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } name := fmt.Sprintf("exec %s", strings.Join(op.Exec.Meta.Args, " ")) - return &execOp{ + return &ExecOp{ op: op.Exec, mm: mounts.NewMountManager(name, cm, sm), cm: cm, @@ -63,6 +65,10 @@ func NewExecOp(v solver.Vertex, op *pb.Op_Exec, platform *pb.Platform, cm cache. }, nil } +func (e *ExecOp) Proto() *pb.ExecOp { + return e.op +} + func cloneExecOp(old *pb.ExecOp) pb.ExecOp { n := *old meta := *n.Meta @@ -80,7 +86,7 @@ func cloneExecOp(old *pb.ExecOp) pb.ExecOp { return n } -func (e *execOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { +func (e *ExecOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { op := cloneExecOp(e.op) for i := range op.Meta.ExtraHosts { h := op.Meta.ExtraHosts[i] @@ -157,9 +163,9 @@ func (e *execOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol cm.Deps[i].Selector = digest.FromBytes(bytes.Join(dgsts, []byte{0})) } if !dep.NoContentBasedHash { - cm.Deps[i].ComputeDigestFunc = llbsolver.NewContentHashFunc(toSelectors(dedupePaths(dep.Selectors))) + cm.Deps[i].ComputeDigestFunc = opsutils.NewContentHashFunc(toSelectors(dedupePaths(dep.Selectors))) } - cm.Deps[i].PreprocessFunc = llbsolver.UnlazyResultFunc + cm.Deps[i].PreprocessFunc = unlazyResultFunc } return cm, true, nil @@ -189,10 +195,10 @@ func dedupePaths(inp []string) []string { return paths } -func toSelectors(p []string) []llbsolver.Selector { - sel := make([]llbsolver.Selector, 0, len(p)) +func toSelectors(p []string) []opsutils.Selector { + sel := make([]opsutils.Selector, 0, len(p)) for _, p := range p { - sel = append(sel, llbsolver.Selector{Path: p, FollowLinks: true}) + sel = append(sel, opsutils.Selector{Path: p, FollowLinks: true}) } return sel } @@ -202,7 +208,7 @@ type dep struct { NoContentBasedHash bool } -func (e *execOp) getMountDeps() ([]dep, error) { +func (e *ExecOp) getMountDeps() ([]dep, error) { deps := make([]dep, e.numInputs) for _, m := range e.op.Mounts { if m.Input == pb.Empty { @@ -234,7 +240,7 @@ func addDefaultEnvvar(env []string, k, v string) []string { return append(env, k+"="+v) } -func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (results []solver.Result, err error) { +func (e *ExecOp) Exec(ctx context.Context, g session.Group, inputs []solver.Result) (results []solver.Result, err error) { trace.SpanFromContext(ctx).AddEvent("ExecOp started") refs := make([]*worker.WorkerRef, len(inputs)) @@ -393,7 +399,7 @@ func proxyEnvList(p *pb.ProxyEnv) []string { return out } -func (e *execOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { +func (e *ExecOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { if e.parallelism == nil { return func() {}, nil } @@ -406,7 +412,7 @@ func (e *execOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { }, nil } -func (e *execOp) loadSecretEnv(ctx context.Context, g session.Group) ([]string, error) { +func (e *ExecOp) loadSecretEnv(ctx context.Context, g session.Group) ([]string, error) { secretenv := e.op.Secretenv if len(secretenv) == 0 { return nil, nil @@ -436,3 +442,6 @@ func (e *execOp) loadSecretEnv(ctx context.Context, g session.Group) ([]string, } return out, nil } + +func (e *ExecOp) IsProvenanceProvider() { +} diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index b1f3e0cfe474..7bbb3276797c 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -13,10 +13,10 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" "github.com/moby/buildkit/solver/llbsolver/errdefs" "github.com/moby/buildkit/solver/llbsolver/file" "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/worker" @@ -38,7 +38,7 @@ type fileOp struct { } func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, parallelism *semaphore.Weighted, w worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } return &fileOp{ @@ -52,7 +52,7 @@ func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, parallelism *s } func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { - selectors := map[int][]llbsolver.Selector{} + selectors := map[int][]opsutils.Selector{} invalidSelectors := map[int]struct{}{} actions := make([][]byte, 0, len(f.op.Actions)) @@ -149,10 +149,10 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol }) cm.Deps[idx].Selector = digest.FromBytes(bytes.Join(dgsts, []byte{0})) - cm.Deps[idx].ComputeDigestFunc = llbsolver.NewContentHashFunc(dedupeSelectors(m)) + cm.Deps[idx].ComputeDigestFunc = opsutils.NewContentHashFunc(dedupeSelectors(m)) } for idx := range cm.Deps { - cm.Deps[idx].PreprocessFunc = llbsolver.UnlazyResultFunc + cm.Deps[idx].PreprocessFunc = unlazyResultFunc } return cm, true, nil @@ -194,8 +194,8 @@ func (f *fileOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { }, nil } -func addSelector(m map[int][]llbsolver.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns []string) { - s := llbsolver.Selector{ +func addSelector(m map[int][]opsutils.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns []string) { + s := opsutils.Selector{ Path: sel, FollowLinks: followLinks, Wildcard: wildcard && containsWildcards(sel), @@ -219,7 +219,7 @@ func containsWildcards(name string) bool { return false } -func dedupeSelectors(m []llbsolver.Selector) []llbsolver.Selector { +func dedupeSelectors(m []opsutils.Selector) []opsutils.Selector { paths := make([]string, 0, len(m)) pathsFollow := make([]string, 0, len(m)) for _, sel := range m { @@ -233,13 +233,13 @@ func dedupeSelectors(m []llbsolver.Selector) []llbsolver.Selector { } paths = dedupePaths(paths) pathsFollow = dedupePaths(pathsFollow) - selectors := make([]llbsolver.Selector, 0, len(m)) + selectors := make([]opsutils.Selector, 0, len(m)) for _, p := range paths { - selectors = append(selectors, llbsolver.Selector{Path: p}) + selectors = append(selectors, opsutils.Selector{Path: p}) } for _, p := range pathsFollow { - selectors = append(selectors, llbsolver.Selector{Path: p, FollowLinks: true}) + selectors = append(selectors, opsutils.Selector{Path: p, FollowLinks: true}) } for _, sel := range m { @@ -255,7 +255,7 @@ func dedupeSelectors(m []llbsolver.Selector) []llbsolver.Selector { return selectors } -func processOwner(chopt *pb.ChownOpt, selectors map[int][]llbsolver.Selector) error { +func processOwner(chopt *pb.ChownOpt, selectors map[int][]opsutils.Selector) error { if chopt == nil { return nil } @@ -665,3 +665,14 @@ func isDefaultIndexes(idxs [][]int) bool { } return true } + +func unlazyResultFunc(ctx context.Context, res solver.Result, g session.Group) error { + ref, ok := res.Sys().(*worker.WorkerRef) + if !ok { + return errors.Errorf("invalid reference: %T", res) + } + if ref.ImmutableRef == nil { + return nil + } + return ref.ImmutableRef.Extract(ctx, g) +} diff --git a/solver/llbsolver/ops/merge.go b/solver/llbsolver/ops/merge.go index 38976f3fbe1b..db1b025bff40 100644 --- a/solver/llbsolver/ops/merge.go +++ b/solver/llbsolver/ops/merge.go @@ -10,7 +10,7 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" digest "github.com/opencontainers/go-digest" ) @@ -24,7 +24,7 @@ type mergeOp struct { } func NewMergeOp(v solver.Vertex, op *pb.Op_Merge, w worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } return &mergeOp{ diff --git a/solver/llbsolver/ops/opsutils/contenthash.go b/solver/llbsolver/ops/opsutils/contenthash.go new file mode 100644 index 000000000000..8bdd8f939e15 --- /dev/null +++ b/solver/llbsolver/ops/opsutils/contenthash.go @@ -0,0 +1,71 @@ +package opsutils + +import ( + "bytes" + "context" + "path" + + "github.com/moby/buildkit/cache/contenthash" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +type Selector struct { + Path string + Wildcard bool + FollowLinks bool + IncludePatterns []string + ExcludePatterns []string +} + +func (sel Selector) HasWildcardOrFilters() bool { + return sel.Wildcard || len(sel.IncludePatterns) != 0 || len(sel.ExcludePatterns) != 0 +} + +func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc { + return func(ctx context.Context, res solver.Result, s session.Group) (digest.Digest, error) { + ref, ok := res.Sys().(*worker.WorkerRef) + if !ok { + return "", errors.Errorf("invalid reference: %T", res) + } + + if len(selectors) == 0 { + selectors = []Selector{{}} + } + + dgsts := make([][]byte, len(selectors)) + + eg, ctx := errgroup.WithContext(ctx) + + for i, sel := range selectors { + i, sel := i, sel + eg.Go(func() error { + dgst, err := contenthash.Checksum( + ctx, ref.ImmutableRef, path.Join("/", sel.Path), + contenthash.ChecksumOpts{ + Wildcard: sel.Wildcard, + FollowLinks: sel.FollowLinks, + IncludePatterns: sel.IncludePatterns, + ExcludePatterns: sel.ExcludePatterns, + }, + s, + ) + if err != nil { + return errors.Wrapf(err, "failed to calculate checksum of ref %s", ref.ID()) + } + dgsts[i] = []byte(dgst) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return "", err + } + + return digest.FromBytes(bytes.Join(dgsts, []byte{0})), nil + } +} diff --git a/solver/llbsolver/ops/opsutils/validate.go b/solver/llbsolver/ops/opsutils/validate.go new file mode 100644 index 000000000000..8e0d30d9ecf3 --- /dev/null +++ b/solver/llbsolver/ops/opsutils/validate.go @@ -0,0 +1,63 @@ +package opsutils + +import ( + "github.com/moby/buildkit/solver/pb" + "github.com/pkg/errors" +) + +func Validate(op *pb.Op) error { + if op == nil { + return errors.Errorf("invalid nil op") + } + + switch op := op.Op.(type) { + case *pb.Op_Source: + if op.Source == nil { + return errors.Errorf("invalid nil source op") + } + case *pb.Op_Exec: + if op.Exec == nil { + return errors.Errorf("invalid nil exec op") + } + if op.Exec.Meta == nil { + return errors.Errorf("invalid exec op with no meta") + } + if len(op.Exec.Meta.Args) == 0 { + return errors.Errorf("invalid exec op with no args") + } + if len(op.Exec.Mounts) == 0 { + return errors.Errorf("invalid exec op with no mounts") + } + + isRoot := false + for _, m := range op.Exec.Mounts { + if m.Dest == pb.RootMount { + isRoot = true + break + } + } + if !isRoot { + return errors.Errorf("invalid exec op with no rootfs") + } + case *pb.Op_File: + if op.File == nil { + return errors.Errorf("invalid nil file op") + } + if len(op.File.Actions) == 0 { + return errors.Errorf("invalid file op with no actions") + } + case *pb.Op_Build: + if op.Build == nil { + return errors.Errorf("invalid nil build op") + } + case *pb.Op_Merge: + if op.Merge == nil { + return errors.Errorf("invalid nil merge op") + } + case *pb.Op_Diff: + if op.Diff == nil { + return errors.Errorf("invalid nil diff op") + } + } + return nil +} diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index d24a902da570..fabd300d4b5c 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -7,7 +7,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" - "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" "github.com/moby/buildkit/worker" @@ -17,7 +17,7 @@ import ( const sourceCacheType = "buildkit.source.v0" -type sourceOp struct { +type SourceOp struct { mu sync.Mutex op *pb.Op_Source platform *pb.Platform @@ -27,13 +27,17 @@ type sourceOp struct { w worker.Worker vtx solver.Vertex parallelism *semaphore.Weighted + pin string + id source.Identifier } -func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm *source.Manager, parallelism *semaphore.Weighted, sessM *session.Manager, w worker.Worker) (solver.Op, error) { - if err := llbsolver.ValidateOp(&pb.Op{Op: op}); err != nil { +var _ solver.Op = &SourceOp{} + +func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm *source.Manager, parallelism *semaphore.Weighted, sessM *session.Manager, w worker.Worker) (*SourceOp, error) { + if err := opsutils.Validate(&pb.Op{Op: op}); err != nil { return nil, err } - return &sourceOp{ + return &SourceOp{ op: op, sm: sm, w: w, @@ -44,7 +48,13 @@ func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm }, nil } -func (s *sourceOp) instance(ctx context.Context) (source.SourceInstance, error) { +func (s *SourceOp) IsProvenanceProvider() {} + +func (s *SourceOp) Pin() (source.Identifier, string) { + return s.id, s.pin +} + +func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error) { s.mu.Lock() defer s.mu.Unlock() if s.src != nil { @@ -59,10 +69,11 @@ func (s *sourceOp) instance(ctx context.Context) (source.SourceInstance, error) return nil, err } s.src = src + s.id = id return s.src, nil } -func (s *sourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { +func (s *SourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { src, err := s.instance(ctx) if err != nil { return nil, false, err @@ -73,25 +84,23 @@ func (s *sourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*s return nil, false, err } + if s.pin == "" { + s.pin = pin + } + dgst := digest.FromBytes([]byte(sourceCacheType + ":" + k)) if strings.HasPrefix(k, "session:") { dgst = digest.Digest("random:" + strings.TrimPrefix(dgst.String(), dgst.Algorithm().String()+":")) } - var buildSources map[string]string - if !strings.HasPrefix(s.op.Source.GetIdentifier(), "local://") { - buildSources = map[string]string{s.op.Source.GetIdentifier(): pin} - } - return &solver.CacheMap{ // TODO: add os/arch - Digest: dgst, - Opts: cacheOpts, - BuildSources: buildSources, + Digest: dgst, + Opts: cacheOpts, }, done, nil } -func (s *sourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) (outputs []solver.Result, err error) { +func (s *SourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) (outputs []solver.Result, err error) { src, err := s.instance(ctx) if err != nil { return nil, err @@ -103,7 +112,7 @@ func (s *sourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) return []solver.Result{worker.NewWorkerRefResult(ref, s.w)}, nil } -func (s *sourceOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { +func (s *SourceOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { if s.parallelism == nil { return func() {}, nil } diff --git a/solver/llbsolver/proc/provenance.go b/solver/llbsolver/proc/provenance.go index 3a1d19f31af3..4186082dc7e6 100644 --- a/solver/llbsolver/proc/provenance.go +++ b/solver/llbsolver/proc/provenance.go @@ -13,24 +13,20 @@ import ( "github.com/moby/buildkit/cache/config" "github.com/moby/buildkit/exporter/containerimage" "github.com/moby/buildkit/exporter/containerimage/exptypes" - "github.com/moby/buildkit/frontend" gatewaypb "github.com/moby/buildkit/frontend/gateway/pb" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/result" - binfotypes "github.com/moby/buildkit/util/buildinfo/types" - provenance "github.com/moby/buildkit/util/provenance" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) -var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" - func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { - return func(ctx context.Context, res *frontend.Result, s *llbsolver.Solver, j *solver.Job) (*frontend.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { if len(res.Refs) == 0 { return nil, errors.New("provided result has no refs") } @@ -43,7 +39,7 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { var ps exptypes.Platforms if len(platformsBytes) > 0 { if err := json.Unmarshal(platformsBytes, &ps); err != nil { - return nil, errors.Wrapf(err, "failed to parse platforms passed to sbom processor") + return nil, errors.Wrapf(err, "failed to parse platforms passed to provenance processor") } } @@ -73,17 +69,12 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { } for _, p := range ps.Platforms { - dt, ok := res.Metadata[exptypes.ExporterBuildInfo+"/"+p.ID] + cp, ok := res.Provenance.Refs[p.ID] if !ok { return nil, errors.New("no build info found for provenance") } - var bi binfotypes.BuildInfo - if err := json.Unmarshal(dt, &bi); err != nil { - return nil, errors.Wrap(err, "failed to parse build info") - } - - pr, err := provenance.FromBuildInfo(bi) + pr, err := provenance.NewPredicate(cp) if err != nil { return nil, err } @@ -97,15 +88,17 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { var addLayers func() error if mode != "max" { - param := make(map[string]*string) - for k, v := range pr.Invocation.Parameters.(map[string]*string) { + args := make(map[string]string) + for k, v := range pr.Invocation.Parameters.Args { if strings.HasPrefix(k, "build-arg:") || strings.HasPrefix(k, "label:") { pr.Metadata.Completeness.Parameters = false continue } - param[k] = v + args[k] = v } - pr.Invocation.Parameters = param + pr.Invocation.Parameters.Args = args + pr.Invocation.Parameters.Secrets = nil + pr.Invocation.Parameters.SSH = nil } else { dgsts, err := provenance.AddBuildConfig(ctx, pr, res.Refs[p.ID]) if err != nil { @@ -139,7 +132,11 @@ func ProvenanceProcessor(attrs map[string]string) llbsolver.Processor { } if len(m) != 0 { - pr.Layers = m + if pr.Metadata == nil { + pr.Metadata = &provenance.ProvenanceMetadata{} + } + + pr.Metadata.BuildKitMetadata.Layers = m } return nil diff --git a/solver/llbsolver/proc/refs.go b/solver/llbsolver/proc/refs.go index 9b6475b77e9b..96ffa211c1f1 100644 --- a/solver/llbsolver/proc/refs.go +++ b/solver/llbsolver/proc/refs.go @@ -8,9 +8,9 @@ import ( "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/exporter/containerimage/exptypes" - "github.com/moby/buildkit/frontend" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/provenance" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -19,7 +19,7 @@ import ( // // This is useful for cases where a frontend produces a single-platform image, // but we need to add additional Refs to it (e.g. attestations). -func ForceRefsProcessor(ctx context.Context, result *frontend.Result, s *llbsolver.Solver, j *solver.Job) (*frontend.Result, error) { +func ForceRefsProcessor(ctx context.Context, result *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { if len(result.Refs) > 0 { return result, nil } @@ -72,5 +72,10 @@ func ForceRefsProcessor(ctx context.Context, result *frontend.Result, s *llbsolv } result.AddMeta(exptypes.ExporterPlatformsKey, dt) + result.Provenance.Refs = map[string]*provenance.Capture{ + pk: result.Provenance.Ref, + } + result.Provenance.Ref = nil + return result, nil } diff --git a/solver/llbsolver/proc/sbom.go b/solver/llbsolver/proc/sbom.go index 3b66f7693a24..c055fed1f85c 100644 --- a/solver/llbsolver/proc/sbom.go +++ b/solver/llbsolver/proc/sbom.go @@ -14,9 +14,9 @@ import ( ) func SBOMProcessor(scannerRef string) llbsolver.Processor { - return func(ctx context.Context, res *frontend.Result, s *llbsolver.Solver, j *solver.Job) (*frontend.Result, error) { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job) (*llbsolver.Result, error) { // skip sbom generation if we already have an sbom - if attest.HasSBOM(res) { + if attest.HasSBOM(res.Result) { return res, nil } @@ -61,7 +61,7 @@ func SBOMProcessor(scannerRef string) llbsolver.Processor { return nil, err } - r, err := s.Bridge(j).Solve(ctx, frontend.SolveRequest{ + r, err := s.Bridge(j).Solve(ctx, frontend.SolveRequest{ // TODO: buildinfo Definition: def.ToPB(), }, j.SessionID) if err != nil { diff --git a/solver/llbsolver/provenance.go b/solver/llbsolver/provenance.go new file mode 100644 index 000000000000..923703021cdf --- /dev/null +++ b/solver/llbsolver/provenance.go @@ -0,0 +1,342 @@ +package llbsolver + +import ( + "context" + "encoding/json" + "sync" + + "github.com/containerd/containerd/platforms" + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/ops" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/source" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +type resultWithBridge struct { + res *frontend.Result + bridge *provenanceBridge +} + +// provenanceBridge provides scoped access to LLBBridge and captures the request it makes for provenance +type provenanceBridge struct { + *llbBridge + mu sync.Mutex + req *frontend.SolveRequest + + images []provenance.ImageSource + builds []resultWithBridge + subBridges []*provenanceBridge +} + +func (b *provenanceBridge) eachRef(f func(r solver.ResultProxy) error) error { + for _, b := range b.builds { + if err := b.res.EachRef(f); err != nil { + return err + } + } + for _, b := range b.subBridges { + if err := b.eachRef(f); err != nil { + return err + } + } + return nil +} + +func (b *provenanceBridge) allImages() []provenance.ImageSource { + res := make([]provenance.ImageSource, 0, len(b.images)) + res = append(res, b.images...) + for _, sb := range b.subBridges { + res = append(res, sb.allImages()...) + } + return res +} + +func (b *provenanceBridge) requests(r *frontend.Result) (*resultRequests, error) { + reqs := &resultRequests{refs: make(map[string]*resultWithBridge)} + + if r.Ref != nil { + ref, ok := b.findByResult(r.Ref) + if !ok { + return nil, errors.Errorf("could not find request for ref %s", r.Ref.ID()) + } + reqs.ref = ref + } + + for k, ref := range r.Refs { + r, ok := b.findByResult(ref) + if !ok { + return nil, errors.Errorf("could not find request for ref %s", ref.ID()) + } + reqs.refs[k] = r + } + + if platformsBytes, ok := r.Metadata[exptypes.ExporterPlatformsKey]; ok { + var ps exptypes.Platforms + if len(platformsBytes) > 0 { + if err := json.Unmarshal(platformsBytes, &ps); err != nil { + return nil, errors.Wrapf(err, "failed to parse platforms passed to provenance processor") + } + reqs.platforms = ps.Platforms + } + } + + return reqs, nil +} + +func (b *provenanceBridge) findByResult(rp solver.ResultProxy) (*resultWithBridge, bool) { + for _, br := range b.subBridges { + if req, ok := br.findByResult(rp); ok { + return req, true + } + } + for _, bld := range b.builds { + if bld.res.Ref != nil { + if bld.res.Ref.ID() == rp.ID() { + return &bld, true + } + } + for _, ref := range bld.res.Refs { + if ref.ID() == rp.ID() { + return &bld, true + } + } + } + return nil, false +} + +func (b *provenanceBridge) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt) (dgst digest.Digest, config []byte, err error) { + dgst, config, err = b.llbBridge.ResolveImageConfig(ctx, ref, opt) + if err != nil { + return "", nil, err + } + + b.images = append(b.images, provenance.ImageSource{ + Ref: ref, + Platform: opt.Platform, + Digest: dgst, + }) + return dgst, config, nil +} + +func (b *provenanceBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) { + if req.Definition != nil && req.Definition.Def != nil && req.Frontend != "" { + return nil, errors.New("cannot solve with both Definition and Frontend specified") + } + + if req.Definition != nil && req.Definition.Def != nil { + rp := newResultProxy(b, req) + res = &frontend.Result{Ref: rp} + b.mu.Lock() + b.builds = append(b.builds, resultWithBridge{res: res, bridge: b}) + b.mu.Unlock() + } else if req.Frontend != "" { + f, ok := b.llbBridge.frontends[req.Frontend] + if !ok { + return nil, errors.Errorf("invalid frontend: %s", req.Frontend) + } + wb := &provenanceBridge{llbBridge: b.llbBridge, req: &req} + res, err = f.Solve(ctx, wb, req.FrontendOpt, req.FrontendInputs, sid, b.llbBridge.sm) + if err != nil { + return nil, err + } + wb.builds = append(wb.builds, resultWithBridge{res: res, bridge: wb}) + b.mu.Lock() + b.subBridges = append(b.subBridges, wb) + b.mu.Unlock() + } else { + return &frontend.Result{}, nil + } + if req.Evaluate { + err = res.EachRef(func(ref solver.ResultProxy) error { + _, err := res.Ref.Result(ctx) + return err + }) + } + return +} + +type resultRequests struct { + ref *resultWithBridge + refs map[string]*resultWithBridge + platforms []exptypes.Platform +} + +// filterImagePlatforms filter out images that not for the current platform if an image existist for every platform in a result +func (reqs *resultRequests) filterImagePlatforms(k string, imgs []provenance.ImageSource) []provenance.ImageSource { + if len(reqs.platforms) == 0 { + return imgs + } + m := map[string]string{} + for _, img := range imgs { + if _, ok := m[img.Ref]; ok { + continue + } + hasPlatform := true + for _, p := range reqs.platforms { + matcher := platforms.NewMatcher(p.Platform) + found := false + for _, img2 := range imgs { + if img.Ref == img2.Ref && img2.Platform != nil { + if matcher.Match(*img2.Platform) { + found = true + break + } + } + } + if !found { + hasPlatform = false + break + } + } + if hasPlatform { + m[img.Ref] = img.Ref + } + } + + var current ocispecs.Platform + for _, p := range reqs.platforms { + if p.ID == k { + current = p.Platform + } + } + + out := make([]provenance.ImageSource, 0, len(imgs)) + for _, img := range imgs { + if _, ok := m[img.Ref]; ok && img.Platform != nil { + if current.OS == img.Platform.OS && current.Architecture == img.Platform.Architecture { + out = append(out, img) + } + } else { + out = append(out, img) + } + } + return out +} + +func (reqs *resultRequests) allRes() map[string]struct{} { + res := make(map[string]struct{}) + if reqs.ref != nil { + res[reqs.ref.res.Ref.ID()] = struct{}{} + } + for _, r := range reqs.refs { + res[r.res.Ref.ID()] = struct{}{} + } + return res +} + +func captureProvenance(ctx context.Context, res solver.CachedResultWithProvenance) (*provenance.Capture, error) { + if res == nil { + return nil, nil + } + c := &provenance.Capture{} + + err := res.WalkProvenance(ctx, func(pp solver.ProvenanceProvider) error { + switch op := pp.(type) { + case *ops.SourceOp: + id, pin := op.Pin() + switch s := id.(type) { + case *source.ImageIdentifier: + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse image digest %s", pin) + } + c.AddImage(provenance.ImageSource{ + Ref: s.Reference.String(), + Platform: s.Platform, + Digest: dgst, + }) + case *source.LocalIdentifier: + c.AddLocal(provenance.LocalSource{ + Name: s.Name, + }) + case *source.GitIdentifier: + url := s.Remote + if s.Ref != "" { + url += "#" + s.Ref + } + c.AddGit(provenance.GitSource{ + URL: url, + Commit: pin, + }) + if s.AuthTokenSecret != "" { + c.AddSecret(provenance.Secret{ + ID: s.AuthTokenSecret, + Optional: true, + }) + } + if s.AuthHeaderSecret != "" { + c.AddSecret(provenance.Secret{ + ID: s.AuthHeaderSecret, + Optional: true, + }) + } + if s.MountSSHSock != "" { + c.AddSSH(provenance.SSH{ + ID: s.MountSSHSock, + Optional: true, + }) + } + case *source.HTTPIdentifier: + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse HTTP digest %s", pin) + } + c.AddHTTP(provenance.HTTPSource{ + URL: s.URL, + Digest: dgst, + }) + case *source.OCIIdentifier: + dgst, err := digest.Parse(pin) + if err != nil { + return errors.Wrapf(err, "failed to parse OCI digest %s", pin) + } + c.AddLocalImage(provenance.ImageSource{ + Ref: s.Name, + Platform: s.Platform, + Digest: dgst, + }) + default: + return errors.Errorf("unknown source identifier %T", id) + } + case *ops.ExecOp: + pr := op.Proto() + for _, m := range pr.Mounts { + if m.MountType == pb.MountType_SECRET { + c.AddSecret(provenance.Secret{ + ID: m.SecretOpt.GetID(), + Optional: m.SecretOpt.GetOptional(), + }) + } + if m.MountType == pb.MountType_SSH { + c.AddSSH(provenance.SSH{ + ID: m.SSHOpt.GetID(), + Optional: m.SSHOpt.GetOptional(), + }) + } + } + for _, se := range pr.Secretenv { + c.AddSecret(provenance.Secret{ + ID: se.GetID(), + Optional: se.GetOptional(), + }) + } + if pr.Network != pb.NetMode_NONE { + c.NetworkAccess = true + } + case *ops.BuildOp: + c.IncompleteMaterials = true // not supported yet + } + return nil + }) + if err != nil { + return nil, err + } + return c, err +} diff --git a/util/provenance/buildconfig.go b/solver/llbsolver/provenance/buildconfig.go similarity index 95% rename from util/provenance/buildconfig.go rename to solver/llbsolver/provenance/buildconfig.go index 90e7cd05279a..26148fb99b5d 100644 --- a/util/provenance/buildconfig.go +++ b/solver/llbsolver/provenance/buildconfig.go @@ -15,7 +15,7 @@ type BuildConfig struct { } type BuildStep struct { - ID string `json:"ID,omitempty"` + ID string `json:"id,omitempty"` Op interface{} `json:"op,omitempty"` Inputs []string `json:"inputs,omitempty"` } @@ -69,7 +69,10 @@ func AddBuildConfig(ctx context.Context, p *ProvenancePredicate, rp solver.Resul locs[fmt.Sprintf("step%d", idx)] = l } - p.Source = &Source{ + if p.Metadata == nil { + p.Metadata = &ProvenanceMetadata{} + } + p.Metadata.BuildKitMetadata.Source = &Source{ Infos: sis, Locations: locs, } diff --git a/solver/llbsolver/provenance/capture.go b/solver/llbsolver/provenance/capture.go new file mode 100644 index 000000000000..3d94e8f5c2f3 --- /dev/null +++ b/solver/llbsolver/provenance/capture.go @@ -0,0 +1,250 @@ +package provenance + +import ( + "sort" + + distreference "github.com/docker/distribution/reference" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" +) + +type Result struct { + Ref *Capture + Refs map[string]*Capture +} + +type ImageSource struct { + Ref string + Platform *ocispecs.Platform + Digest digest.Digest +} + +type GitSource struct { + URL string + Commit string +} + +type HTTPSource struct { + URL string + Digest digest.Digest +} + +type LocalSource struct { + Name string `json:"name"` +} + +type Secret struct { + ID string `json:"id"` + Optional bool `json:"optional,omitempty"` +} + +type SSH struct { + ID string `json:"id"` + Optional bool `json:"optional,omitempty"` +} + +type Sources struct { + Images []ImageSource + LocalImages []ImageSource + Git []GitSource + HTTP []HTTPSource + Local []LocalSource +} + +type Capture struct { + Frontend string + Args map[string]string + Sources Sources + Secrets []Secret + SSH []SSH + NetworkAccess bool + IncompleteMaterials bool +} + +func (c *Capture) Merge(c2 *Capture) error { + if c2 == nil { + return nil + } + for _, i := range c2.Sources.Images { + c.AddImage(i) + } + for _, i := range c2.Sources.LocalImages { + c.AddLocalImage(i) + } + for _, l := range c2.Sources.Local { + c.AddLocal(l) + } + for _, g := range c2.Sources.Git { + c.AddGit(g) + } + for _, h := range c2.Sources.HTTP { + c.AddHTTP(h) + } + for _, s := range c2.Secrets { + c.AddSecret(s) + } + for _, s := range c2.SSH { + c.AddSSH(s) + } + if c2.NetworkAccess { + c.NetworkAccess = true + } + if c2.IncompleteMaterials { + c.IncompleteMaterials = true + } + return nil +} + +func (c *Capture) Sort() { + sort.Slice(c.Sources.Images, func(i, j int) bool { + return c.Sources.Images[i].Ref < c.Sources.Images[j].Ref + }) + sort.Slice(c.Sources.LocalImages, func(i, j int) bool { + return c.Sources.LocalImages[i].Ref < c.Sources.LocalImages[j].Ref + }) + sort.Slice(c.Sources.Local, func(i, j int) bool { + return c.Sources.Local[i].Name < c.Sources.Local[j].Name + }) + sort.Slice(c.Sources.Git, func(i, j int) bool { + return c.Sources.Git[i].URL < c.Sources.Git[j].URL + }) + sort.Slice(c.Sources.HTTP, func(i, j int) bool { + return c.Sources.HTTP[i].URL < c.Sources.HTTP[j].URL + }) + sort.Slice(c.Secrets, func(i, j int) bool { + return c.Secrets[i].ID < c.Secrets[j].ID + }) + sort.Slice(c.SSH, func(i, j int) bool { + return c.SSH[i].ID < c.SSH[j].ID + }) +} + +// OptimizeImageSources filters out image sources by digest reference if same digest +// is already present by a tag reference. +func (c *Capture) OptimizeImageSources() error { + m := map[string]struct{}{} + for _, i := range c.Sources.Images { + ref, nameTag, err := parseRefName(i.Ref) + if err != nil { + return err + } + if _, ok := ref.(distreference.Canonical); !ok { + m[nameTag] = struct{}{} + } + } + + images := make([]ImageSource, 0, len(c.Sources.Images)) + for _, i := range c.Sources.Images { + ref, nameTag, err := parseRefName(i.Ref) + if err != nil { + return err + } + if _, ok := ref.(distreference.Canonical); ok { + if _, ok := m[nameTag]; ok { + continue + } + } + images = append(images, i) + } + c.Sources.Images = images + return nil +} + +func (c *Capture) AddImage(i ImageSource) { + for _, v := range c.Sources.Images { + if v.Ref == i.Ref { + if v.Platform == i.Platform { + return + } + if v.Platform != nil && i.Platform != nil { + if v.Platform.Architecture == i.Platform.Architecture && v.Platform.OS == i.Platform.OS && v.Platform.Variant == i.Platform.Variant { + return + } + } + } + } + c.Sources.Images = append(c.Sources.Images, i) +} + +func (c *Capture) AddLocalImage(i ImageSource) { + for _, v := range c.Sources.LocalImages { + if v.Ref == i.Ref { + if v.Platform == i.Platform { + return + } + if v.Platform != nil && i.Platform != nil { + if v.Platform.Architecture == i.Platform.Architecture && v.Platform.OS == i.Platform.OS && v.Platform.Variant == i.Platform.Variant { + return + } + } + } + } + c.Sources.LocalImages = append(c.Sources.LocalImages, i) +} + +func (c *Capture) AddLocal(l LocalSource) { + for _, v := range c.Sources.Local { + if v.Name == l.Name { + return + } + } + c.Sources.Local = append(c.Sources.Local, l) +} + +func (c *Capture) AddGit(g GitSource) { + for _, v := range c.Sources.Git { + if v.URL == g.URL { + return + } + } + c.Sources.Git = append(c.Sources.Git, g) +} + +func (c *Capture) AddHTTP(h HTTPSource) { + for _, v := range c.Sources.HTTP { + if v.URL == h.URL { + return + } + } + c.Sources.HTTP = append(c.Sources.HTTP, h) +} + +func (c *Capture) AddSecret(s Secret) { + for i, v := range c.Secrets { + if v.ID == s.ID { + if !s.Optional { + c.Secrets[i].Optional = false + } + return + } + } + c.Secrets = append(c.Secrets, s) +} + +func (c *Capture) AddSSH(s SSH) { + if s.ID == "" { + s.ID = "default" + } + for i, v := range c.SSH { + if v.ID == s.ID { + if !s.Optional { + c.SSH[i].Optional = false + } + return + } + } + c.SSH = append(c.SSH, s) +} + +func parseRefName(s string) (distreference.Named, string, error) { + ref, err := distreference.ParseNormalizedNamed(s) + if err != nil { + return nil, "", err + } + name := ref.Name() + tag := "latest" + if r, ok := ref.(distreference.Tagged); ok { + tag = r.Tag() + } + return ref, name + ":" + tag, nil +} diff --git a/solver/llbsolver/provenance/predicate.go b/solver/llbsolver/provenance/predicate.go new file mode 100644 index 000000000000..8e5593e62550 --- /dev/null +++ b/solver/llbsolver/provenance/predicate.go @@ -0,0 +1,248 @@ +package provenance + +import ( + "strings" + + "github.com/containerd/containerd/platforms" + slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" + "github.com/moby/buildkit/util/purl" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/package-url/packageurl-go" +) + +const ( + BuildKitBuildType = "https://mobyproject.org/buildkit@v1" +) + +type ProvenancePredicate struct { + slsa.ProvenancePredicate + Invocation ProvenanceInvocation `json:"invocation,omitempty"` + BuildConfig *BuildConfig `json:"buildConfig,omitempty"` + Metadata *ProvenanceMetadata `json:"metadata,omitempty"` +} + +type ProvenanceInvocation struct { + ConfigSource slsa.ConfigSource `json:"configSource,omitempty"` + Parameters Parameters `json:"parameters,omitempty"` + Environment Environment `json:"environment,omitempty"` +} + +type Parameters struct { + Frontend string `json:"frontend,omitempty"` + Args map[string]string `json:"args,omitempty"` + Secrets []*Secret `json:"secrets,omitempty"` + SSH []*SSH `json:"ssh,omitempty"` + Locals []*LocalSource `json:"locals,omitempty"` + // TODO: select export attributes + // TODO: frontend inputs +} + +type Environment struct { + Platform string `json:"platform"` +} + +type ProvenanceMetadata struct { + slsa.ProvenanceMetadata + Completeness ProvenanceComplete `json:"completeness"` + BuildKitMetadata BuildKitMetadata `json:"https://mobyproject.org/buildkit@v1#metadata,omitempty"` +} + +type ProvenanceComplete struct { + slsa.ProvenanceComplete + Hermetic bool `json:"https://mobyproject.org/buildkit@v1#hermetic,omitempty"` +} + +type BuildKitMetadata struct { + VCS map[string]string `json:"vcs,omitempty"` + Source *Source `json:"source,omitempty"` + Layers map[string][][]ocispecs.Descriptor `json:"layers,omitempty"` +} + +func slsaMaterials(srcs Sources) ([]slsa.ProvenanceMaterial, error) { + count := len(srcs.Images) + len(srcs.Git) + len(srcs.HTTP) + len(srcs.LocalImages) + out := make([]slsa.ProvenanceMaterial, 0, count) + + for _, s := range srcs.Images { + uri, err := purl.RefToPURL(s.Ref, s.Platform) + if err != nil { + return nil, err + } + out = append(out, slsa.ProvenanceMaterial{ + URI: uri, + Digest: slsa.DigestSet{ + s.Digest.Algorithm().String(): s.Digest.Hex(), + }, + }) + } + + for _, s := range srcs.Git { + out = append(out, slsa.ProvenanceMaterial{ + URI: s.URL, + Digest: slsa.DigestSet{ + "sha1": s.Commit, + }, + }) + } + + for _, s := range srcs.HTTP { + out = append(out, slsa.ProvenanceMaterial{ + URI: s.URL, + Digest: slsa.DigestSet{ + s.Digest.Algorithm().String(): s.Digest.Hex(), + }, + }) + } + + for _, s := range srcs.LocalImages { + q := []packageurl.Qualifier{} + if s.Platform != nil { + q = append(q, packageurl.Qualifier{ + Key: "platform", + Value: platforms.Format(*s.Platform), + }) + } + packageurl.NewPackageURL(packageurl.TypeOCI, "", s.Ref, "", q, "") + out = append(out, slsa.ProvenanceMaterial{ + URI: s.Ref, + Digest: slsa.DigestSet{ + s.Digest.Algorithm().String(): s.Digest.Hex(), + }, + }) + } + return out, nil +} + +func findMaterial(srcs Sources, uri string) (*slsa.ProvenanceMaterial, bool) { + for _, s := range srcs.Git { + if s.URL == uri { + return &slsa.ProvenanceMaterial{ + URI: s.URL, + Digest: slsa.DigestSet{ + "sha1": s.Commit, + }, + }, true + } + } + for _, s := range srcs.HTTP { + if s.URL == uri { + return &slsa.ProvenanceMaterial{ + URI: s.URL, + Digest: slsa.DigestSet{ + s.Digest.Algorithm().String(): s.Digest.Hex(), + }, + }, true + } + } + return nil, false +} + +func NewPredicate(c *Capture) (*ProvenancePredicate, error) { + materials, err := slsaMaterials(c.Sources) + if err != nil { + return nil, err + } + inv := ProvenanceInvocation{} + + contextKey := "context" + if v, ok := c.Args["contextkey"]; ok && v != "" { + contextKey = v + } + + if v, ok := c.Args[contextKey]; ok && v != "" { + if m, ok := findMaterial(c.Sources, v); ok { + inv.ConfigSource.URI = m.URI + inv.ConfigSource.Digest = m.Digest + } else { + inv.ConfigSource.URI = v + } + delete(c.Args, contextKey) + } + + if v, ok := c.Args["filename"]; ok && v != "" { + inv.ConfigSource.EntryPoint = v + delete(c.Args, "filename") + } + + vcs := make(map[string]string) + for k, v := range c.Args { + if strings.HasPrefix(k, "vcs:") { + delete(c.Args, k) + if v != "" { + vcs[strings.TrimPrefix(k, "vcs:")] = v + } + } + } + + inv.Environment.Platform = platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + + inv.Parameters.Frontend = c.Frontend + inv.Parameters.Args = c.Args + + for _, s := range c.Secrets { + inv.Parameters.Secrets = append(inv.Parameters.Secrets, &Secret{ + ID: s.ID, + Optional: s.Optional, + }) + } + for _, s := range c.SSH { + inv.Parameters.SSH = append(inv.Parameters.SSH, &SSH{ + ID: s.ID, + Optional: s.Optional, + }) + } + for _, s := range c.Sources.Local { + inv.Parameters.Locals = append(inv.Parameters.Locals, &LocalSource{ + Name: s.Name, + }) + } + + incompleteMaterials := c.IncompleteMaterials + if !incompleteMaterials { + if len(c.Sources.Local) > 0 { + incompleteMaterials = true + } + } + + pr := &ProvenancePredicate{ + Invocation: inv, + ProvenancePredicate: slsa.ProvenancePredicate{ + BuildType: BuildKitBuildType, + Materials: materials, + }, + Metadata: &ProvenanceMetadata{ + Completeness: ProvenanceComplete{ + ProvenanceComplete: slsa.ProvenanceComplete{ + Parameters: c.Frontend != "", + Environment: true, + Materials: !incompleteMaterials, + }, + Hermetic: !incompleteMaterials && !c.NetworkAccess, + }, + }, + } + + if len(vcs) > 0 { + pr.Metadata.BuildKitMetadata.VCS = vcs + } + + return pr, nil +} + +func FilterArgs(m map[string]string) map[string]string { + var hostSpecificArgs = map[string]struct{}{ + "cgroup-parent": {}, + "image-resolve-mode": {}, + "platform": {}, + } + out := make(map[string]string) + for k, v := range m { + if _, ok := hostSpecificArgs[k]; ok { + continue + } + if strings.HasPrefix(k, "attest:") { + continue + } + out[k] = v + } + return out +} diff --git a/solver/llbsolver/result.go b/solver/llbsolver/result.go index 0cadda547d54..7cd08754ae84 100644 --- a/solver/llbsolver/result.go +++ b/solver/llbsolver/result.go @@ -1,85 +1,20 @@ package llbsolver import ( - "bytes" "context" - "path" cacheconfig "github.com/moby/buildkit/cache/config" - "github.com/moby/buildkit/cache/contenthash" + "github.com/moby/buildkit/frontend" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/worker" - digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" - "golang.org/x/sync/errgroup" ) -type Selector struct { - Path string - Wildcard bool - FollowLinks bool - IncludePatterns []string - ExcludePatterns []string -} - -func (sel Selector) HasWildcardOrFilters() bool { - return sel.Wildcard || len(sel.IncludePatterns) != 0 || len(sel.ExcludePatterns) != 0 -} - -func UnlazyResultFunc(ctx context.Context, res solver.Result, g session.Group) error { - ref, ok := res.Sys().(*worker.WorkerRef) - if !ok { - return errors.Errorf("invalid reference: %T", res) - } - if ref.ImmutableRef == nil { - return nil - } - return ref.ImmutableRef.Extract(ctx, g) -} - -func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc { - return func(ctx context.Context, res solver.Result, s session.Group) (digest.Digest, error) { - ref, ok := res.Sys().(*worker.WorkerRef) - if !ok { - return "", errors.Errorf("invalid reference: %T", res) - } - - if len(selectors) == 0 { - selectors = []Selector{{}} - } - - dgsts := make([][]byte, len(selectors)) - - eg, ctx := errgroup.WithContext(ctx) - - for i, sel := range selectors { - i, sel := i, sel - eg.Go(func() error { - dgst, err := contenthash.Checksum( - ctx, ref.ImmutableRef, path.Join("/", sel.Path), - contenthash.ChecksumOpts{ - Wildcard: sel.Wildcard, - FollowLinks: sel.FollowLinks, - IncludePatterns: sel.IncludePatterns, - ExcludePatterns: sel.ExcludePatterns, - }, - s, - ) - if err != nil { - return errors.Wrapf(err, "failed to calculate checksum of ref %s", ref.ID()) - } - dgsts[i] = []byte(dgst) - return nil - }) - } - - if err := eg.Wait(); err != nil { - return "", err - } - - return digest.FromBytes(bytes.Join(dgsts, []byte{0})), nil - } +type Result struct { + *frontend.Result + Provenance *provenance.Result } func workerRefResolver(refCfg cacheconfig.RefConfig, all bool, g session.Group) func(ctx context.Context, res solver.Result) ([]*solver.Remote, error) { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index e4d9ff4a56eb..56d5622a212c 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -19,6 +19,7 @@ import ( "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/solver/result" "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/compression" @@ -70,7 +71,7 @@ type Solver struct { // Processor defines a processing function to be applied after solving, but // before exporting -type Processor func(ctx context.Context, result *frontend.Result, s *Solver, j *solver.Job) (*frontend.Result, error) +type Processor func(ctx context.Context, result *Result, s *Solver, j *solver.Job) (*Result, error) func New(opt Opt) (*Solver, error) { s := &Solver{ @@ -101,8 +102,8 @@ func (s *Solver) resolver() solver.ResolveOpFunc { } } -func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { - return &llbBridge{ +func (s *Solver) bridge(b solver.Builder) *provenanceBridge { + return &provenanceBridge{llbBridge: &llbBridge{ builder: b, frontends: s.frontends, resolveWorker: s.resolveWorker, @@ -110,7 +111,11 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { resolveCacheImporterFuncs: s.resolveCacheImporterFuncs, cms: map[string]solver.CacheManager{}, sm: s.sm, - } + }} +} + +func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { + return s.bridge(b) } func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor) (*client.SolveResponse, error) { @@ -130,8 +135,9 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID var res *frontend.Result + br := s.bridge(j) if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" { - fwd := gateway.NewBridgeForwarder(ctx, s.Bridge(j), s.workerController, req.FrontendInputs, sessionID, s.sm) + fwd := gateway.NewBridgeForwarder(ctx, br, s.workerController, req.FrontendInputs, sessionID, s.sm) defer fwd.Discard() if err := s.gatewayForwarder.RegisterBuild(ctx, id, fwd); err != nil { return nil, err @@ -149,7 +155,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } } else { - res, err = s.Bridge(j).Solve(ctx, req, sessionID) + res, err = br.Solve(ctx, req, sessionID) if err != nil { return nil, err } @@ -178,35 +184,19 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - if r := res.Ref; r != nil { - dtbi, err := buildinfo.Encode(ctx, res.Metadata, exptypes.ExporterBuildInfo, r.BuildSources()) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - res.AddMeta(exptypes.ExporterBuildInfo, dtbi) - } - } - for k, r := range res.Refs { - if r == nil { - continue - } - dtbi, err := buildinfo.Encode(ctx, res.Metadata, fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k), r.BuildSources()) - if err != nil { - return nil, err - } - if len(dtbi) > 0 { - res.AddMeta(fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k), dtbi) - } + resProv, err := addProvenanceToResult(res, br) + if err != nil { + return nil, err } for _, post := range post { - res2, err := post(ctx, res, s, j) + res2, err := post(ctx, resProv, s, j) if err != nil { return nil, err } - res = res2 + resProv = res2 } + res = resProv.Result cached, err := result.ConvertResult(res, func(res solver.ResultProxy) (solver.CachedResult, error) { return res.Result(ctx) @@ -361,6 +351,120 @@ func splitCacheExporters(exporters []RemoteCacheExporter) (rest []RemoteCacheExp return rest, inline } +func addProvenanceToResult(res *frontend.Result, br *provenanceBridge) (*Result, error) { + if res == nil { + return nil, nil + } + reqs, err := br.requests(res) + if err != nil { + return nil, err + } + out := &Result{ + Result: res, + Provenance: &provenance.Result{}, + } + if res.Ref != nil { + cp, err := getProvenance(res.Ref, reqs.ref.bridge, "", reqs) + if err != nil { + return nil, err + } + out.Provenance.Ref = cp + if res.Metadata == nil { + res.Metadata = map[string][]byte{} + } + if err := buildinfo.AddMetadata(res.Metadata, exptypes.ExporterBuildInfo, cp); err != nil { + return nil, err + } + } + if len(res.Refs) == 0 { + return out, nil + } + out.Provenance.Refs = make(map[string]*provenance.Capture, len(res.Refs)) + + for k, ref := range res.Refs { + cp, err := getProvenance(ref, reqs.refs[k].bridge, k, reqs) + if err != nil { + return nil, err + } + out.Provenance.Refs[k] = cp + if res.Metadata == nil { + res.Metadata = map[string][]byte{} + } + if err := buildinfo.AddMetadata(res.Metadata, fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k), cp); err != nil { + return nil, err + } + } + return out, nil +} + +func getRefProvenance(ref solver.ResultProxy, br *provenanceBridge) (*provenance.Capture, error) { + if ref == nil { + return nil, nil + } + p := ref.Provenance() + if p == nil { + return nil, errors.Errorf("missing provenance for %s", ref.ID()) + } + pr, ok := p.(*provenance.Capture) + if !ok { + return nil, errors.Errorf("invalid provenance type %T", p) + } + + if br.req != nil { + pr.Frontend = br.req.Frontend + pr.Args = provenance.FilterArgs(br.req.FrontendOpt) + // TODO: should also save some output options like compression + + if len(br.req.FrontendInputs) > 0 { + pr.IncompleteMaterials = true // not implemented + } + } + + return pr, nil +} + +func getProvenance(ref solver.ResultProxy, br *provenanceBridge, id string, reqs *resultRequests) (*provenance.Capture, error) { + pr, err := getRefProvenance(ref, br) + if err != nil { + return nil, err + } + if pr == nil { + return nil, nil + } + + visited := reqs.allRes() + visited[ref.ID()] = struct{}{} + // provenance for all the refs not directly in the result needs to be captured as well + if err := br.eachRef(func(r solver.ResultProxy) error { + if _, ok := visited[r.ID()]; ok { + return nil + } + visited[r.ID()] = struct{}{} + pr2, err := getRefProvenance(r, br) + if err != nil { + return err + } + return pr.Merge(pr2) + }); err != nil { + return nil, err + } + + imgs := br.allImages() + if id != "" { + imgs = reqs.filterImagePlatforms(id, imgs) + } + for _, img := range imgs { + pr.AddImage(img) + } + + if err := pr.OptimizeImageSources(); err != nil { + return nil, err + } + pr.Sort() + + return pr, nil +} + type inlineCacheExporter interface { ExportForLayers(context.Context, []digest.Digest) ([]byte, error) } diff --git a/solver/llbsolver/vertex.go b/solver/llbsolver/vertex.go index 4f36c2eddbb3..0be9a5e8940c 100644 --- a/solver/llbsolver/vertex.go +++ b/solver/llbsolver/vertex.go @@ -6,6 +6,7 @@ import ( "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/entitlements" @@ -228,7 +229,7 @@ func loadLLB(def *pb.Definition, fn func(digest.Digest, *pb.Op, func(digest.Dige return nil, errors.Errorf("invalid missing input digest %s", dgst) } - if err := ValidateOp(op); err != nil { + if err := opsutils.Validate(op); err != nil { return nil, err } @@ -301,63 +302,6 @@ func llbOpName(pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (st } } -func ValidateOp(op *pb.Op) error { - if op == nil { - return errors.Errorf("invalid nil op") - } - - switch op := op.Op.(type) { - case *pb.Op_Source: - if op.Source == nil { - return errors.Errorf("invalid nil source op") - } - case *pb.Op_Exec: - if op.Exec == nil { - return errors.Errorf("invalid nil exec op") - } - if op.Exec.Meta == nil { - return errors.Errorf("invalid exec op with no meta") - } - if len(op.Exec.Meta.Args) == 0 { - return errors.Errorf("invalid exec op with no args") - } - if len(op.Exec.Mounts) == 0 { - return errors.Errorf("invalid exec op with no mounts") - } - - isRoot := false - for _, m := range op.Exec.Mounts { - if m.Dest == pb.RootMount { - isRoot = true - break - } - } - if !isRoot { - return errors.Errorf("invalid exec op with no rootfs") - } - case *pb.Op_File: - if op.File == nil { - return errors.Errorf("invalid nil file op") - } - if len(op.File.Actions) == 0 { - return errors.Errorf("invalid file op with no actions") - } - case *pb.Op_Build: - if op.Build == nil { - return errors.Errorf("invalid nil build op") - } - case *pb.Op_Merge: - if op.Merge == nil { - return errors.Errorf("invalid nil merge op") - } - case *pb.Op_Diff: - if op.Diff == nil { - return errors.Errorf("invalid nil diff op") - } - } - return nil -} - func fileOpName(actions []*pb.FileAction) string { names := make([]string, 0, len(actions)) for _, action := range actions { diff --git a/solver/scheduler_test.go b/solver/scheduler_test.go index 6917baa81f7d..2e4c602bba5d 100644 --- a/solver/scheduler_test.go +++ b/solver/scheduler_test.go @@ -54,11 +54,10 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.NotNil(t, res) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -81,10 +80,9 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -113,10 +111,9 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -149,10 +146,9 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g3.Vertex.(*vertex).setupCallCounters() - res, bi, err = j3.Build(ctx, g3) + res, err = j3.Build(ctx, g3) require.NoError(t, err) require.Equal(t, unwrap(res), "result3") - require.Equal(t, len(bi), 0) require.Equal(t, *g3.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g3.Vertex.(*vertex).execCallCount, int64(1)) @@ -192,18 +188,16 @@ func TestSingleLevelActiveGraph(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, bi, err := j4.Build(ctx, g4) + res, err := j4.Build(ctx, g4) require.NoError(t, err) require.Equal(t, unwrap(res), "result4") - require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, bi, err := j5.Build(ctx, g4) + res, err := j5.Build(ctx, g4) require.NoError(t, err) require.Equal(t, unwrap(res), "result4") - require.Equal(t, len(bi), 0) return err }) @@ -240,10 +234,9 @@ func TestSingleLevelCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -267,10 +260,9 @@ func TestSingleLevelCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result1") - require.Equal(t, len(bi), 0) require.Equal(t, *g1.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g1.Vertex.(*vertex).execCallCount, int64(1)) @@ -298,10 +290,9 @@ func TestSingleLevelCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -366,18 +357,16 @@ func TestSingleLevelCacheParallel(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, bi, err := j1.Build(ctx, g1) + res, err := j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) return err }) @@ -460,18 +449,16 @@ func TestMultiLevelCacheParallel(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, bi, err := j1.Build(ctx, g1) + res, err := j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) return err }) @@ -514,7 +501,7 @@ func TestSingleCancelCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - _, _, err = j0.Build(ctx, g0) + _, err = j0.Build(ctx, g0) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -556,7 +543,7 @@ func TestSingleCancelExec(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - _, _, err = j1.Build(ctx, g1) + _, err = j1.Build(ctx, g1) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -609,7 +596,7 @@ func TestSingleCancelParallel(t *testing.T) { }), } - _, _, err = j.Build(ctx, g) + _, err = j.Build(ctx, g) close(firstErrored) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -633,10 +620,9 @@ func TestSingleCancelParallel(t *testing.T) { } <-firstReady - res, bi, err := j.Build(ctx, g) + res, err := j.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") - require.Equal(t, len(bi), 0) return err }) @@ -683,10 +669,9 @@ func TestMultiLevelCalculation(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g) + res, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 42) // 1 + 2*(7 + 2) + 2 + 2 + 19 - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -722,10 +707,9 @@ func TestMultiLevelCalculation(t *testing.T) { }, }), } - res, bi, err = j1.Build(ctx, g2) + res, err = j1.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrapInt(res), 42) - require.Equal(t, len(bi), 0) } func TestHugeGraph(t *testing.T) { @@ -757,10 +741,9 @@ func TestHugeGraph(t *testing.T) { // printGraph(g, "") g.Vertex.(*vertexSum).setupCallCounters() - res, bi, err := j0.Build(ctx, g) + res, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), v) - require.Equal(t, len(bi), 0) require.Equal(t, int64(nodes), *g.Vertex.(*vertexSum).cacheCallCount) // execCount := *g.Vertex.(*vertexSum).execCallCount // require.True(t, execCount < 1000) @@ -780,10 +763,9 @@ func TestHugeGraph(t *testing.T) { }() g.Vertex.(*vertexSum).setupCallCounters() - res, bi, err = j1.Build(ctx, g) + res, err = j1.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), v) - require.Equal(t, len(bi), 0) require.Equal(t, int64(nodes), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g.Vertex.(*vertexSum).execCallCount) @@ -837,10 +819,9 @@ func TestOptimizedCacheAccess(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(3), *g0.Vertex.(*vertex).execCallCount) @@ -884,10 +865,9 @@ func TestOptimizedCacheAccess(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -947,10 +927,9 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(3), *g0.Vertex.(*vertex).execCallCount) @@ -995,10 +974,9 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -1042,10 +1020,9 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g2.Vertex.(*vertex).execCallCount) @@ -1093,10 +1070,9 @@ func TestSlowCache(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -1128,10 +1104,9 @@ func TestSlowCache(t *testing.T) { }), } - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -1186,11 +1161,9 @@ func TestParallelInputs(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) - require.NoError(t, j0.Discard()) j0 = nil @@ -1241,7 +1214,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, _, err = j0.Build(ctx, g0) + _, err = j0.Build(ctx, g0) require.Error(t, err) require.Contains(t, err.Error(), "error-from-test") @@ -1282,7 +1255,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, _, err = j1.Build(ctx, g1) + _, err = j1.Build(ctx, g1) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -1323,7 +1296,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, _, err = j2.Build(ctx, g2) + _, err = j2.Build(ctx, g2) require.Error(t, err) require.Contains(t, err.Error(), "exec-error-from-test") @@ -1367,10 +1340,9 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -1410,10 +1382,10 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(1), cacheManager.loadCounter) require.Equal(t, int64(0), cacheManager2.loadCounter) @@ -1439,10 +1411,10 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, bi, err = j1.Build(ctx, g2) + res, err = j1.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), cacheManager.loadCounter) require.Equal(t, int64(0), cacheManager2.loadCounter) @@ -1484,10 +1456,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1523,10 +1495,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-1") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g1.Vertex.(*vertex).execCallCount) @@ -1561,10 +1533,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-2") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g2.Vertex.(*vertex).execCallCount) @@ -1611,10 +1583,10 @@ func TestIgnoreCacheResumeFromSlowCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1652,10 +1624,10 @@ func TestIgnoreCacheResumeFromSlowCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -1690,10 +1662,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) // match by vertex digest j1, err := l.NewJob("j1") @@ -1715,10 +1686,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result1") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -1744,10 +1714,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") - require.Equal(t, len(bi), 0) // match by cache key j3, err := l.NewJob("j3") @@ -1769,10 +1738,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g3.Vertex.(*vertex).setupCallCounters() - res, bi, err = j3.Build(ctx, g3) + res, err = j3.Build(ctx, g3) require.NoError(t, err) require.Equal(t, unwrap(res), "result3") - require.Equal(t, len(bi), 0) // add another ignorecache merges now @@ -1795,10 +1763,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g4.Vertex.(*vertex).setupCallCounters() - res, bi, err = j4.Build(ctx, g4) + res, err = j4.Build(ctx, g4) require.NoError(t, err) require.Equal(t, unwrap(res), "result3") - require.Equal(t, len(bi), 0) // add another !ignorecache merges now @@ -1820,10 +1787,9 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g5.Vertex.(*vertex).setupCallCounters() - res, bi, err = j5.Build(ctx, g5) + res, err = j5.Build(ctx, g5) require.NoError(t, err) require.Equal(t, unwrap(res), "result3") - require.Equal(t, len(bi), 0) } func TestSubbuild(t *testing.T) { @@ -1855,10 +1821,9 @@ func TestSubbuild(t *testing.T) { } g0.Vertex.(*vertexSum).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 8) - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).execCallCount) @@ -1877,10 +1842,9 @@ func TestSubbuild(t *testing.T) { g0.Vertex.(*vertexSum).setupCallCounters() - res, bi, err = j1.Build(ctx, g0) + res, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 8) - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g0.Vertex.(*vertexSum).execCallCount) @@ -1929,10 +1893,9 @@ func TestCacheWithSelector(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1971,10 +1934,9 @@ func TestCacheWithSelector(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(0), *g1.Vertex.(*vertex).execCallCount) @@ -2013,10 +1975,9 @@ func TestCacheWithSelector(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-1") - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g2.Vertex.(*vertex).execCallCount) @@ -2069,10 +2030,9 @@ func TestCacheSlowWithSelector(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -2114,10 +2074,9 @@ func TestCacheSlowWithSelector(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(0), *g1.Vertex.(*vertex).execCallCount) @@ -2157,10 +2116,9 @@ func TestCacheExporting(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 6) - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2189,10 +2147,9 @@ func TestCacheExporting(t *testing.T) { } }() - res, bi, err = j1.Build(ctx, g0) + res, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 6) - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2247,10 +2204,9 @@ func TestCacheExportingModeMin(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2281,10 +2237,9 @@ func TestCacheExportingModeMin(t *testing.T) { } }() - res, bi, err = j1.Build(ctx, g0) + res, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2316,10 +2271,9 @@ func TestCacheExportingModeMin(t *testing.T) { } }() - res, bi, err = j2.Build(ctx, g0) + res, err = j2.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) - require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2399,10 +2353,10 @@ func TestSlowCacheAvoidAccess(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -2419,10 +2373,9 @@ func TestSlowCacheAvoidAccess(t *testing.T) { } }() - res, bi, err = j1.Build(ctx, g0) + res, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2502,10 +2455,10 @@ func TestSlowCacheAvoidLoadOnCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "resultmain") - require.Equal(t, len(bi), 0) + require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -2576,10 +2529,9 @@ func TestSlowCacheAvoidLoadOnCache(t *testing.T) { } }() - res, bi, err = j1.Build(ctx, g0) + res, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "resultmain") - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2621,10 +2573,9 @@ func TestCacheMultipleMaps(t *testing.T) { value: "result0", }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2658,10 +2609,9 @@ func TestCacheMultipleMaps(t *testing.T) { }), } - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2694,10 +2644,9 @@ func TestCacheMultipleMaps(t *testing.T) { }), } - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2749,10 +2698,9 @@ func TestCacheInputMultipleMaps(t *testing.T) { }}, }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) expTarget := newTestExporterTarget() @@ -2791,10 +2739,9 @@ func TestCacheInputMultipleMaps(t *testing.T) { }}, }), } - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) _, err = res.CacheKeys()[0].Exporter.ExportTo(ctx, expTarget, testExporterOpts(true)) require.NoError(t, err) @@ -2848,10 +2795,9 @@ func TestCacheExportingPartialSelector(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2882,10 +2828,9 @@ func TestCacheExportingPartialSelector(t *testing.T) { g1 := g0 - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2937,10 +2882,9 @@ func TestCacheExportingPartialSelector(t *testing.T) { }), } - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2984,10 +2928,9 @@ func TestCacheExportingPartialSelector(t *testing.T) { ), } - res, bi, err = j3.Build(ctx, g3) + res, err = j3.Build(ctx, g3) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") - require.Equal(t, len(bi), 0) require.NoError(t, j3.Discard()) j3 = nil @@ -3083,10 +3026,9 @@ func TestCacheExportingMergedKey(t *testing.T) { }), } - res, bi, err := j0.Build(ctx, g0) + res, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -3144,10 +3086,10 @@ func TestMergedEdgesLookup(t *testing.T) { } g.Vertex.(*vertexSum).setupCallCounters() - res, bi, err := j0.Build(ctx, g) + res, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) - require.Equal(t, len(bi), 0) + require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), cacheManager.loadCounter) @@ -3196,13 +3138,12 @@ func TestCacheLoadError(t *testing.T) { } g.Vertex.(*vertexSum).setupCallCounters() - res, bi, err := j0.Build(ctx, g) + res, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(5), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(0), cacheManager.loadCounter) - require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -3221,13 +3162,12 @@ func TestCacheLoadError(t *testing.T) { g1.Vertex.(*vertexSum).setupCallCounters() - res, bi, err = j1.Build(ctx, g1) + res, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(1), cacheManager.loadCounter) - require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -3248,13 +3188,12 @@ func TestCacheLoadError(t *testing.T) { cacheManager.forceFail = true - res, bi, err = j2.Build(ctx, g2) + res, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(5), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(6), cacheManager.loadCounter) - require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -3302,7 +3241,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, _, err = j0.Build(ctx, g0) + _, err = j0.Build(ctx, g0) require.NoError(t, err) require.NoError(t, j0.Discard()) j0 = nil @@ -3340,7 +3279,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, _, err = j1.Build(ctx, g1) + _, err = j1.Build(ctx, g1) require.NoError(t, err) require.NoError(t, j1.Discard()) j1 = nil @@ -3381,7 +3320,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, _, err = j2.Build(ctx, g2) + _, err = j2.Build(ctx, g2) require.NoError(t, err) require.NoError(t, j2.Discard()) j2 = nil @@ -3684,7 +3623,7 @@ func (v *vertexSubBuild) Exec(ctx context.Context, g session.Group, inputs []Res if err := v.exec(ctx, inputs); err != nil { return nil, err } - res, _, err := v.b.Build(ctx, v.g) + res, err := v.b.Build(ctx, v.g) if err != nil { return nil, err } diff --git a/solver/types.go b/solver/types.go index b62da7680bb3..6635daef0e65 100644 --- a/solver/types.go +++ b/solver/types.go @@ -72,11 +72,17 @@ type CachedResult interface { CacheKeys() []ExportableCacheKey } +type CachedResultWithProvenance interface { + CachedResult + WalkProvenance(context.Context, func(ProvenanceProvider) error) error +} + type ResultProxy interface { + ID() string Result(context.Context) (CachedResult, error) Release(context.Context) error Definition() *pb.Definition - BuildSources() BuildSources + Provenance() interface{} } // CacheExportMode is the type for setting cache exporting modes @@ -161,6 +167,10 @@ type Op interface { Acquire(ctx context.Context) (release ReleaseFunc, err error) } +type ProvenanceProvider interface { + IsProvenanceProvider() +} + type ResultBasedCacheFunc func(context.Context, Result, session.Group) (digest.Digest, error) type PreprocessFunc func(context.Context, Result, session.Group) error @@ -198,15 +208,8 @@ type CacheMap struct { // such as oci descriptor content providers and progress writers to be passed to // the cache. Opts should not have any impact on the computed cache key. Opts CacheOpts - - // BuildSources contains build dependencies that will be set from source - // operation. - BuildSources BuildSources } -// BuildSources contains solved build dependencies. -type BuildSources map[string]string - // ExportableCacheKey is a cache key connected with an exporter that can export // a chain of cacherecords pointing to that key type ExportableCacheKey struct { diff --git a/source/manager.go b/source/manager.go index 3f4a0cb4783d..6a9c831c9048 100644 --- a/source/manager.go +++ b/source/manager.go @@ -16,7 +16,7 @@ type Source interface { } type SourceInstance interface { - CacheKey(ctx context.Context, g session.Group, index int) (string, string, solver.CacheOpts, bool, error) + CacheKey(ctx context.Context, g session.Group, index int) (key, pin string, opts solver.CacheOpts, done bool, err error) Snapshot(ctx context.Context, g session.Group) (cache.ImmutableRef, error) } diff --git a/util/buildinfo/buildinfo.go b/util/buildinfo/buildinfo.go index a36b767ac154..64b9ea48e146 100644 --- a/util/buildinfo/buildinfo.go +++ b/util/buildinfo/buildinfo.go @@ -10,12 +10,70 @@ import ( ctnref "github.com/containerd/containerd/reference" "github.com/docker/distribution/reference" "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/llbsolver/provenance" "github.com/moby/buildkit/source" binfotypes "github.com/moby/buildkit/util/buildinfo/types" "github.com/moby/buildkit/util/urlutil" "github.com/pkg/errors" ) +// BuildInfo format has been deprecated and will be removed in a future release. +// Use provenance attestations instead. + +func FromProvenance(c *provenance.Capture) (*binfotypes.BuildInfo, error) { + var bi binfotypes.BuildInfo + + bi.Frontend = c.Frontend + bi.Attrs = map[string]*string{} + for k, v := range c.Args { + v := v + bi.Attrs[k] = &v + } + + for _, s := range c.Sources.Images { + bi.Sources = append(bi.Sources, binfotypes.Source{ + Type: binfotypes.SourceTypeDockerImage, + Ref: s.Ref, + Pin: s.Digest.String(), + }) + } + + for _, s := range c.Sources.HTTP { + bi.Sources = append(bi.Sources, binfotypes.Source{ + Type: binfotypes.SourceTypeHTTP, + Ref: s.URL, + Pin: s.Digest.String(), + }) + } + + for _, s := range c.Sources.Git { + bi.Sources = append(bi.Sources, binfotypes.Source{ + Type: binfotypes.SourceTypeGit, + Ref: s.URL, + Pin: s.Commit, + }) + } + + sort.Slice(bi.Sources, func(i, j int) bool { + return bi.Sources[i].Ref < bi.Sources[j].Ref + }) + + return &bi, nil +} + +func AddMetadata(metadata map[string][]byte, key string, c *provenance.Capture) error { + bi, err := FromProvenance(c) + if err != nil { + return err + } + dt, err := json.Marshal(bi) + if err != nil { + return err + } + metadata[key] = dt + return nil +} + // Decode decodes a base64 encoded build info. func Decode(enc string) (bi binfotypes.BuildInfo, _ error) { dec, err := base64.StdEncoding.DecodeString(enc) @@ -377,49 +435,6 @@ func isControlArg(attrKey string) bool { return false } -// GetMetadata returns buildinfo metadata for the specified key. If the key -// is already there, result will be merged. -func GetMetadata(metadata map[string][]byte, key string, reqFrontend string, reqAttrs map[string]string) ([]byte, error) { - if metadata == nil { - metadata = make(map[string][]byte) - } - var dtbi []byte - if v, ok := metadata[key]; ok && v != nil { - var mbi binfotypes.BuildInfo - if errm := json.Unmarshal(v, &mbi); errm != nil { - return nil, errors.Wrapf(errm, "failed to unmarshal build info for %q", key) - } - if reqFrontend != "" { - mbi.Frontend = reqFrontend - } - if deps, err := decodeDeps(key, convertMap(reduceMapString(reqAttrs, mbi.Attrs))); err == nil { - mbi.Deps = reduceMapBuildInfo(deps, mbi.Deps) - } else { - return nil, err - } - mbi.Attrs = filterAttrs(key, convertMap(reduceMapString(reqAttrs, mbi.Attrs))) - var err error - dtbi, err = json.Marshal(mbi) - if err != nil { - return nil, errors.Wrapf(err, "failed to marshal build info for %q", key) - } - } else { - deps, err := decodeDeps(key, convertMap(reqAttrs)) - if err != nil { - return nil, err - } - dtbi, err = json.Marshal(binfotypes.BuildInfo{ - Frontend: reqFrontend, - Attrs: filterAttrs(key, convertMap(reqAttrs)), - Deps: deps, - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to marshal build info for %q", key) - } - } - return dtbi, nil -} - func reduceMapString(m1 map[string]string, m2 map[string]*string) map[string]string { if m1 == nil && m2 == nil { return nil @@ -434,25 +449,3 @@ func reduceMapString(m1 map[string]string, m2 map[string]*string) map[string]str } return m1 } - -func reduceMapBuildInfo(m1 map[string]binfotypes.BuildInfo, m2 map[string]binfotypes.BuildInfo) map[string]binfotypes.BuildInfo { - if m1 == nil && m2 == nil { - return nil - } - if m1 == nil { - m1 = map[string]binfotypes.BuildInfo{} - } - for k, v := range m2 { - m1[k] = v - } - return m1 -} - -func convertMap(m map[string]string) map[string]*string { - res := make(map[string]*string) - for k, v := range m { - value := v - res[k] = &value - } - return res -} diff --git a/util/provenance/buildinfo.go b/util/provenance/buildinfo.go deleted file mode 100644 index 90dc8028777d..000000000000 --- a/util/provenance/buildinfo.go +++ /dev/null @@ -1,164 +0,0 @@ -package provenance - -import ( - "encoding/hex" - "strings" - - distreference "github.com/docker/distribution/reference" - slsa "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v0.2" - binfotypes "github.com/moby/buildkit/util/buildinfo/types" - digest "github.com/opencontainers/go-digest" - ocispecs "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -var BuildKitBuildType = "https://mobyproject.org/buildkit@v1" - -type ProvenancePredicate struct { - slsa.ProvenancePredicate - Metadata *ProvenanceMetadata `json:"metadata,omitempty"` - Source *Source `json:"buildSource,omitempty"` - Layers map[string][][]ocispecs.Descriptor `json:"buildLayers,omitempty"` -} - -type ProvenanceMetadata struct { - slsa.ProvenanceMetadata - VCS map[string]string `json:"vcs,omitempty"` -} - -func convertMaterial(s binfotypes.Source) (*slsa.ProvenanceMaterial, error) { - // https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst - switch s.Type { - case binfotypes.SourceTypeDockerImage: - dgst, err := digest.Parse(s.Pin) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse digest %q for %s", s.Pin, s.Ref) - } - named, err := distreference.ParseNamed(s.Ref) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse ref %q", s.Ref) - } - version := "" - if tagged, ok := named.(distreference.Tagged); ok { - version = tagged.Tag() - } else { - if canonical, ok := named.(distreference.Canonical); ok { - version = canonical.Digest().String() - } - } - uri := "pkg:docker/" + distreference.FamiliarName(named) - if version != "" { - uri += "@" + version - } - return &slsa.ProvenanceMaterial{ - URI: uri, - Digest: slsa.DigestSet{ - dgst.Algorithm().String(): dgst.Hex(), - }, - }, nil - case binfotypes.SourceTypeGit: - if _, err := hex.DecodeString(s.Pin); err != nil { - return nil, errors.Wrapf(err, "failed to parse commit %q for %s", s.Pin, s.Ref) - } - return &slsa.ProvenanceMaterial{ - URI: s.Ref, - Digest: slsa.DigestSet{ - "sha1": s.Pin, // TODO: check length? - }, - }, nil - case binfotypes.SourceTypeHTTP: - dgst, err := digest.Parse(s.Pin) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse digest %q for %s", s.Pin, s.Ref) - } - return &slsa.ProvenanceMaterial{ - URI: s.Ref, - Digest: slsa.DigestSet{ - dgst.Algorithm().String(): dgst.Hex(), - }, - }, nil - default: - return nil, errors.Errorf("unsupported source type %q", s.Type) - } -} - -func findMaterial(srcs []binfotypes.Source, uri string) (*slsa.ProvenanceMaterial, bool) { - for _, s := range srcs { - if s.Ref == uri { - m, err := convertMaterial(s) - if err != nil { - continue - } - return m, true - } - } - return nil, false -} - -func FromBuildInfo(bi binfotypes.BuildInfo) (*ProvenancePredicate, error) { - materials := make([]slsa.ProvenanceMaterial, len(bi.Sources)) - for i, s := range bi.Sources { - m, err := convertMaterial(s) - if err != nil { - return nil, err - } - materials[i] = *m - } - - inv := slsa.ProvenanceInvocation{} - - contextKey := "context" - if v, ok := bi.Attrs["contextkey"]; ok && v != nil { - contextKey = *v - } - - if v, ok := bi.Attrs[contextKey]; ok && v != nil { - if m, ok := findMaterial(bi.Sources, *v); ok { - inv.ConfigSource.URI = m.URI - inv.ConfigSource.Digest = m.Digest - } else { - inv.ConfigSource.URI = *v - } - delete(bi.Attrs, contextKey) - } - - if v, ok := bi.Attrs["filename"]; ok && v != nil { - inv.ConfigSource.EntryPoint = *v - delete(bi.Attrs, "filename") - } - - vcs := make(map[string]string) - for k, v := range bi.Attrs { - if strings.HasPrefix(k, "vcs:") { - delete(bi.Attrs, k) - if v != nil { - vcs[strings.TrimPrefix(k, "vcs:")] = *v - } - } - } - - inv.Parameters = bi.Attrs - - pr := &ProvenancePredicate{ - ProvenancePredicate: slsa.ProvenancePredicate{ - BuildType: BuildKitBuildType, - Invocation: inv, - Materials: materials, - }, - Metadata: &ProvenanceMetadata{ - ProvenanceMetadata: slsa.ProvenanceMetadata{ - Completeness: slsa.ProvenanceComplete{ - Parameters: true, - Environment: true, - Materials: true, // TODO: check that there were no local sources - }, - }, - }, - } - - if len(vcs) > 0 { - pr.Metadata.VCS = vcs - } - - return pr, nil -} diff --git a/util/testutil/imageinfo.go b/util/testutil/imageinfo.go index 64e01dafa5b1..11cecc726792 100644 --- a/util/testutil/imageinfo.go +++ b/util/testutil/imageinfo.go @@ -43,6 +43,19 @@ func (idx ImagesInfo) Filter(platform string) *ImagesInfo { return result } +func (idx ImagesInfo) FindAttestation(platform string) *ImageInfo { + img := idx.Find(platform) + if img == nil { + return nil + } + for _, info := range idx.Images { + if info.Desc.Annotations["vnd.docker.reference.digest"] == string(img.Desc.Digest) { + return info + } + } + return nil +} + func ReadImages(ctx context.Context, p content.Provider, desc ocispecs.Descriptor) (*ImagesInfo, error) { idx := &ImagesInfo{Desc: desc} diff --git a/vendor/github.com/package-url/packageurl-go/.gitignore b/vendor/github.com/package-url/packageurl-go/.gitignore index d373807a9374..a1338d68517e 100644 --- a/vendor/github.com/package-url/packageurl-go/.gitignore +++ b/vendor/github.com/package-url/packageurl-go/.gitignore @@ -7,8 +7,6 @@ # Test binary, build with `go test -c` *.test -testdata/*json - # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/vendor/github.com/package-url/packageurl-go/.golangci.yaml b/vendor/github.com/package-url/packageurl-go/.golangci.yaml new file mode 100644 index 000000000000..73a5741c9270 --- /dev/null +++ b/vendor/github.com/package-url/packageurl-go/.golangci.yaml @@ -0,0 +1,17 @@ +# individual linter configs go here +linters-settings: + +# default linters are enabled `golangci-lint help linters` +linters: + disable-all: true + enable: + - deadcode + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck \ No newline at end of file diff --git a/vendor/github.com/package-url/packageurl-go/.travis.yml b/vendor/github.com/package-url/packageurl-go/.travis.yml deleted file mode 100644 index 1bb07d03a3af..000000000000 --- a/vendor/github.com/package-url/packageurl-go/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -language: go - -go: - - 1.12 - - tip - -install: true - -matrix: - allow_failures: - - go: tip - fast_finish: true - -script: - - make lint - - make test - -notifications: - email: false diff --git a/vendor/github.com/package-url/packageurl-go/mit.LICENSE b/vendor/github.com/package-url/packageurl-go/LICENSE similarity index 100% rename from vendor/github.com/package-url/packageurl-go/mit.LICENSE rename to vendor/github.com/package-url/packageurl-go/LICENSE diff --git a/vendor/github.com/package-url/packageurl-go/README.md b/vendor/github.com/package-url/packageurl-go/README.md index 68b42ac18e07..783985498b0b 100644 --- a/vendor/github.com/package-url/packageurl-go/README.md +++ b/vendor/github.com/package-url/packageurl-go/README.md @@ -1,8 +1,8 @@ # packageurl-go -Go implementation of the package url spec +[![build](https://github.com/package-url/packageurl-go/workflows/test/badge.svg)](https://github.com/package-url/packageurl-go/actions?query=workflow%3Atest) [![Coverage Status](https://coveralls.io/repos/github/package-url/packageurl-go/badge.svg)](https://coveralls.io/github/package-url/packageurl-go) [![PkgGoDev](https://pkg.go.dev/badge/github.com/package-url/packageurl-go)](https://pkg.go.dev/github.com/package-url/packageurl-go) [![Go Report Card](https://goreportcard.com/badge/github.com/package-url/packageurl-go)](https://goreportcard.com/report/github.com/package-url/packageurl-go) -[![Build Status](https://travis-ci.com/package-url/packageurl-go.svg)](https://travis-ci.com/package-url/packageurl-go) +Go implementation of the package url spec. ## Install @@ -55,7 +55,7 @@ func main() { ## Test -Testing using the normal ``go test`` command. Using ``make test`` will pull down the test fixtures shared between all package-url projects and then execute the tests. +Testing using the normal ``go test`` command. Using ``make test`` will pull the test fixtures shared between all package-url projects and then execute the tests. ``` $ make test diff --git a/vendor/github.com/package-url/packageurl-go/packageurl.go b/vendor/github.com/package-url/packageurl-go/packageurl.go index b521429f58d2..3cba7095d5f1 100644 --- a/vendor/github.com/package-url/packageurl-go/packageurl.go +++ b/vendor/github.com/package-url/packageurl-go/packageurl.go @@ -47,10 +47,20 @@ var ( var ( // TypeBitbucket is a pkg:bitbucket purl. TypeBitbucket = "bitbucket" + // TypeCocoapods is a pkg:cocoapods purl. + TypeCocoapods = "cocoapods" + // TypeCargo is a pkg:cargo purl. + TypeCargo = "cargo" // TypeComposer is a pkg:composer purl. TypeComposer = "composer" + // TypeConan is a pkg:conan purl. + TypeConan = "conan" + // TypeConda is a pkg:conda purl. + TypeConda = "conda" + // TypeCran is a pkg:cran purl. + TypeCran = "cran" // TypeDebian is a pkg:deb purl. - TypeDebian = "debian" + TypeDebian = "deb" // TypeDocker is a pkg:docker purl. TypeDocker = "docker" // TypeGem is a pkg:gem purl. @@ -61,16 +71,24 @@ var ( TypeGithub = "github" // TypeGolang is a pkg:golang purl. TypeGolang = "golang" + // TypeHackage is a pkg:hackage purl. + TypeHackage = "hackage" + // TypeHex is a pkg:hex purl. + TypeHex = "hex" // TypeMaven is a pkg:maven purl. TypeMaven = "maven" // TypeNPM is a pkg:npm purl. TypeNPM = "npm" // TypeNuget is a pkg:nuget purl. TypeNuget = "nuget" + // TypeOCI is a pkg:oci purl + TypeOCI = "oci" // TypePyPi is a pkg:pypi purl. TypePyPi = "pypi" // TypeRPM is a pkg:rpm purl. TypeRPM = "rpm" + // TypeSwift is pkg:swift purl + TypeSwift = "swift" ) // Qualifier represents a single key=value qualifier in the package url @@ -80,7 +98,7 @@ type Qualifier struct { } func (q Qualifier) String() string { - // A value must be must be a percent-encoded string + // A value must be a percent-encoded string return fmt.Sprintf("%s=%s", q.Key, url.PathEscape(q.Value)) } @@ -106,7 +124,7 @@ func QualifiersFromMap(mm map[string]string) Qualifiers { // Map converts a Qualifiers struct to a string map. func (qq Qualifiers) Map() map[string]string { - m := make(map[string]string, 0) + m := make(map[string]string) for i := 0; i < len(qq); i++ { k := qq[i].Key @@ -149,21 +167,22 @@ func NewPackageURL(purlType, namespace, name, version string, } } -// ToString returns the human readable instance of the PackageURL structure. +// ToString returns the human-readable instance of the PackageURL structure. // This is the literal purl as defined by the spec. func (p *PackageURL) ToString() string { // Start with the type and a colon purl := fmt.Sprintf("pkg:%s/", p.Type) // Add namespaces if provided if p.Namespace != "" { - ns := []string{} + var ns []string for _, item := range strings.Split(p.Namespace, "/") { ns = append(ns, url.QueryEscape(item)) } purl = purl + strings.Join(ns, "/") + "/" } // The name is always required and must be a percent-encoded string - purl = purl + url.PathEscape(p.Name) + // Use url.QueryEscape instead of PathEscape, as it handles @ signs + purl = purl + url.QueryEscape(p.Name) // If a version is provided, add it after the at symbol if p.Version != "" { // A name must be a percent-encoded string @@ -175,7 +194,7 @@ func (p *PackageURL) ToString() string { for _, q := range p.Qualifiers { qualifiers = append(qualifiers, q.String()) } - // If there one or more key=value pairs then append on the package url + // If there are one or more key=value pairs, append on the package url if len(qualifiers) != 0 { purl = purl + "?" + strings.Join(qualifiers, "&") } @@ -186,7 +205,7 @@ func (p *PackageURL) ToString() string { return purl } -func (p *PackageURL) String() string { +func (p PackageURL) String() string { return p.ToString() } @@ -274,9 +293,14 @@ func FromString(purl string) (PackageURL, error) { return PackageURL{}, fmt.Errorf("failed to unescape purl version: %s", err) } version = v - name = name[:atIndex] + + unecapeName, err := url.PathUnescape(name[:atIndex]) + if err != nil { + return PackageURL{}, fmt.Errorf("failed to unescape purl name: %s", err) + } + name = unecapeName } - namespaces := []string{} + var namespaces []string if index != -1 { remainder = remainder[:index] @@ -299,6 +323,11 @@ func FromString(purl string) (PackageURL, error) { return PackageURL{}, errors.New("name is required") } + err := validCustomRules(purlType, name, namespace, version, qualifiers) + if err != nil { + return PackageURL{}, err + } + return PackageURL{ Type: purlType, Namespace: namespace, @@ -331,6 +360,43 @@ func typeAdjustName(purlType, name string) string { return name } +// validQualifierKey validates a qualifierKey against our QualifierKeyPattern. func validQualifierKey(key string) bool { return QualifierKeyPattern.MatchString(key) } + +// validCustomRules evaluates additional rules for each package url type, as specified in the package-url specification. +// On success, it returns nil. On failure, a descriptive error will be returned. +func validCustomRules(purlType, name, ns, version string, qualifiers Qualifiers) error { + q := qualifiers.Map() + switch purlType { + case TypeConan: + if ns != "" { + if val, ok := q["channel"]; ok { + if val == "" { + return errors.New("the qualifier channel must be not empty if namespace is present") + } + } else { + return errors.New("channel qualifier does not exist") + } + } else { + if val, ok := q["channel"]; ok { + if val != "" { + return errors.New("namespace is required if channel is non empty") + } + } + } + case TypeSwift: + if ns == "" { + return errors.New("namespace is required") + } + if version == "" { + return errors.New("version is required") + } + case TypeCran: + if version == "" { + return errors.New("version is required") + } + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 25e8d7a79682..b96f60c4061b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -587,8 +587,8 @@ github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk github.com/opencontainers/selinux/pkg/pwalkdir -# github.com/package-url/packageurl-go v0.1.0 -## explicit; go 1.12 +# github.com/package-url/packageurl-go v0.1.1-0.20220428063043-89078438f170 +## explicit; go 1.17 github.com/package-url/packageurl-go # github.com/pelletier/go-toml v1.9.4 ## explicit; go 1.12 From fb917b9ec9527706ea6691f8332d37d8f65c4d93 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 13 Nov 2022 22:04:48 -0800 Subject: [PATCH 9/9] dockerfile: temporarily disable deps tests Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile_buildinfo_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/dockerfile/dockerfile_buildinfo_test.go b/frontend/dockerfile/dockerfile_buildinfo_test.go index ac9460d2b66e..55963c614549 100644 --- a/frontend/dockerfile/dockerfile_buildinfo_test.go +++ b/frontend/dockerfile/dockerfile_buildinfo_test.go @@ -497,6 +497,8 @@ COPY --from=base /o* / } func testBuildInfoDeps(t *testing.T, sb integration.Sandbox) { + t.Skip("deps temporarily disabled with SLSA provenance support") + ctx := sb.Context() f := getFrontend(t, sb) f.RequiresBuildctl(t) @@ -631,6 +633,8 @@ COPY --from=build /foo /out / } func testBuildInfoDepsMultiPlatform(t *testing.T, sb integration.Sandbox) { + t.Skip("deps temporarily disabled with SLSA provenance support") + ctx := sb.Context() f := getFrontend(t, sb) f.RequiresBuildctl(t) @@ -772,6 +776,8 @@ COPY --from=build /foo /out / } func testBuildInfoDepsMainNoSource(t *testing.T, sb integration.Sandbox) { + t.Skip("deps temporarily disabled with SLSA provenance support") + ctx := sb.Context() f := getFrontend(t, sb) f.RequiresBuildctl(t)