diff options
author | Tibor Vass <tibor@docker.com> | 2020-11-13 02:14:57 +0000 |
---|---|---|
committer | Tibor Vass <tibor@docker.com> | 2020-11-14 03:57:36 +0000 |
commit | de56a90929c5cde9697fc7c83f379cbddb21e323 (patch) | |
tree | 11429a1ca0af08144421526156489f18b90e9d40 /builder | |
parent | 13b09180b57a7478d72665d74ae62f688553f025 (diff) | |
download | docker-de56a90929c5cde9697fc7c83f379cbddb21e323.tar.gz |
vendor buildkit 6861f17f15364de0fe1fd1e6e8da07598a485123
Signed-off-by: Tibor Vass <tibor@docker.com>
Diffstat (limited to 'builder')
-rw-r--r-- | builder/builder-next/adapters/containerimage/pull.go | 44 | ||||
-rw-r--r-- | builder/builder-next/controller.go | 4 | ||||
-rw-r--r-- | builder/builder-next/executor_unix.go | 5 | ||||
-rw-r--r-- | builder/builder-next/executor_windows.go | 3 | ||||
-rw-r--r-- | builder/builder-next/exporter/writer.go | 2 | ||||
-rw-r--r-- | builder/builder-next/worker/worker.go | 24 | ||||
-rw-r--r-- | builder/dockerfile/builder.go | 6 | ||||
-rw-r--r-- | builder/dockerfile/dispatchers.go | 19 | ||||
-rw-r--r-- | builder/dockerfile/dispatchers_test.go | 6 | ||||
-rw-r--r-- | builder/dockerignore/deprecated.go | 17 | ||||
-rw-r--r-- | builder/dockerignore/dockerignore.go | 64 | ||||
-rw-r--r-- | builder/dockerignore/dockerignore_test.go | 69 | ||||
-rw-r--r-- | builder/remotecontext/detect.go | 2 |
13 files changed, 82 insertions, 183 deletions
diff --git a/builder/builder-next/adapters/containerimage/pull.go b/builder/builder-next/adapters/containerimage/pull.go index 65ee7a329c..a979d1b4d6 100644 --- a/builder/builder-next/adapters/containerimage/pull.go +++ b/builder/builder-next/adapters/containerimage/pull.go @@ -31,11 +31,12 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" "github.com/moby/buildkit/source" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/progress" - "github.com/moby/buildkit/util/resolver" + _ "github.com/moby/buildkit/util/resolver" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -67,7 +68,7 @@ type Source struct { func NewSource(opt SourceOpt) (*Source, error) { is := &Source{ SourceOpt: opt, - resolverCache: newResolverCache(), + resolverCache: nil, //newResolverCache(), } return is, nil @@ -82,10 +83,11 @@ func (is *Source) getResolver(hosts docker.RegistryHosts, ref string, sm *sessio if res := is.resolverCache.Get(ref, g); res != nil { return res } - auth := resolver.NewSessionAuthenticator(sm, g) - r := resolver.New(hosts, auth) - r = is.resolverCache.Add(ref, auth, r, g) - return r + //auth := resolver.NewSessionAuthenticator(sm, g) + //r := resolver.New(hosts, auth) + //r = is.resolverCache.Add(ref, auth, r, g) + //return r + return nil } func (is *Source) resolveLocal(refStr string) (*image.Image, error) { @@ -168,7 +170,7 @@ func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.Re } // Resolve returns access to pulling for an identifier -func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) { +func (is *Source) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, vtx solver.Vertex) (source.SourceInstance, error) { imageIdentifier, ok := id.(*source.ImageIdentifier) if !ok { return nil, errors.Errorf("invalid image identifier %v", id) @@ -319,47 +321,47 @@ func (p *puller) resolve(ctx context.Context, g session.Group) error { return p.resolveErr } -func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (string, bool, error) { +func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) { p.resolveLocal() if p.desc.Digest != "" && index == 0 { dgst, err := p.mainManifestKey(p.desc.Digest, p.platform) if err != nil { - return "", false, err + return "", nil, false, err } - return dgst.String(), false, nil + return dgst.String(), nil, false, nil } if p.config != nil { k := cacheKeyFromConfig(p.config).String() if k == "" { - return digest.FromBytes(p.config).String(), true, nil + return digest.FromBytes(p.config).String(), nil, true, nil } - return k, true, nil + return k, nil, true, nil } if err := p.resolve(ctx, g); err != nil { - return "", false, err + return "", nil, false, err } if p.desc.Digest != "" && index == 0 { dgst, err := p.mainManifestKey(p.desc.Digest, p.platform) if err != nil { - return "", false, err + return "", nil, false, err } - return dgst.String(), false, nil + return dgst.String(), nil, false, nil } k := cacheKeyFromConfig(p.config).String() if k == "" { dgst, err := p.mainManifestKey(p.desc.Digest, p.platform) if err != nil { - return "", false, err + return "", nil, false, err } - return dgst.String(), true, nil + return dgst.String(), nil, true, nil } - return k, true, nil + return k, nil, true, nil } func (p *puller) getRef(ctx context.Context, diffIDs []layer.DiffID, opts ...cache.RefOption) (cache.ImmutableRef, error) { @@ -854,7 +856,7 @@ type cachedResolver struct { counter int64 // needs to be 64bit aligned for 32bit systems timeout time.Time remotes.Resolver - auth *resolver.SessionAuthenticator + auth *struct{} } func (cr *cachedResolver) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { @@ -862,7 +864,7 @@ func (cr *cachedResolver) Resolve(ctx context.Context, ref string) (name string, return cr.Resolver.Resolve(ctx, ref) } -func (r *resolverCache) Add(ref string, auth *resolver.SessionAuthenticator, resolver remotes.Resolver, g session.Group) *cachedResolver { +func (r *resolverCache) Add(ref string, auth *struct{}, resolver remotes.Resolver, g session.Group) *cachedResolver { r.mu.Lock() defer r.mu.Unlock() @@ -871,7 +873,6 @@ func (r *resolverCache) Add(ref string, auth *resolver.SessionAuthenticator, res cr, ok := r.m[ref] cr.timeout = time.Now().Add(time.Minute) if ok { - cr.auth.AddSession(g) return &cr } @@ -897,7 +898,6 @@ func (r *resolverCache) Get(ref string, g session.Group) *cachedResolver { cr, ok := r.m[ref] if ok { - cr.auth.AddSession(g) return &cr } return nil diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index 5decff89f6..5aac0cb538 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -34,7 +34,7 @@ import ( "github.com/moby/buildkit/frontend/gateway/forwarder" containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" "github.com/moby/buildkit/solver/bboltcachestorage" - "github.com/moby/buildkit/util/binfmt_misc" + "github.com/moby/buildkit/util/archutil" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/worker" @@ -166,7 +166,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) { return nil, errors.Errorf("snapshotter doesn't support differ") } - p, err := parsePlatforms(binfmt_misc.SupportedPlatforms(true)) + p, err := parsePlatforms(archutil.SupportedPlatforms(true)) if err != nil { return nil, err } diff --git a/builder/builder-next/executor_unix.go b/builder/builder-next/executor_unix.go index d684b9f6e2..c052ec707f 100644 --- a/builder/builder-next/executor_unix.go +++ b/builder/builder-next/executor_unix.go @@ -95,11 +95,11 @@ func (iface *lnInterface) init(c libnetwork.NetworkController, n libnetwork.Netw iface.ep = ep } -func (iface *lnInterface) Set(s *specs.Spec) { +func (iface *lnInterface) Set(s *specs.Spec) error { <-iface.ready if iface.err != nil { logrus.WithError(iface.err).Error("failed to set networking spec") - return + return iface.err } shortNetCtlrID := stringid.TruncateID(iface.provider.NetworkController.ID()) // attach netns to bridge within the container namespace, using reexec in a prestart hook @@ -109,6 +109,7 @@ func (iface *lnInterface) Set(s *specs.Spec) { Args: []string{"libnetwork-setkey", "-exec-root=" + iface.provider.Config().Daemon.ExecRoot, iface.sbx.ContainerID(), shortNetCtlrID}, }}, } + return nil } func (iface *lnInterface) Close() error { diff --git a/builder/builder-next/executor_windows.go b/builder/builder-next/executor_windows.go index 5f33bcbe44..f63d8aba9e 100644 --- a/builder/builder-next/executor_windows.go +++ b/builder/builder-next/executor_windows.go @@ -7,7 +7,6 @@ import ( "github.com/docker/docker/daemon/config" "github.com/docker/docker/pkg/idtools" "github.com/docker/libnetwork" - "github.com/moby/buildkit/cache" "github.com/moby/buildkit/executor" "github.com/moby/buildkit/executor/oci" ) @@ -19,7 +18,7 @@ func newExecutor(_, _ string, _ libnetwork.NetworkController, _ *oci.DNSConfig, type winExecutor struct { } -func (w *winExecutor) Run(ctx context.Context, id string, root cache.Mountable, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (err error) { +func (w *winExecutor) Run(ctx context.Context, id string, root executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (err error) { return errors.New("buildkit executor not implemented for windows") } diff --git a/builder/builder-next/exporter/writer.go b/builder/builder-next/exporter/writer.go index 64d260f230..53305caf85 100644 --- a/builder/builder-next/exporter/writer.go +++ b/builder/builder-next/exporter/writer.go @@ -26,7 +26,7 @@ func emptyImageConfig() ([]byte, error) { } img.RootFS.Type = "layers" img.Config.WorkingDir = "/" - img.Config.Env = []string{"PATH=" + system.DefaultPathEnv} + img.Config.Env = []string{"PATH=" + system.DefaultPathEnvUnix} dt, err := json.Marshal(img) return dt, errors.Wrap(err, "failed to create empty image config") } diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index f8c3ef9dde..8d24646569 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -33,13 +33,14 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/mounts" "github.com/moby/buildkit/solver/llbsolver/ops" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/source" "github.com/moby/buildkit/source/git" "github.com/moby/buildkit/source/http" "github.com/moby/buildkit/source/local" - "github.com/moby/buildkit/util/binfmt_misc" + "github.com/moby/buildkit/util/archutil" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/progress" digest "github.com/opencontainers/go-digest" @@ -147,7 +148,7 @@ func (w *Worker) Platforms(noCache bool) []ocispec.Platform { for _, p := range w.Opt.Platforms { pm[platforms.Format(p)] = struct{}{} } - for _, p := range binfmt_misc.SupportedPlatforms(noCache) { + for _, p := range archutil.SupportedPlatforms(noCache) { if _, ok := pm[p]; !ok { pp, _ := platforms.Parse(p) w.Opt.Platforms = append(w.Opt.Platforms, pp) @@ -170,13 +171,18 @@ func (w *Worker) ContentStore() content.Store { return w.Opt.ContentStore } +// MetadataStore returns the metadata store +func (w *Worker) MetadataStore() *metadata.Store { + return w.Opt.MetadataStore +} + // LoadRef loads a reference by ID -func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) { +func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.ImmutableRef, error) { var opts []cache.RefOption if hidden { opts = append(opts, cache.NoUpdateLastUsed) } - return w.CacheManager().Get(context.TODO(), id, opts...) + return w.CacheManager().Get(ctx, id, opts...) } // ResolveOp converts a LLB vertex into a LLB operation @@ -186,9 +192,9 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *se case *pb.Op_Source: return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w) case *pb.Op_Exec: - return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager(), sm, w.MetadataStore, w.Executor(), w) + return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager(), sm, w.Opt.MetadataStore, w.Executor(), w) case *pb.Op_File: - return ops.NewFileOp(v, op, w.CacheManager(), w.MetadataStore, w) + return ops.NewFileOp(v, op, w.CacheManager(), w.Opt.MetadataStore, w) case *pb.Op_Build: return ops.NewBuildOp(v, op, s, w) } @@ -265,13 +271,13 @@ func (w *Worker) GetRemote(ctx context.Context, ref cache.ImmutableRef, createIf // PruneCacheMounts removes the current cache snapshots for specified IDs func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error { - mu := ops.CacheMountsLocker() + mu := mounts.CacheMountsLocker() mu.Lock() defer mu.Unlock() for _, id := range ids { id = "cache-dir:" + id - sis, err := w.MetadataStore.Search(id) + sis, err := w.Opt.MetadataStore.Search(id) if err != nil { return err } @@ -300,7 +306,7 @@ func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error { } } - ops.ClearActiveCacheMounts() + mounts.ClearActiveCacheMounts() return nil } diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index d8f48ae2c0..a0bfb289c2 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -235,8 +235,10 @@ func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *BuildA }); err != nil { return err } - args.AddArg(meta.Key, meta.Value) - args.AddMetaArg(meta.Key, meta.Value) + for _, arg := range meta.Args { + args.AddArg(arg.Key, arg.Value) + args.AddMetaArg(arg.Key, arg.Value) + } return nil } diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go index bf28a66a49..36335df952 100644 --- a/builder/dockerfile/dispatchers.go +++ b/builder/dockerfile/dispatchers.go @@ -587,14 +587,21 @@ func dispatchStopSignal(d dispatchRequest, c *instructions.StopSignalCommand) er // to builder using the --build-arg flag for expansion/substitution or passing to 'run'. // Dockerfile author may optionally set a default value of this variable. func dispatchArg(d dispatchRequest, c *instructions.ArgCommand) error { - - commitStr := "ARG " + c.Key - if c.Value != nil { - commitStr += "=" + *c.Value + var commitStr strings.Builder + commitStr.WriteString("ARG ") + for i, arg := range c.Args { + if i > 0 { + commitStr.WriteString(" ") + } + commitStr.WriteString(arg.Key) + if arg.Value != nil { + commitStr.WriteString("=") + commitStr.WriteString(*arg.Value) + } + d.state.buildArgs.AddArg(arg.Key, arg.Value) } - d.state.buildArgs.AddArg(c.Key, c.Value) - return d.builder.commit(d.state, commitStr) + return d.builder.commit(d.state, commitStr.String()) } // SHELL powershell -command diff --git a/builder/dockerfile/dispatchers_test.go b/builder/dockerfile/dispatchers_test.go index 03e0245c8c..d5f6bb96a9 100644 --- a/builder/dockerfile/dispatchers_test.go +++ b/builder/dockerfile/dispatchers_test.go @@ -139,10 +139,10 @@ func TestFromWithArg(t *testing.T) { args := NewBuildArgs(make(map[string]*string)) val := "sometag" - metaArg := instructions.ArgCommand{KeyValuePairOptional: instructions.KeyValuePairOptional{ + metaArg := instructions.ArgCommand{Args: []instructions.KeyValuePairOptional{{ Key: "THETAG", Value: &val, - }} + }}} cmd := &instructions.Stage{ BaseName: "alpine:${THETAG}", } @@ -395,7 +395,7 @@ func TestArg(t *testing.T) { argName := "foo" argVal := "bar" - cmd := &instructions.ArgCommand{KeyValuePairOptional: instructions.KeyValuePairOptional{Key: argName, Value: &argVal}} + cmd := &instructions.ArgCommand{Args: []instructions.KeyValuePairOptional{{Key: argName, Value: &argVal}}} err := dispatch(sb, cmd) assert.NilError(t, err) diff --git a/builder/dockerignore/deprecated.go b/builder/dockerignore/deprecated.go new file mode 100644 index 0000000000..e387cc8ed2 --- /dev/null +++ b/builder/dockerignore/deprecated.go @@ -0,0 +1,17 @@ +// Package dockerignore is deprecated. Use github.com/moby/buildkit/frontend/dockerfile/dockerignore instead. +package dockerignore + +import ( + "io" + + "github.com/moby/buildkit/frontend/dockerfile/dockerignore" +) + +// ReadAll reads a .dockerignore file and returns the list of file patterns +// to ignore. Note this will trim whitespace from each line as well +// as use GO's "clean" func to get the shortest/cleanest path for each. +// +// Deprecated: use github.com/moby/buildkit/frontend/dockerfile/dockerignore.ReadAll instead. +func ReadAll(reader io.Reader) ([]string, error) { + return dockerignore.ReadAll(reader) +} diff --git a/builder/dockerignore/dockerignore.go b/builder/dockerignore/dockerignore.go deleted file mode 100644 index 57f224afc8..0000000000 --- a/builder/dockerignore/dockerignore.go +++ /dev/null @@ -1,64 +0,0 @@ -package dockerignore // import "github.com/docker/docker/builder/dockerignore" - -import ( - "bufio" - "bytes" - "fmt" - "io" - "path/filepath" - "strings" -) - -// ReadAll reads a .dockerignore file and returns the list of file patterns -// to ignore. Note this will trim whitespace from each line as well -// as use GO's "clean" func to get the shortest/cleanest path for each. -func ReadAll(reader io.Reader) ([]string, error) { - if reader == nil { - return nil, nil - } - - scanner := bufio.NewScanner(reader) - var excludes []string - currentLine := 0 - - utf8bom := []byte{0xEF, 0xBB, 0xBF} - for scanner.Scan() { - scannedBytes := scanner.Bytes() - // We trim UTF8 BOM - if currentLine == 0 { - scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom) - } - pattern := string(scannedBytes) - currentLine++ - // Lines starting with # (comments) are ignored before processing - if strings.HasPrefix(pattern, "#") { - continue - } - pattern = strings.TrimSpace(pattern) - if pattern == "" { - continue - } - // normalize absolute paths to paths relative to the context - // (taking care of '!' prefix) - invert := pattern[0] == '!' - if invert { - pattern = strings.TrimSpace(pattern[1:]) - } - if len(pattern) > 0 { - pattern = filepath.Clean(pattern) - pattern = filepath.ToSlash(pattern) - if len(pattern) > 1 && pattern[0] == '/' { - pattern = pattern[1:] - } - } - if invert { - pattern = "!" + pattern - } - - excludes = append(excludes, pattern) - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("Error reading .dockerignore: %v", err) - } - return excludes, nil -} diff --git a/builder/dockerignore/dockerignore_test.go b/builder/dockerignore/dockerignore_test.go deleted file mode 100644 index 655bd6f332..0000000000 --- a/builder/dockerignore/dockerignore_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package dockerignore // import "github.com/docker/docker/builder/dockerignore" - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "testing" -) - -func TestReadAll(t *testing.T) { - tmpDir, err := ioutil.TempDir("", "dockerignore-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - di, err := ReadAll(nil) - if err != nil { - t.Fatalf("Expected not to have error, got %v", err) - } - - if diLen := len(di); diLen != 0 { - t.Fatalf("Expected to have zero dockerignore entry, got %d", diLen) - } - - diName := filepath.Join(tmpDir, ".dockerignore") - content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile\n# this is a comment\n! /inverted/abs/path\n!\n! \n") - err = ioutil.WriteFile(diName, []byte(content), 0777) - if err != nil { - t.Fatal(err) - } - - diFd, err := os.Open(diName) - if err != nil { - t.Fatal(err) - } - defer diFd.Close() - - di, err = ReadAll(diFd) - if err != nil { - t.Fatal(err) - } - - if len(di) != 7 { - t.Fatalf("Expected 7 entries, got %v", len(di)) - } - if di[0] != "test1" { - t.Fatal("First element is not test1") - } - if di[1] != "test2" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar - t.Fatal("Second element is not test2") - } - if di[2] != "a/file/here" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar - t.Fatal("Third element is not a/file/here") - } - if di[3] != "lastfile" { - t.Fatal("Fourth element is not lastfile") - } - if di[4] != "!inverted/abs/path" { - t.Fatal("Fifth element is not !inverted/abs/path") - } - if di[5] != "!" { - t.Fatalf("Sixth element is not !, but %s", di[5]) - } - if di[6] != "!" { - t.Fatalf("Seventh element is not !, but %s", di[6]) - } -} diff --git a/builder/remotecontext/detect.go b/builder/remotecontext/detect.go index 251fd08931..9b126ef775 100644 --- a/builder/remotecontext/detect.go +++ b/builder/remotecontext/detect.go @@ -11,10 +11,10 @@ import ( "github.com/containerd/continuity/driver" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/dockerignore" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/urlutil" + "github.com/moby/buildkit/frontend/dockerfile/dockerignore" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/pkg/errors" "github.com/sirupsen/logrus" |