publish env_file references as opaque hash to prevent paths conflicts

Signed-off-by: Nicolas De Loof <[email protected]>

Author: ndeloof

pkg/compose/publish.go

@@ -39,6 +39,7 @@ import (
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 )
 
 func (s *composeService) Publish(ctx context.Context, project *types.Project, repository string, options api.PublishOptions) error {
@@ -65,54 +66,33 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 		return err
 	}
 
-	named, err := reference.ParseDockerRef(repository)
+	layers, err := s.createLayers(ctx, project, options)
 	if err != nil {
 		return err
 	}
 
-	config := s.dockerCli.ConfigFile()
-
-	resolver := oci.NewResolver(config)
-
-	var layers []v1.Descriptor
-	extFiles := map[string]string{}
-	for _, file := range project.ComposeFiles {
-		data, err := processFile(ctx, file, project, extFiles)
-		if err != nil {
-			return err
-		}
-
-		layerDescriptor := oci.DescriptorForComposeFile(file, data)
-		layers = append(layers, layerDescriptor)
-	}
-
-	extLayers, err := processExtends(ctx, project, extFiles)
-	if err != nil {
-		return err
-	}
-	layers = append(layers, extLayers...)
-
-	if options.WithEnvironment {
-		layers = append(layers, envFileLayers(project)...)
-	}
-
-	if options.ResolveImageDigests {
-		yaml, err := s.generateImageDigestsOverride(ctx, project)
-		if err != nil {
-			return err
-		}
-
-		layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
-		layers = append(layers, layerDescriptor)
-	}
-
 	w := progress.ContextWriter(ctx)
 	w.Event(progress.Event{
 		ID:     repository,
 		Text:   "publishing",
 		Status: progress.Working,
 	})
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		logrus.Debug("publishing layers")
+		for _, layer := range layers {
+			indent, _ := json.MarshalIndent(layer, "", "  ")
+			fmt.Println(string(indent))
+		}
+	}
 	if !s.dryRun {
+		named, err := reference.ParseDockerRef(repository)
+		if err != nil {
+			return err
+		}
+
+		config := s.dockerCli.ConfigFile()
+		resolver := oci.NewResolver(config)
+
 		descriptor, err := oci.PushManifest(ctx, resolver, named, layers, options.OCIVersion)
 		if err != nil {
 			w.Event(progress.Event{
@@ -175,11 +155,47 @@ func (s *composeService) publish(ctx context.Context, project *types.Project, re
 	return nil
 }
 
+func (s *composeService) createLayers(ctx context.Context, project *types.Project, options api.PublishOptions) ([]v1.Descriptor, error) {
+	var layers []v1.Descriptor
+	extFiles := map[string]string{}
+	envFiles := map[string]string{}
+	for _, file := range project.ComposeFiles {
+		data, err := processFile(ctx, file, project, extFiles, envFiles)
+		if err != nil {
+			return nil, err
+		}
+
+		layerDescriptor := oci.DescriptorForComposeFile(file, data)
+		layers = append(layers, layerDescriptor)
+	}
+
+	extLayers, err := processExtends(ctx, project, extFiles)
+	if err != nil {
+		return nil, err
+	}
+	layers = append(layers, extLayers...)
+
+	if options.WithEnvironment {
+		layers = append(layers, envFileLayers(envFiles)...)
+	}
+
+	if options.ResolveImageDigests {
+		yaml, err := s.generateImageDigestsOverride(ctx, project)
+		if err != nil {
+			return nil, err
+		}
+
+		layerDescriptor := oci.DescriptorForComposeFile("image-digests.yaml", yaml)
+		layers = append(layers, layerDescriptor)
+	}
+	return layers, nil
+}
+
 func processExtends(ctx context.Context, project *types.Project, extFiles map[string]string) ([]v1.Descriptor, error) {
 	var layers []v1.Descriptor
 	moreExtFiles := map[string]string{}
 	for xf, hash := range extFiles {
-		data, err := processFile(ctx, xf, project, moreExtFiles)
+		data, err := processFile(ctx, xf, project, moreExtFiles, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -204,7 +220,7 @@ func processExtends(ctx context.Context, project *types.Project, extFiles map[st
 	return layers, nil
 }
 
-func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string) ([]byte, error) {
+func processFile(ctx context.Context, file string, project *types.Project, extFiles map[string]string, envFiles map[string]string) ([]byte, error) {
 	f, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
@@ -230,6 +246,15 @@ func processFile(ctx context.Context, file string, project *types.Project, extFi
 		return nil, err
 	}
 	for name, service := range base.Services {
+		for i, envFile := range service.EnvFiles {
+			hash := fmt.Sprintf("%x.env", sha256.Sum256([]byte(envFile.Path)))
+			envFiles[envFile.Path] = hash
+			f, err = transform.ReplaceEnvFile(f, name, i, hash)
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		if service.Extends == nil {
 			continue
 		}
@@ -376,18 +401,16 @@ func (s *composeService) checkEnvironmentVariables(project *types.Project, optio
 	return envVarList, nil
 }
 
-func envFileLayers(project *types.Project) []v1.Descriptor {
+func envFileLayers(files map[string]string) []v1.Descriptor {
 	var layers []v1.Descriptor
-	for _, service := range project.Services {
-		for _, envFile := range service.EnvFiles {
-			f, err := os.ReadFile(envFile.Path)
-			if err != nil {
-				// if we can't read the file, skip to the next one
-				continue
-			}
-			layerDescriptor := oci.DescriptorForEnvFile(envFile.Path, f)
-			layers = append(layers, layerDescriptor)
+	for file, hash := range files {
+		f, err := os.ReadFile(file)
+		if err != nil {
+			// if we can't read the file, skip to the next one
+			continue
 		}
+		layerDescriptor := oci.DescriptorForEnvFile(hash, f)
+		layers = append(layers, layerDescriptor)
 	}
 	return layers
 }

pkg/compose/publish_test.go

@@ -18,17 +18,19 @@ package compose
 
 import (
 	"context"
-	"os"
+	"slices"
 	"testing"
 
 	"github.com/compose-spec/compose-go/v2/loader"
 	"github.com/compose-spec/compose-go/v2/types"
+	"github.com/docker/compose/v2/internal"
 	"github.com/docker/compose/v2/pkg/api"
+	"github.com/google/go-cmp/cmp"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"gotest.tools/v3/assert"
 )
 
-func Test_processExtends(t *testing.T) {
+func Test_createLayers(t *testing.T) {
 	project, err := loader.LoadWithContext(context.TODO(), types.ConfigDetails{
 		WorkingDir:  "testdata/publish/",
 		Environment: types.Mapping{},
@@ -39,35 +41,62 @@ func Test_processExtends(t *testing.T) {
 		},
 	})
 	assert.NilError(t, err)
-	extFiles := map[string]string{}
-	file, err := processFile(context.TODO(), "testdata/publish/compose.yaml", project, extFiles)
+	project.ComposeFiles = []string{"testdata/publish/compose.yaml"}
+
+	service := &composeService{}
+	layers, err := service.createLayers(context.TODO(), project, api.PublishOptions{
+		WithEnvironment: true,
+	})
 	assert.NilError(t, err)
 
-	v := string(file)
-	assert.Equal(t, v, `name: test
+	published := string(layers[0].Data)
+	assert.Equal(t, published, `name: test
 services:
   test:
     extends:
       file: f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml
       service: foo
-`)
 
-	layers, err := processExtends(context.TODO(), project, extFiles)
-	assert.NilError(t, err)
+  string:
+    image: test
+    env_file: 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env
 
-	b, err := os.ReadFile("testdata/publish/common.yaml")
-	assert.NilError(t, err)
-	assert.DeepEqual(t, []v1.Descriptor{
+  list:
+    image: test
+    env_file:
+      - 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env
+
+  mapping:
+    image: test
+    env_file:
+      - path: 5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3.env
+`)
+
+	expectedLayers := []v1.Descriptor{
+		{
+			MediaType: "application/vnd.docker.compose.file+yaml",
+			Annotations: map[string]string{
+				"com.docker.compose.file":    "compose.yaml",
+				"com.docker.compose.version": internal.Version},
+		},
 		{
 			MediaType: "application/vnd.docker.compose.file+yaml",
-			Digest:    "sha256:d3ba84507b56ec783f4b6d24306b99a15285f0a23a835f0b668c2dbf9c59c241",
-			Size:      32,
 			Annotations: map[string]string{
 				"com.docker.compose.extends": "true",
-				"com.docker.compose.file":    "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c.yaml",
-				"com.docker.compose.version": api.ComposeVersion,
+				"com.docker.compose.file":    "f8f9ede3d201ec37d5a5e3a77bbadab79af26035e53135e19571f50d541d390c",
+				"com.docker.compose.version": internal.Version,
+			},
+		},
+		{
+			MediaType: "application/vnd.docker.compose.envfile",
+			Annotations: map[string]string{
+				"com.docker.compose.envfile": "5efca9cdbac9f5394c6c2e2094b1b42661f988f57fcab165a0bf72b205451af3",
+				"com.docker.compose.version": internal.Version,
 			},
-			Data: b,
 		},
-	}, layers)
+	}
+	assert.DeepEqual(t, expectedLayers, layers, cmp.FilterPath(func(path cmp.Path) bool {
+		return !slices.Contains([]string{".Data", ".Digest", ".Size"}, path.String())
+	}, cmp.Ignore()))
+
 }

pkg/compose/testdata/publish/compose.yaml

@@ -4,3 +4,17 @@ services:
     extends:
       file: common.yaml
       service: foo
+
+  string:
+    image: test
+    env_file: test.env
+
+  list:
+    image: test
+    env_file:
+      - test.env
+
+  mapping:
+    image: test
+    env_file:
+      - path: test.env

pkg/compose/testdata/publish/test.env

@@ -0,0 +1 @@
+HELLO=WORLD

pkg/compose/transform/replace.go

@@ -61,6 +61,52 @@ func ReplaceExtendsFile(in []byte, service string, value string) ([]byte, error)
 	return replace(in, file.Line, file.Column, value), nil
 }
 
+// ReplaceEnvFile changes value for service.extends.env_file in input yaml stream, preserving formatting
+func ReplaceEnvFile(in []byte, service string, i int, value string) ([]byte, error) {
+	var doc yaml.Node
+	err := yaml.Unmarshal(in, &doc)
+	if err != nil {
+		return nil, err
+	}
+	if doc.Kind != yaml.DocumentNode {
+		return nil, fmt.Errorf("expected document kind %v, got %v", yaml.DocumentNode, doc.Kind)
+	}
+	root := doc.Content[0]
+	if root.Kind != yaml.MappingNode {
+		return nil, fmt.Errorf("expected document root to be a mapping, got %v", root.Kind)
+	}
+
+	services, err := getMapping(root, "services")
+	if err != nil {
+		return nil, err
+	}
+
+	target, err := getMapping(services, service)
+	if err != nil {
+		return nil, err
+	}
+
+	envFile, err := getMapping(target, "env_file")
+	if err != nil {
+		return nil, err
+	}
+
+	// env_file can be either a string, sequence of strings, or sequence of mappings with path attribute
+	if envFile.Kind == yaml.SequenceNode {
+		envFile = envFile.Content[i]
+		if envFile.Kind == yaml.MappingNode {
+			envFile, err = getMapping(envFile, "path")
+			if err != nil {
+				return nil, err
+			}
+		}
+		return replace(in, envFile.Line, envFile.Column, value), nil
+	} else {
+		return replace(in, envFile.Line, envFile.Column, value), nil
+	}
+
+}
+
 func getMapping(root *yaml.Node, key string) (*yaml.Node, error) {
 	var node *yaml.Node
 	l := len(root.Content)

pkg/remote/oci.go

@@ -217,8 +217,9 @@ func (g ociRemoteLoader) pullComposeFiles(ctx context.Context, local string, man
 
 func writeComposeFile(layer spec.Descriptor, i int, local string, content []byte) error {
 	file := "compose.yaml"
-	if extends, ok := layer.Annotations["com.docker.compose.extends"]; ok {
-		if err := validatePathInBase(local, extends); err != nil {
+	if _, ok := layer.Annotations["com.docker.compose.extends"]; ok {
+		file = layer.Annotations["com.docker.compose.file"]
+		if err := validatePathInBase(local, file); err != nil {
 			return err
 		}
 	}