WIP: prototype of nested package library

go.mod

@@ -16,12 +16,15 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.18.10
+	k8s.io/apiextensions-apiserver v0.18.10
 	k8s.io/apimachinery v0.18.10
 	k8s.io/cli-runtime v0.18.10
 	k8s.io/client-go v0.18.10
 	k8s.io/klog v1.0.0
 	k8s.io/kubectl v0.18.10
 	sigs.k8s.io/cli-utils v0.22.5-0.20210127192708-27cfaa675296
 	sigs.k8s.io/kustomize/cmd/config v0.8.7-0.20201211170716-cc43a2d732d1
-	sigs.k8s.io/kustomize/kyaml v0.10.6
+	sigs.k8s.io/kustomize/kyaml v0.10.9
 )
+
+replace sigs.k8s.io/cli-utils v0.22.5-0.20210127192708-27cfaa675296 => ../../sigs.k8s.io/cli-utils

go.sum

@@ -697,6 +697,10 @@ sigs.k8s.io/kustomize/kyaml v0.10.3 h1:ARSJUMN/c3k31DYxRfZ+vp/UepUQjg9zCwny7Oj90
 sigs.k8s.io/kustomize/kyaml v0.10.3/go.mod h1:RA+iCHA2wPCOfv6uG6TfXXWhYsHpgErq/AljxWKuxtg=
 sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc=
 sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k=
+sigs.k8s.io/kustomize/kyaml v0.10.7 h1:r0r8UEL0bL7X56HKUmhJZ+TP+nvRNGrDHHSLO7izlcQ=
+sigs.k8s.io/kustomize/kyaml v0.10.7/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k=
+sigs.k8s.io/kustomize/kyaml v0.10.9 h1:n3WNdvPPReRNDxW+XXd2JlyZ8EII721I21D1DBpBVBE=
+sigs.k8s.io/kustomize/kyaml v0.10.9/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

pkg/kptfile/kptfileutil/util.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/GoogleContainerTools/kpt/pkg/kptfile"
 	"sigs.k8s.io/kustomize/kyaml/errors"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
@@ -48,6 +49,12 @@ func ReadFile(dir string) (kptfile.KptFile, error) {
 	if err = d.Decode(&kpgfile); err != nil {
 		return kptfile.KptFile{}, errors.Errorf("unable to parse %q: %v", kptfile.KptFileName, err)
 	}
+	annotations := kpgfile.Annotations
+	if annotations == nil {
+		annotations = make(map[string]string)
+	}
+	annotations[kioutil.PathAnnotation] = dir
+	kpgfile.Annotations = annotations
 	return kpgfile, nil
 }
 

pkg/live/inventoryrg.go

@@ -30,6 +30,8 @@ import (
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
+var AllGroups []string
+
 // ResourceGroupGVK is the group/version/kind of the custom
 // resource used to store inventory.
 var ResourceGroupGVK = schema.GroupVersionKind{
@@ -92,6 +94,19 @@ func (icm *InventoryResourceGroup) ID() string {
 	return ""
 }
 
+func (icm *InventoryResourceGroup) Match(id string) bool {
+	invID := icm.ID()
+	if invID == id {
+		return true
+	}
+	for _, g := range AllGroups {
+		if g == id {
+			return true
+		}
+	}
+	return false
+}
+
 // Load is an Inventory interface function returning the set of
 // object metadata from the wrapped ResourceGroup, or an error.
 func (icm *InventoryResourceGroup) Load() ([]object.ObjMetadata, error) {
@@ -187,6 +202,116 @@ func (icm *InventoryResourceGroup) GetObject() (*unstructured.Unstructured, erro
 	return invCopy, nil
 }
 
+func (icm *InventoryResourceGroup) LoadSubgroups() ([]object.ObjMetadata, error) {
+	objs := []object.ObjMetadata{}
+	if icm.inv == nil {
+		return objs, fmt.Errorf("inventory info is nil")
+	}
+	klog.V(4).Infof("loading inventory...")
+	items, exists, err := unstructured.NestedSlice(icm.inv.Object, "spec", "subgroups")
+	if err != nil {
+		err := fmt.Errorf("error retrieving object metadata from inventory object")
+		return objs, err
+	}
+	if !exists {
+		klog.V(4).Infof("Inventory (spec.resources) does not exist")
+		return objs, nil
+	}
+	klog.V(4).Infof("loading %d inventory items", len(items))
+	for _, itemUncast := range items {
+		item := itemUncast.(map[string]interface{})
+		namespace, _, err := unstructured.NestedString(item, "namespace")
+		if err != nil {
+			return []object.ObjMetadata{}, err
+		}
+		name, _, err := unstructured.NestedString(item, "name")
+		if err != nil {
+			return []object.ObjMetadata{}, err
+		}
+		group, _, err := unstructured.NestedString(item, "group")
+		if err != nil {
+			return []object.ObjMetadata{}, err
+		}
+		kind, _, err := unstructured.NestedString(item, "kind")
+		if err != nil {
+			return []object.ObjMetadata{}, err
+		}
+		groupKind := schema.GroupKind{
+			Group: strings.TrimSpace(group),
+			Kind:  strings.TrimSpace(kind),
+		}
+		klog.V(4).Infof(`creating obj metadata: "%s/%s/%s"`, namespace, name, groupKind)
+		objMeta, err := object.CreateObjMetadata(namespace, name, groupKind)
+		if err != nil {
+			return []object.ObjMetadata{}, err
+		}
+		objs = append(objs, objMeta)
+	}
+	return objs, nil
+
+}
+
+func (icm *InventoryResourceGroup) StoreSubgroups(objMetas []object.ObjMetadata) (*unstructured.Unstructured, error){
+	if icm.inv == nil {
+		return nil, fmt.Errorf("inventory info is nil")
+	}
+	klog.V(4).Infof("getting inventory resource group")
+	// Create a slice of Resources as empty Interface
+	klog.V(4).Infof("Creating list of %d resources", len(objMetas))
+	var objs []interface{}
+	for _, objMeta := range objMetas {
+		klog.V(4).Infof(`storing inventory obj reference: "%s/%s"`, objMeta.Namespace, objMeta.Name)
+		objs = append(objs, map[string]interface{}{
+			"group":     objMeta.GroupKind.Group,
+			"kind":      objMeta.GroupKind.Kind,
+			"namespace": objMeta.Namespace,
+			"name":      objMeta.Name,
+		})
+	}
+	// Create the inventory object by copying the template.
+	// Adds or clears the inventory ObjMetadata to the ResourceGroup "spec.resources" section
+	if len(objs) == 0 {
+		klog.V(4).Infoln("clearing inventory resources")
+		unstructured.RemoveNestedField(icm.inv.UnstructuredContent(),
+			"spec", "subgroups")
+	} else {
+		klog.V(4).Infof("storing inventory (%d) resources", len(objs))
+		err := unstructured.SetNestedSlice(icm.inv.UnstructuredContent(),
+			objs, "spec", "subgroups")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(icm.objMetas) > 0 {
+		var objs []interface{}
+		for _, objMeta := range icm.objMetas {
+			klog.V(4).Infof(`storing inventory obj reference: "%s/%s"`, objMeta.Namespace, objMeta.Name)
+			objs = append(objs, map[string]interface{}{
+				"group":     objMeta.GroupKind.Group,
+				"kind":      objMeta.GroupKind.Kind,
+				"namespace": objMeta.Namespace,
+				"name":      objMeta.Name,
+			})
+		}
+		// Adds or clears the inventory ObjMetadata to the ResourceGroup "spec.resources" section
+		if len(objs) == 0 {
+			klog.V(4).Infoln("clearing inventory resources")
+			unstructured.RemoveNestedField(icm.inv.UnstructuredContent(),
+				"spec", "resources")
+		} else {
+			klog.V(4).Infof("storing inventory (%d) resources", len(objs))
+			err := unstructured.SetNestedSlice(icm.inv.UnstructuredContent(),
+				objs, "spec", "resources")
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return icm.inv, nil
+}
+
 // IsResourceGroupInventory returns true if the passed object is
 // a ResourceGroup inventory object; false otherwise. If an error
 // occurs, then false is returned and the error.

pkg/live/preprocess/process_test.go

@@ -129,3 +129,7 @@ func (f *fakeInventoryClient) UpdateLabels(inv inventory.InventoryInfo, labels m
 	f.inventory.SetLabels(labels)
 	return nil
 }
+
+func (f *fakeInventoryClient) ApplyInventoryObj(u *unstructured.Unstructured) error {
+	return nil
+}

pkg/live/rgloader.go

@@ -19,6 +19,7 @@ var _ manifestreader.ManifestLoader = &ResourceGroupManifestLoader{}
 // ResourceGroup versions of some kpt live apply structures.
 type ResourceGroupManifestLoader struct {
 	factory util.Factory
+	nested bool
 }
 
 // NewResourceGroupProvider encapsulates the passed values, and returns a pointer to an ResourceGroupProvider.
@@ -28,6 +29,13 @@ func NewResourceGroupManifestLoader(f util.Factory) *ResourceGroupManifestLoader
 	}
 }
 
+func NewResourceGroupManifestLoaderNested(f util.Factory) *ResourceGroupManifestLoader {
+	return &ResourceGroupManifestLoader{
+		factory: f,
+		nested: true,
+	}
+}
+
 // Factory returns the kubectl factory.
 func (f *ResourceGroupManifestLoader) InventoryInfo(objs []*unstructured.Unstructured) (inventory.InventoryInfo, []*unstructured.Unstructured, error) {
 	objs, invObj := findResourceGroupInv(objs)
@@ -79,6 +87,7 @@ func (f *ResourceGroupManifestLoader) ManifestReader(reader io.Reader, args []st
 				Path:          args[0],
 				ReaderOptions: readerOptions,
 			},
+			nested: f.nested,
 		}
 	}
 	return rgReader, nil

pkg/live/rgpath.go

@@ -5,19 +5,24 @@ package live
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"github.com/GoogleContainerTools/kpt/pkg/kptfile"
 	"github.com/GoogleContainerTools/kpt/pkg/kptfile/kptfileutil"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/klog"
 	"sigs.k8s.io/cli-utils/pkg/common"
 	"sigs.k8s.io/cli-utils/pkg/manifestreader"
+	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
 )
 
 // ResourceGroupPathManifestReader encapsulates the default path
 // manifest reader.
 type ResourceGroupPathManifestReader struct {
 	pathReader *manifestreader.PathManifestReader
+	nested bool
 }
 
 // Read reads the manifests and returns them as Info objects.
@@ -43,10 +48,33 @@ func (p *ResourceGroupPathManifestReader) Read() ([]*unstructured.Unstructured,
 	if err == nil {
 		klog.V(4).Infof(`from Kptfile generating ResourceGroup inventory object "%s/%s/%s"`,
 			inv.Namespace, inv.Name, inv.InventoryID)
+		if p.nested {
+			annotations := invObj.GetAnnotations()
+			if annotations == nil {
+				annotations = make(map[string]string)
+			}
+			for k, v := range kf.Annotations {
+				if k == kioutil.PathAnnotation {
+					v = strings.TrimPrefix(v, p.pathReader.Path)
+				}
+				annotations[k] = v
+			}
+			invObj.SetAnnotations(annotations)
+		}
 		objs = append(objs, invObj)
 	} else {
 		klog.V(4).Infof("unable to generate ResourceGroup inventory: %s", err)
 	}
+
+	// Read Kptfile from the subdirectories
+	if p.nested {
+		rgs, err := getSubDirResourceGroups(p.pathReader.Path)
+		if err != nil {
+			klog.V(4).Infof("unable to read the sub package level ResourceGroup: %s", err)
+		}
+		objs = append(objs, rgs...)
+	}
+
 	return objs, nil
 }
 
@@ -89,3 +117,45 @@ func ResourceGroupUnstructured(name, namespace, id string) *unstructured.Unstruc
 	}
 	return inventoryObj
 }
+
+func getSubDirResourceGroups(dir string) ([]*unstructured.Unstructured, error) {
+	objs := []*unstructured.Unstructured{}
+
+	err := filepath.Walk(dir,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if path == dir {
+				return nil
+			}
+			if !info.IsDir() {
+				return nil
+			}
+			kf, err := kptfileutil.ReadFile(path)
+			if err != nil {
+				klog.V(4).Infof("unable to parse Kptfile for ResourceGroup inventory: %s", err)
+				return err
+			}
+			inv := kf.Inventory
+			invObj, err := generateInventoryObj(inv)
+			if err == nil {
+				klog.V(4).Infof(`from Kptfile generating ResourceGroup inventory object "%s/%s/%s"`,
+					inv.Namespace, inv.Name, inv.InventoryID)
+				annotations := invObj.GetAnnotations()
+				if annotations == nil {
+					annotations = make(map[string]string)
+				}
+				for k, v := range kf.Annotations {
+					if k == kioutil.PathAnnotation {
+						v = strings.TrimPrefix(strings.TrimPrefix(v, dir), string(filepath.Separator))
+					}
+					annotations[k] = v
+				}
+				invObj.SetAnnotations(annotations)
+				objs = append(objs, invObj)
+			}
+			return nil
+		})
+	return objs, err
+}