client-go: parse kubernetes json files to k8s structures

Question

i would like to parse kubernetes manifest file (json/yaml) and be able to convert them to k8s structures (to later on manipulate them)

I know there is the NewYAMLOrJSONDecoder().Decode() function (https://github.com/kubernetes/apimachinery/blob/master/pkg/util/yaml/decoder.go) to read a json/yaml file, but the next step is: how to convert them to k8s structure/type?

i.e. if I read a yaml file with a Namespace object, how to convert it to a core/v1/namespace interface for example

Regards,

Solution 1

Thanks svenwltr, I was not aware we can do like this.

In the same time, I manage to find not a better approach but a different one:

package main

import (
    "flag"
    "fmt"
    "os"
    "io"
    "path/filepath"
    "log"
    "encoding/json"
    //"time"

    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/client-go/kubernetes"
    "k8s.io/client-go/tools/clientcmd"
    "k8s.io/client-go/discovery"
    "k8s.io/client-go/dynamic"
    "k8s.io/apimachinery/pkg/util/yaml"
    "k8s.io/apimachinery/pkg/runtime"
    "k8s.io/apimachinery/pkg/runtime/schema"
    "k8s.io/apimachinery/pkg/api/meta"
    "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

)

func main() {
    var kubeconfig *string
    if home := homeDir(); home != "" {
        kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
    } else {
        kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
    }
    flag.Parse()

    // use the current context in kubeconfig
    config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
    if err != nil {
        panic(err.Error())
    }

    // create the clientset
    clientset, err := kubernetes.NewForConfig(config)
    if err != nil {
        panic(err.Error())
    }

    f,err := os.Open("namespace.yaml")
    if err!=nil {
        log.Fatal(err)
    }
    d := yaml.NewYAMLOrJSONDecoder(f,4096)
    dd := clientset.Discovery()
    apigroups,err := discovery.GetAPIGroupResources(dd)
    if err != nil {
        log.Fatal(err)
    }

    restmapper := discovery.NewRESTMapper(apigroups,meta.InterfacesForUnstructured)


    for {
        // https://github.com/kubernetes/apimachinery/blob/master/pkg/runtime/types.go
        ext := runtime.RawExtension{}
        if err := d.Decode(&ext); err!=nil {
            if err == io.EOF {
                break
            }
            log.Fatal(err)
        }
        fmt.Println("raw: ",string(ext.Raw))
        versions := &runtime.VersionedObjects{}
        //_, gvk, err := objectdecoder.Decode(ext.Raw,nil,versions)
        obj, gvk, err := unstructured.UnstructuredJSONScheme.Decode(ext.Raw,nil,versions)
        fmt.Println("obj: ",obj)

        // https://github.com/kubernetes/apimachinery/blob/master/pkg/api/meta/interfaces.go
        mapping, err := restmapper.RESTMapping(gvk.GroupKind(), gvk.Version)
        if err != nil {
            log.Fatal(err)
        }

        restconfig := config
        restconfig.GroupVersion = &schema.GroupVersion {
            Group: mapping.GroupVersionKind.Group,
            Version: mapping.GroupVersionKind.Version,
        } 
        dclient,err := dynamic.NewClient(restconfig)
        if err != nil {
            log.Fatal(err)
        }

        // https://github.com/kubernetes/client-go/blob/master/discovery/discovery_client.go
        apiresourcelist, err := dd.ServerResources()
        if err != nil {
            log.Fatal(err)
        }
        var myapiresource metav1.APIResource
        for _,apiresourcegroup := range(apiresourcelist) {
            if apiresourcegroup.GroupVersion == mapping.GroupVersionKind.Version {
                for _,apiresource := range(apiresourcegroup.APIResources) {
                    //fmt.Println(apiresource)

                    if apiresource.Name == mapping.Resource && apiresource.Kind == mapping.GroupVersionKind.Kind {
                        myapiresource = apiresource
                    }
                }
            }
        }
        fmt.Println(myapiresource)
        // https://github.com/kubernetes/client-go/blob/master/dynamic/client.go

        var unstruct unstructured.Unstructured
        unstruct.Object = make(map[string]interface{})
        var blob interface{}
        if err := json.Unmarshal(ext.Raw,&blob); err != nil {
            log.Fatal(err)
        }
        unstruct.Object = blob.(map[string]interface{})
        fmt.Println("unstruct:",unstruct)
        ns := "default"
        if md,ok := unstruct.Object["metadata"]; ok {
            metadata := md.(map[string]interface{})
            if internalns,ok := metadata["namespace"]; ok {
                ns = internalns.(string)
            }
        }
        res := dclient.Resource(&myapiresource,ns)
        fmt.Println(res)
        us,err := res.Create(&unstruct)
        if err != nil {
            log.Fatal(err)
        }
        fmt.Println("unstruct response:",us)


    }
}

func homeDir() string {
    if h := os.Getenv("HOME"); h != "" {
        return h
    }
    return os.Getenv("USERPROFILE") // windows
}

Solution 2

   import (
        v1 "k8s.io/api/core/v1"
        "k8s.io/apimachinery/pkg/runtime"
        "k8s.io/apimachinery/pkg/runtime/serializer"
        "k8s.io/client-go/kubernetes/scheme"
    )
    
    func decode(data []byte) (*v1.Namespace, error) {
        decoder := serializer.NewCodecFactory(scheme.Scheme).UniversalDecoder()
        object := &v1.Namespace{}
        err := runtime.DecodeInto(decoder, data, object)
        if err != nil {
            return nil, err
        }
        return object, nil
    }

Pass SchemeGroupVersion generated for your CRDs to UniversalDecoder call, if needed.

Use runtime.Decode instead of runtime.DecodeInto to decode unspecified object type.

Solution 3

This question is very similar to How to deserialize Kubernetes YAML file, but this question is a bit outdated, since the package names changed.

Also it doesn't directly use the go client, which means there might be another solution to this.

Here is a example:

package main

import (
    "fmt"

    "k8s.io/kubernetes/pkg/api"
    _ "k8s.io/kubernetes/pkg/api/install"
    _ "k8s.io/kubernetes/pkg/apis/extensions/install"
    "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
)

var json = `
{
  "apiVersion": "extensions/v1beta1",
  "kind": "Deployment",
  "metadata": null,
  "name": "my-nginx",
  "replicas": 2,
  "spec": null,
  "template": {
    "metadata": {
      "labels": {
        "run": "my-nginx"
      }
    },
    "spec": {
      "containers": [
        {
          "image": "nginx",
          "name": "my-nginx",
          "ports": [
            {
              "containerPort": 80
            }
          ]
        }
      ]
    }
  }
}
`

func main() {
    // decode := api.Codecs.UniversalDecoder().Decode
    decode := api.Codecs.UniversalDeserializer().Decode

    obj, _, err := decode([]byte(json), nil, nil)
    if err != nil {
        fmt.Printf("%#v", err)
    }

    deployment := obj.(*v1beta1.Deployment)

    fmt.Printf("%#v\n", deployment)
}

Notes

• the .../install packages are important, since they define which types can get decoded
• it's able to decode JSON, YAML and probably also all other supported file types
• not sure where the difference between UniversalDecoder and UniversalDeserializer is