Skip to content

Instantly share code, notes, and snippets.

@dragonsinth
Last active August 1, 2024 13:47
Show Gist options
  • Save dragonsinth/aea365732b60da3adc928dc18fff56ed to your computer and use it in GitHub Desktop.
Save dragonsinth/aea365732b60da3adc928dc18fff56ed to your computer and use it in GitHub Desktop.
Connect to Google Kubernetes with GCP credentials and pure Golang
package main
import (
"context"
"encoding/base64"
"flag"
"fmt"
"log"
container "google.golang.org/api/container/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/clientcmd/api"
_ "k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp" // register GCP auth provider
)
var fProjectId = flag.String("projectId", "", "specify a project id to examine")
func main() {
flag.Parse()
if *fProjectId == "" {
log.Fatal("must specific -projectId")
}
if err := run(context.Background(), *fProjectId); err != nil {
log.Fatal(err)
}
}
func run(ctx context.Context, projectId string) error {
kubeConfig, err := getK8sClusterConfigs(ctx, projectId)
if err != nil {
return err
}
// Just list all the namespaces found in the project to test the API.
for clusterName := range kubeConfig.Clusters {
cfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeConfig, clusterName, &clientcmd.ConfigOverrides{CurrentContext: clusterName}, nil).ClientConfig()
if err != nil {
return fmt.Errorf("failed to create Kubernetes configuration cluster=%s: %w", clusterName, err)
}
k8s, err := kubernetes.NewForConfig(cfg)
if err != nil {
return fmt.Errorf("failed to create Kubernetes client cluster=%s: %w", clusterName, err)
}
ns, err := k8s.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
if err != nil {
return fmt.Errorf("failed to list namespaces cluster=%s: %w", clusterName, err)
}
log.Printf("Namespaces found in cluster=%s", clusterName)
for _, item := range ns.Items {
log.Println(item.Name)
}
}
return nil
}
func getK8sClusterConfigs(ctx context.Context, projectId string) (*api.Config, error) {
svc, err := container.NewService(ctx)
if err != nil {
return nil, fmt.Errorf("container.NewService: %w", err)
}
// Basic config structure
ret := api.Config{
APIVersion: "v1",
Kind: "Config",
Clusters: map[string]*api.Cluster{}, // Clusters is a map of referencable names to cluster configs
AuthInfos: map[string]*api.AuthInfo{}, // AuthInfos is a map of referencable names to user configs
Contexts: map[string]*api.Context{}, // Contexts is a map of referencable names to context configs
}
// Ask Google for a list of all kube clusters in the given project.
resp, err := svc.Projects.Zones.Clusters.List(projectId, "-").Context(ctx).Do()
if err != nil {
return nil, fmt.Errorf("clusters list project=%s: %w", projectId, err)
}
for _, f := range resp.Clusters {
name := fmt.Sprintf("gke_%s_%s_%s", projectId, f.Zone, f.Name)
cert, err := base64.StdEncoding.DecodeString(f.MasterAuth.ClusterCaCertificate)
if err != nil {
return nil, fmt.Errorf("invalid certificate cluster=%s cert=%s: %w", name, f.MasterAuth.ClusterCaCertificate, err)
}
// example: gke_my-project_us-central1-b_cluster-1 => https://XX.XX.XX.XX
ret.Clusters[name] = &api.Cluster{
CertificateAuthorityData: cert,
Server: "https://" + f.Endpoint,
}
// Just reuse the context name as an auth name.
ret.Contexts[name] = &api.Context{
Cluster: name,
AuthInfo: name,
}
// GCP specific configation; use cloud platform scope.
ret.AuthInfos[name] = &api.AuthInfo{
AuthProvider: &api.AuthProviderConfig{
Name: "gcp",
Config: map[string]string{
"scopes": "https://www.googleapis.com/auth/cloud-platform",
},
},
}
}
return &ret, nil
}
module github.com/dragonsinth/gcp-kube
go 1.21.6
require (
google.golang.org/api v0.63.0
k8s.io/apimachinery v0.26.0
k8s.io/client-go v0.26.0
k8s.io/cloud-provider-gcp v0.0.0-20230119221216-bb1acae5826d
)
require (
cloud.google.com/go v0.99.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/term v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.49.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.26.0 // indirect
k8s.io/klog/v2 v2.80.1 // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
@benpoliquin
Copy link

Just wanted to say thank you and this helped answer a question I had. The other thing I needed to do was generate a token using https://pkg.go.dev/google.golang.org/api/transport#Creds and place it inside of api.AuthInfo{}

@dragonsinth
Copy link
Author

@benpoliquin glad it was helpful!

Did you get here from my blog post?

@benpoliquin
Copy link

@benpoliquin glad it was helpful!

Did you get here from my blog post?

That I did. Thank you for the write-up!

@shakilbd009
Copy link

@benpoliquin glad it was helpful!

Did you get here from my blog post?

Oh man i cannot thank you enough, your post single handedly saved me.
However, for some reason finding your post while googling took some time. Just when I about to gave up i tried searching "k8s.io/client-go/plugin/pkg/client/auth/gcp" and your post was no 5 in the search list.

if your title was "authenticating to gke without kubeconfig" or similar, should be picked up by google search.

it was a great post , i cannot appreciate enough and i believe we can use same approach for AKS or EKS ?

@dragonsinth
Copy link
Author

dragonsinth commented Feb 18, 2021

@shakilbd009 oh man, sorry it was hard to find, but thank you very much for the suggestion. I've updated the blog post to add a subtitle, so who knows maybe Google will find it more easily under those search terms in the future!

EDIT: looks like we're now on page 1 for "authenticate to gke without kubeconfig". Thanks so much for the idea!

I don't know about authenticating to other Kube providers-- I only have access to GKE myself. But I would imagine something similar should work?

@zreigz
Copy link

zreigz commented Oct 21, 2021

Just wanted to say thank you and this helped answer a question I had. The other thing I needed to do was generate a token using https://pkg.go.dev/google.golang.org/api/transport#Creds and place it inside of api.AuthInfo{}

@benpoliquin can you paste some snippet of how to do this?

@nstogner
Copy link

@zreigz

	creds, err := transport.Creds(context.Background(), option.WithCredentialsFile("path/to/keyfile.json"), option.WithScopes("https://www.googleapis.com/auth/cloud-platform"))
	if err != nil {
		return fmt.Errorf("generating gcp creds: %w", err)
	}
	tkn, err := creds.TokenSource.Token()
	if err != nil {
		return fmt.Errorf("generating token: %w", err)
	}
	ret.AuthInfos[name] = &k8sapi.AuthInfo{
		Token: tkn.AccessToken,
		AuthProvider: &k8sapi.AuthProviderConfig{
			Name: "gcp",
			Config: map[string]string{
				"scopes": "https://www.googleapis.com/auth/cloud-platform",
			},
		},
	}

@zreigz
Copy link

zreigz commented Oct 26, 2021

@nstogner thanks a lot. I used this method for the token:

creds, err := google.CredentialsFromJSON(ctx, b, container.CloudPlatformScope)

@ucguy4u
Copy link

ucguy4u commented Jan 23, 2023

I am getting following error while executing

2023/01/23 11:47:49 failed to create Kubernetes client cluster=<cluster_name>: The gcp auth plugin has been removed.
Please use the "gke-gcloud-auth-plugin" kubectl/client-go credential plugin instead.
See https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke for further details
exit status 1

@dragonsinth
Copy link
Author

I'll have to re-test, there's a new this works in 1.2.26, with respect to the CLI. I'm not sure what impact this has on pure Go. I'll dig around and see what I find.

@dragonsinth
Copy link
Author

Okay, I have an updated solution that should work with newer libs.

  1. In your go.mod, replace k8s.io/cloud-provider-gcp/providers => k8s.io/cloud-provider-gcp/providers v0.25.5

  2. go get k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp@bb1acae5826dc877953d4854faf414e860db2efa

  3. Change the import: _ "k8s.io/cloud-provider-gcp/pkg/clientauthplugin/gcp" // register GCP auth provider plugin.

The code moved here (but note the library is unsupported): https://github.com/kubernetes/cloud-provider-gcp/tree/bb1acae5826dc877953d4854faf414e860db2efa/pkg/clientauthplugin

@ucguy4u
Copy link

ucguy4u commented Feb 1, 2023

Hey @dragonsinth it's working for me. But when i try to import same with helm packages it is upgrading the kubernetes dependency

i am trying to import the helm dependency :

"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/cli"

I want to fetch all the installed helm releases from running cluster.
But when i do

go get helm.sh/helm/v3/pkg/action
go get helm.sh/helm/v3/pkg/cli

In go.mod file it is upgrading the version of k8s

k8s.io/client-go v0.25.5 - > v0.26.0

Can you please help me with this issue.

@dragonsinth
Copy link
Author

@ucguy4u
Copy link

ucguy4u commented Feb 6, 2023

Client dependencies being upgraded forcefully after installing the helm modules

go get helm.sh/helm/v3/pkg/action
go get helm.sh/helm/v3/pkg/cli
k8s.io/client-go v0.25.5 - > v0.26.0

To simulate the issue :
Can you please try to import the helm module, after that you will see it is not working.

@dragonsinth
Copy link
Author

@ucguy4u I'm on k8s.io/client-go v0.26.0 and it works fine for me; v0.26.1 also works fine

@iamsudip
Copy link

iamsudip commented May 5, 2024

I'm on k8s.io/client-go v0.30.0 which is latest as of today and this is still working as suggested. Thank you so much. This still helps me implement my stuff in pure go so far, last option would be to use the CLI if it breaks someday.

@dragonsinth
Copy link
Author

dragonsinth commented May 5, 2024

👍 glad folks are still finding this useful. I went ahead and made an update for go 1.21.6 and included a working go.mod file here; I think this should just work out of the box in go 1.21 now. (I didn't bother including a go.sum -- seemed spammy? But I could attach my go.sum as well if that's useful for folks.)

@iamsudip
Copy link

iamsudip commented May 5, 2024

Even it's working for 1.22.2 which is latest as of today. Glad that you added go.mod. Thanks.

@dragonsinth
Copy link
Author

Interestingly, when I updated today I found I no longer needed the replace in the go mod... I wonder if that means this could now be transformed into a tiny shared lib.

@iamsudip
Copy link

iamsudip commented May 5, 2024

Yes, you are right, I didn't mention it earlier, the first step in the comment is no longer needed.

@Tang8330
Copy link

Is there a more updated version that is not using clientauthplugin?

Per docs:

Client auth library is a copy of the gcp plugin library that existed in Client-Go until k8s 1.25. Going ahead, this library is not supported and is expected to be replaced by GKE-GCLOUD-AUTH-PLUGIN. The code here is kept as a reference, in case anyone needs to refer to it.

https://github.com/kubernetes/cloud-provider-gcp/tree/master/pkg/clientauthplugin

@dragonsinth
Copy link
Author

@Tang8330 feel free to hack on one, I'm stiil using the old package. According to the new docs:

You will need to install the gke-gcloud-auth-plugin binary on all systems where kubectl or Kubernetes custom clients are used.

I wanted a pure Go solution, whereas it looks like the new stuff requires an installed binary.

@Tang8330
Copy link

@dragonsinth I know :'(. I have it working by installing the binary, but it feels dirty.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment