Last active
February 8, 2020 20:02
-
-
Save paparaka/5c25bfab5169f016a913955f85e7ddfb to your computer and use it in GitHub Desktop.
K8.Testing.CloudBuild
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# A Cloud Build file for the Docker image. We run it manually only on when we update the container. | |
steps: | |
- name: 'gcr.io/cloud-builders/docker' | |
id: 'build_image' | |
args: | |
- 'build' | |
- '--tag=gcr.io/$PROJECT_ID/cloud-builders-python:latest' | |
- '--tag=gcr.io/$PROJECT_ID/cloud-builders-python:$SHORT_SHA' | |
- '.' | |
dir: 'external/cloud-builders-python' | |
images: | |
- 'gcr.io/$PROJECT_ID/cloud-builders-python:latest' | |
- 'gcr.io/$PROJECT_ID/cloud-builders-python:$SHORT_SHA' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
steps: | |
# ...... | |
# Create namespace first, otherwise K8 will complain | |
- name: 'gcr.io/cloud-builders/kubectl' | |
args: ['apply', '-f', 'deployment/kubernetes/namespace.yaml'] | |
env: | |
- 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE' | |
- 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER' | |
# Create configmap as Tests need it | |
- name: 'gcr.io/cloud-builders/kubectl' | |
args: ['apply', '-f', 'deployment/kubernetes/deployments/0.configmap.yaml'] | |
env: | |
- 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE' | |
- 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER' | |
# Remove any old tests and run new one | |
- name: gcr.io/cloud-builders/gcloud | |
entrypoint: '/bin/bash' | |
args: | |
- '-c' | |
- | | |
echo "==========================================================" | |
[[ "$_ENVR" == "prod" ]] && exit 0 || echo "Running Unit Tests" | |
echo "[Deleting old job]" | |
kubectl delete -f deployment/kubernetes/jobs/tests.yaml | |
echo "[Applying new job]" | |
kubectl apply -f deployment/kubernetes/jobs/tests.yaml | |
env: | |
- 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE' | |
- 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER' | |
# Check if Test Job has completed | |
- name: 'gcr.io/$PROJECT_ID/cloud-builders-python:latest' | |
args: ['python3', 'deployment/scripts/k8_watch_job.py'] | |
env: | |
- 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE' | |
- 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER' | |
- '_ENVR=$_ENVR' | |
- 'JOB_NAME=api-test' | |
# Apply K8 Deployment & Services | |
- name: 'gcr.io/cloud-builders/kubectl' | |
args: ['apply', '-f', 'deployment/kubernetes/deployments',] | |
env: | |
- 'CLOUDSDK_COMPUTE_ZONE=$_CLOUDSDK_COMPUTE_ZONE' | |
- 'CLOUDSDK_CONTAINER_CLUSTER=$_CLOUDSDK_CONTAINER_CLUSTER' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FROM gcr.io/cloud-builders/gcloud | |
RUN apt-get -y update && \ | |
apt-get -y install python3 python3-dev python3-setuptools python3-pip | |
RUN pip3 install PyYAML Jinja2 simplejson kubernetes \ | |
google-cloud-storage google-api-python-client | |
RUN apt-get -y remove python-dev python-setuptools wget && \ | |
rm -rf /var/lib/apt/lists/* && \ | |
rm -rf ~/.config/gcloud && \ | |
chmod +x /entrypoint.bash | |
COPY entrypoint.bash /entrypoint.bash | |
ENTRYPOINT ["/entrypoint.bash"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -e | |
####################################################################################### | |
# A Docker ENTRYPOINT script that authenticates with GCP Kubernetes and then runs any CMD | |
# Requires CLOUDSDK_CONTAINER_CLUSTER, CLOUDSDK_COMPUTE_REGION and CLOUDSDK_COMPUTE_ZONE | |
# env variables set | |
# https://success.docker.com/article/use-a-script-to-initialize-stateful-container-data | |
# https://github.com/kubernetes-client/python/blob/master/kubernetes/client/api/batch_v1_api.py | |
# | |
####################################################################################### | |
# If there is no current context, get one. | |
if [[ $(kubectl config current-context 2> /dev/null) == "" ]]; then | |
# This tries to read environment variables. If not set, it grabs from gcloud | |
cluster=${CLOUDSDK_CONTAINER_CLUSTER:-$(gcloud config get-value container/cluster 2> /dev/null)} | |
region=${CLOUDSDK_COMPUTE_REGION:-$(gcloud config get-value compute/region 2> /dev/null)} | |
zone=${CLOUDSDK_COMPUTE_ZONE:-$(gcloud config get-value compute/zone 2> /dev/null)} | |
project=${GCLOUD_PROJECT:-$(gcloud config get-value core/project 2> /dev/null)} | |
function var_usage() { | |
cat <<EOF | |
No cluster is set. To set the cluster (and the region/zone where it is found), set the environment variables | |
CLOUDSDK_COMPUTE_REGION={region} (regional clusters) | |
CLOUDSDK_COMPUTE_ZONE=${zone} (zonal clusters) | |
CLOUDSDK_CONTAINER_CLUSTER=${cluster} | |
EOF | |
exit 1 | |
} | |
[[ -z "$cluster" ]] && var_usage | |
[ ! "$zone" -o "$region" ] && var_usage | |
if [ -n "$region" ]; then | |
echo "Running: gcloud config set container/use_v1_api_client false" | |
gcloud config set container/use_v1_api_client false | |
echo "Running: gcloud beta container clusters get-credentials --project=\"$project\" --region=\"$region\" \"$cluster\"" | |
gcloud beta container clusters get-credentials --project="$project" --region="$region" "$cluster" || exit | |
else | |
echo "Running: gcloud container clusters get-credentials --project=\"$project\" --zone=\"$zone\" \"$cluster\"" | |
gcloud container clusters get-credentials --project="$project" --zone="$zone" "$cluster" || exit | |
fi | |
fi | |
eval "$@" | |
exitCode=$? | |
exit $exitCode |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Bellow are the 3 example outputs of read_namespaced_job_status(), | |
# I collected during the development of this: | |
SUCCESS = {'status': {'active': None, | |
'completion_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()), | |
'conditions': [{'last_probe_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()), | |
'last_transition_time': datetime.datetime(2020, 2, 4, 15, 32, 52, tzinfo=tzlocal()), | |
'message': None, | |
'reason': None, | |
'status': 'True', | |
'type': 'Complete'}], | |
'failed': None, | |
'start_time': datetime.datetime(2020, 2, 4, 15, 32, 35, tzinfo=tzlocal()), | |
'succeeded': 1}} | |
ONGOING = {'status': {'active': 1, | |
'completion_time': None, | |
'conditions': None, | |
'failed': None, | |
'start_time': datetime.datetime(2020, 2, 5, 16, 4, 20, tzinfo=tzlocal()), | |
'succeeded': None}} | |
FAILED ='status': {'active': None, | |
'completion_time': None, | |
'conditions': [{'last_probe_time': datetime.datetime(2020, 2, 5, 16, 6, 18, tzinfo=tzlocal()), | |
'last_transition_time': datetime.datetime(2020, 2, 5, 16, 6, 18, tzinfo=tzlocal()), | |
'message': 'Job has reached the specified backoff ' | |
'limit', | |
'reason': 'BackoffLimitExceeded', | |
'status': 'True', | |
'type': 'Failed'}], | |
'failed': None, | |
'start_time': datetime.datetime(2020, 2, 5, 16, 4, 20, tzinfo=tzlocal()), | |
'succeeded': None}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
A tool that uses the kubernetes API to "watch" a K8 job till completion. | |
If the job failed for any reason an exception is raised with the failure as a message. | |
The script expects the folling ENV vars: | |
- JOB_NAME: the name of the Kubernetes Job | |
- _ENVR: the namespace in Kubernetes | |
In addition the kubernetes.config expects that there is a `~/.kube/config` file that holds the K8 credentials | |
""" | |
import os | |
import time | |
import sys | |
from kubernetes import client, config | |
config.load_kube_config() | |
kclient = client.BatchV1Api() | |
def check_google_auth(): | |
from google.auth import compute_engine | |
credentials = compute_engine.Credentials() | |
print(credentials) | |
def get_timeout(): | |
timeout = os.environ.get('TIMEOUT') | |
if timeout is not None: | |
return int(timeout) | |
else: | |
return 300 | |
def watch_job(timeout=300, **kwargs): | |
""" | |
:param timeout: max seconds/cycles to run the script for | |
:param kwargs: arguments to be passed to `read_namespaced_job_status()` | |
:return: True if success, Exception otherwise | |
""" | |
active = True | |
this_job = None | |
counter = timeout | |
while active: | |
if counter == 0: | |
raise Exception('Timed out after: {}s'.format(timeout)) | |
this_job = kclient.read_namespaced_job_status(**kwargs) | |
active = bool(this_job._status.active) | |
time.sleep(1) | |
counter -= 1 | |
if bool(this_job._status.succeeded): | |
print("Job completed successfully") | |
return True | |
elif this_job._status.succeeded is None: | |
print("Job failed") | |
raise Exception(this_job._status.conditions[0].reason) | |
else: | |
msg = "Job failed for unknown reason" | |
print(msg) | |
print(this_job._status) | |
raise Exception(msg) | |
if __name__ == '__main__': | |
if os.environ['_ENVR'] == 'prod': | |
# The tests don't run in prod, so this is a 'temporary' workaround | |
sys.exit(0) | |
watch_job( | |
timeout=get_timeout(), | |
name=os.environ['JOB_NAME'], | |
namespace=os.environ['_ENVR'] | |
) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment