Created
February 18, 2019 20:31
-
-
Save ned1313/c7756a2d0640412dae8ad55d97c74893 to your computer and use it in GitHub Desktop.
Container script for Kubernetes Cluster on Azure Stack
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
set -e | |
echo "Starting deploying Kubernetes cluster" | |
date | |
echo "Running as:" | |
whoami | |
sleep 20 | |
# Script parameters | |
echo "RESOURCE_GROUP_NAME: $RESOURCE_GROUP_NAME" | |
echo "PUBLICIP_DNS: $PUBLICIP_DNS" | |
echo "TENANT_ID: $TENANT_ID" | |
echo "TENANT_SUBSCRIPTION_ID: $TENANT_SUBSCRIPTION_ID" | |
echo "ADMIN_USERNAME: $ADMIN_USERNAME" | |
echo "MASTER_DNS_PREFIX: $MASTER_DNS_PREFIX" | |
echo "AGENT_COUNT: $AGENT_COUNT" | |
echo "AGENT_SIZE: $AGENT_SIZE" | |
echo "MASTER_COUNT: $MASTER_COUNT" | |
echo "MASTER_SIZE: $MASTER_SIZE" | |
echo "K8S_AZURE_CLOUDPROVIDER_VERSION: $K8S_AZURE_CLOUDPROVIDER_VERSION" | |
echo "REGION_NAME: $REGION_NAME" | |
echo "SSH_PUBLICKEY: $SSH_PUBLICKEY" | |
echo "PUBLICIP_FQDN: $PUBLICIP_FQDN" | |
echo "STORAGE_PROFILE: $STORAGE_PROFILE" | |
echo "IDENTITY_SYSTEM: $IDENTITY_SYSTEM" | |
echo 'Printing the system information' | |
#sudo uname -a | |
retrycmd_if_failure() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } | |
check_and_move_azurestack_configuration() { | |
if [ -s $1 ] ; then | |
echo "Found $1 in $PWD and is > 0 bytes" | |
else | |
echo "File $1 does not exist in $PWD or is zero length. Error happend during building the input API model or cluster definition." | |
exit 1 | |
fi | |
echo "Moving $1 to $2" | |
sudo mv $1 $2 | |
echo "Done building the API model based on the stamp information." | |
} | |
convert_to_cert() { | |
echo 'converting to file' | |
echo $1 | base64 --decode > cert.json | |
cat cert.json | jq '.data' | tr -d \" | base64 --decode > $CERTIFICATE_PFX_LOCATION | |
PASSWORD=$(cat cert.json | jq '.password' | tr -d \") | |
echo "Converting to certificate" | |
openssl pkcs12 -in $CERTIFICATE_PFX_LOCATION -clcerts -nokeys -out $CERTIFICATE_LOCATION -passin pass:$PASSWORD | |
echo "Converting into key" | |
openssl pkcs12 -in $CERTIFICATE_PFX_LOCATION -nocerts -nodes -out $KEY_LOCATION -passin pass:$PASSWORD | |
} | |
function ensureCertificates() | |
{ | |
echo "Updating certificates" | |
# Copying the AzureStack root certificate to the appropriate store to be updated. | |
AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH="/var/lib/waagent/Certificates.pem" | |
AZURESTACK_ROOT_CERTIFICATE_DEST_PATH="/usr/local/share/ca-certificates/azsCertificate.crt" | |
AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT=`openssl x509 -in $AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH -noout -fingerprint` | |
sudo cp $AZURESTACK_ROOT_CERTIFICATE_SOURCE_PATH $AZURESTACK_ROOT_CERTIFICATE_DEST_PATH | |
AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT=`openssl x509 -in $AZURESTACK_ROOT_CERTIFICATE_DEST_PATH -noout -fingerprint` | |
echo "AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT: $AZURESTACK_ROOT_CERTIFICATE_SOURCE_FINGERPRINT" | |
echo "AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT: $AZURESTACK_ROOT_CERTIFICATE_DEST_FINGERPRINT" | |
update-ca-certificates | |
AZURESTACK_RESOURCE_METADATA_ENDPOINT="$TENANT_ENDPOINT/metadata/endpoints?api-version=2015-01-01" | |
curl $AZURESTACK_RESOURCE_METADATA_ENDPOINT | |
} | |
echo "Update the system." | |
retrycmd_if_failure 5 10 sudo apt-get update -y | |
echo "Installing pax for string manipulation." | |
retrycmd_if_failure 5 10 sudo apt-get install pax -y | |
echo "Installing jq for JSON manipulation." | |
retrycmd_if_failure 5 10 sudo apt-get install jq -y | |
echo "Installing curl." | |
retrycmd_if_failure 5 10 sudo apt-get install curl -y | |
echo "Update the system." | |
retrycmd_if_failure 5 10 sudo apt-get update -y | |
echo "Cloning the ACS-Engine repo/branch: msazurestackworkloads, azsmaster" | |
git clone https://github.com/msazurestackworkloads/acs-engine -b azsmaster | |
cd acs-engine | |
echo "We are going to use an existing ACS-Engine binary." | |
echo "Open the zip file from the repo location." | |
sudo mkdir bin | |
sudo tar -zxvf examples/azurestack/acs-engine.tgz | |
sudo mv acs-engine bin/ | |
echo "Checkign if acs-engine binary is available." | |
if [ -f "./bin/acs-engine" ] ; then | |
echo "Found acs-engine.exe" | |
else | |
echo "Missing acs-engine.exe. Exiting!" | |
exit 1 | |
fi | |
EXTERNAL_FQDN="${PUBLICIP_FQDN//$PUBLICIP_DNS.$REGION_NAME.cloudapp.}" | |
TENANT_ENDPOINT="https://management.$REGION_NAME.$EXTERNAL_FQDN" | |
echo "EXTERNAL_FQDN is:$EXTERNAL_FQDN" | |
echo "TENANT_ENDPOINT is:$TENANT_ENDPOINT" | |
retrycmd_if_failure 20 30 ensureCertificates | |
echo 'Retrieve the AzureStack root CA certificate thumbprint' | |
THUMBPRINT=$(openssl x509 -in /var/lib/waagent/Certificates.pem -fingerprint -noout | cut -d'=' -f 2 | tr -d :) | |
echo 'Thumbprint for AzureStack root CA certificate:' $THUMBPRINT | |
SUFFIXES_STORAGE_ENDPOINT=$REGION_NAME.$EXTERNAL_FQDN | |
SUFFIXES_KEYVAULT_DNS=.vault.$REGION_NAME.$EXTERNAL_FQDN | |
FQDN_ENDPOINT_SUFFIX=cloudapp.$EXTERNAL_FQDN | |
ENVIRONMENT_NAME=AzureStackCloud | |
METADATA=`curl --retry 10 $TENANT_ENDPOINT/metadata/endpoints?api-version=2015-01-01` | |
ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID=`echo $METADATA | jq '.authentication.audiences'[0] | tr -d \"` | |
#trim "adfs" from end of login endpoint if it is ADFS | |
if [ $IDENTITY_SYSTEM == "ADFS" ] | |
then | |
echo "Using ADFS" | |
ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=`echo $METADATA | jq '.authentication.loginEndpoint' | tr -d \" | sed -e 's/adfs*$//' | tr -d \" ` | |
echo "ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT is:"$ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT | |
else | |
echo "Using AAD" | |
ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT=`echo $METADATA | jq '.authentication.loginEndpoint' | tr -d \"` | |
fi | |
#if it is adfs, parse SPN_CLIENT_SECRET to get pfx and password | |
#convert pfx to cert and key | |
if [ $IDENTITY_SYSTEM == "ADFS" ] | |
then | |
echo "Using ADFS" | |
CERTIFICATE_LOCATION="spnauth.crt" | |
KEY_LOCATION="spnauth.key" | |
CERTIFICATE_PFX_LOCATION="spnauth.pfx" | |
convert_to_cert $SPN_CLIENT_SECRET $CERTIFICATE_LOCATION $KEY_LOCATION $CERTIFICATE_PFX_LOCATION | |
fi | |
ENDPOINT_GRAPH_ENDPOINT=`echo $METADATA | jq '.graphEndpoint' | tr -d \"` | |
ENDPOINT_GALLERY=`echo $METADATA | jq '.galleryEndpoint' | tr -d \"` | |
echo 'Overriding the default file with the correct values in the API model or the cluster definition.' | |
if [ -f "examples/azurestack/azurestack-kubernetes$K8S_AZURE_CLOUDPROVIDER_VERSION.json" ] | |
then | |
echo "Found azurestack-kubernetes$K8S_AZURE_CLOUDPROVIDER_VERSION.json." | |
else | |
echo "File azurestack-kubernetes$K8S_AZURE_CLOUDPROVIDER_VERSION.json does not exist. Exiting..." | |
exit 1 | |
fi | |
AZURESTACK_CONFIGURATION_TEMP="${AZURESTACK_CONFIGURATION_TEMP:-azurestack_temp.json}" | |
AZURESTACK_CONFIGURATION="${AZURESTACK_CONFIGURATION:-azurestack.json}" | |
echo "Copying the default file API model to $PWD." | |
sudo cp examples/azurestack/azurestack-kubernetes$K8S_AZURE_CLOUDPROVIDER_VERSION.json $AZURESTACK_CONFIGURATION | |
if [ -s "$AZURESTACK_CONFIGURATION" ] ; then | |
echo "Found $AZURESTACK_CONFIGURATION in $PWD and is > 0 bytes" | |
else | |
echo "File $AZURESTACK_CONFIGURATION does not exist in $PWD or is zero length." | |
exit 1 | |
fi | |
STORAGE_PROFILE="${STORAGE_PROFILE:-blobdisk}" | |
# use blobdisk and AvailabilitySet | |
if [ "$STORAGE_PROFILE" == "blobdisk" ] ; then | |
sudo cat $AZURESTACK_CONFIGURATION | jq --arg AvailabilitySet "AvailabilitySet" '.properties.agentPoolProfiles[0].availabilityProfile=$AvailabilitySet' | \ | |
jq --arg StorageAccount "StorageAccount" '.properties.agentPoolProfiles[0].storageProfile=$StorageAccount' | \ | |
jq --arg StorageAccount "StorageAccount" '.properties.masterProfile.storageProfile=$StorageAccount' > $AZURESTACK_CONFIGURATION_TEMP | |
echo "Checking and moving 'use blobdisk and AvailabilitySet'" | |
check_and_move_azurestack_configuration $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION | |
fi | |
#if AzureAD assign SPN_CLIENT_SECRET to properties.servicePrincipalProfile.secret | |
sudo cat $AZURESTACK_CONFIGURATION | jq --arg THUMBPRINT $THUMBPRINT '.properties.cloudProfile.resourceManagerRootCertificate = $THUMBPRINT' | \ | |
jq --arg ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID $ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID '.properties.cloudProfile.serviceManagementEndpoint = $ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID'|\ | |
jq --arg ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT '.properties.cloudProfile.activeDirectoryEndpoint = $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT' | \ | |
jq --arg ENDPOINT_GRAPH_ENDPOINT $ENDPOINT_GRAPH_ENDPOINT '.properties.cloudProfile.graphEndpoint = $ENDPOINT_GRAPH_ENDPOINT' | \ | |
jq --arg TENANT_ENDPOINT $TENANT_ENDPOINT '.properties.cloudProfile.resourceManagerEndpoint = $TENANT_ENDPOINT' | \ | |
jq --arg ENDPOINT_GALLERY $ENDPOINT_GALLERY '.properties.cloudProfile.galleryEndpoint = $ENDPOINT_GALLERY' | \ | |
jq --arg SUFFIXES_STORAGE_ENDPOINT $SUFFIXES_STORAGE_ENDPOINT '.properties.cloudProfile.storageEndpointSuffix = $SUFFIXES_STORAGE_ENDPOINT' | \ | |
jq --arg SUFFIXES_KEYVAULT_DNS $SUFFIXES_KEYVAULT_DNS '.properties.cloudProfile.keyVaultDNSSuffix = $SUFFIXES_KEYVAULT_DNS' | \ | |
jq --arg FQDN_ENDPOINT_SUFFIX $FQDN_ENDPOINT_SUFFIX '.properties.cloudProfile.resourceManagerVMDNSSuffix = $FQDN_ENDPOINT_SUFFIX' | \ | |
jq --arg REGION_NAME $REGION_NAME '.properties.cloudProfile.location = $REGION_NAME' | \ | |
jq --arg MASTER_DNS_PREFIX $MASTER_DNS_PREFIX '.properties.masterProfile.dnsPrefix = $MASTER_DNS_PREFIX' | \ | |
jq '.properties.agentPoolProfiles[0].count'=$AGENT_COUNT | \ | |
jq --arg AGENT_SIZE $AGENT_SIZE '.properties.agentPoolProfiles[0].vmSize=$AGENT_SIZE' | \ | |
jq '.properties.masterProfile.count'=$MASTER_COUNT | \ | |
jq --arg MASTER_SIZE $MASTER_SIZE '.properties.masterProfile.vmSize=$MASTER_SIZE' | \ | |
jq --arg IDENTITY_SYSTEM $IDENTITY_SYSTEM '.properties.cloudProfile.identitySystem=$IDENTITY_SYSTEM' | \ | |
jq --arg ADMIN_USERNAME $ADMIN_USERNAME '.properties.linuxProfile.adminUsername = $ADMIN_USERNAME' | \ | |
jq --arg SSH_PUBLICKEY "${SSH_PUBLICKEY}" '.properties.linuxProfile.ssh.publicKeys[0].keyData = $SSH_PUBLICKEY' > $AZURESTACK_CONFIGURATION_TEMP | |
echo "Checking and moving " | |
check_and_move_azurestack_configuration $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION | |
if [ $IDENTITY_SYSTEM == "ADFS" ] | |
then | |
echo "Using ADFS" | |
sudo cat $AZURESTACK_CONFIGURATION | jq --arg SPN_CLIENT_ID $SPN_CLIENT_ID '.properties.servicePrincipalProfile.clientId = $SPN_CLIENT_ID' | \ | |
jq --arg SPN_CLIENT_SECRET_KEYVAULT_ID $SPN_CLIENT_SECRET_KEYVAULT_ID '.properties.servicePrincipalProfile.keyvaultSecretRef.VaultID = $SPN_CLIENT_SECRET_KEYVAULT_ID' | \ | |
jq --arg SPN_CLIENT_SECRET_KEYVAULT_SECRET_NAME $SPN_CLIENT_SECRET_KEYVAULT_SECRET_NAME '.properties.servicePrincipalProfile.keyvaultSecretRef.SecretName = $SPN_CLIENT_SECRET_KEYVAULT_SECRET_NAME' > $AZURESTACK_CONFIGURATION_TEMP | |
echo "Checking and moving " | |
check_and_move_azurestack_configuration $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION | |
else | |
echo "Using AAD" | |
sudo cat $AZURESTACK_CONFIGURATION | jq --arg SPN_CLIENT_ID $SPN_CLIENT_ID '.properties.servicePrincipalProfile.clientId = $SPN_CLIENT_ID' | \ | |
jq --arg SPN_CLIENT_SECRET $SPN_CLIENT_SECRET '.properties.servicePrincipalProfile.secret = $SPN_CLIENT_SECRET' > $AZURESTACK_CONFIGURATION_TEMP | |
echo "Checking and moving " | |
check_and_move_azurestack_configuration $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION | |
fi | |
echo "Done building the API model based on the stamp information." | |
MYDIR=$PWD | |
echo "Current directory is: $MYDIR" | |
echo "Deploy the template using the API model in resource group $MASTER_DNS_PREFIX." | |
#if AzureAD use below | |
#sudo ./bin/acs-engine deploy --resource-group $RESOURCE_GROUP_NAME --azure-env $ENVIRONMENT_NAME --location $REGION_NAME --subscription-id $TENANT_SUBSCRIPTION_ID --client-id $SPN_CLIENT_ID --client-secret $SPN_CLIENT_SECRET --auth-method client_secret --api-model $AZURESTACK_CONFIGURATION | |
#if adfs use | |
#sudo ./bin/acs-engine deploy --api-model ./1.11.json --location "local" --subscription-id e67ebc3e-1604-439d-b5d0-e70235c6ec69 --auth-method "client_certificate" --certificate-path "c:\domain.cer" --private-key-path "c:\domain.key" --azure-env "AzurestackCloud" --client-id "23e22dc8-ad94-4ad0-9b09-e329969f5500" | |
if [ $IDENTITY_SYSTEM == "ADFS" ] | |
then | |
echo "Using ADFS" | |
sudo ./bin/acs-engine deploy --resource-group $RESOURCE_GROUP_NAME --api-model $AZURESTACK_CONFIGURATION --location $REGION_NAME --subscription-id $TENANT_SUBSCRIPTION_ID --auth-method client_certificate --certificate-path $CERTIFICATE_LOCATION --private-key-path $KEY_LOCATION --azure-env $ENVIRONMENT_NAME --client-id $SPN_CLIENT_ID | |
else | |
echo "Using AAD" | |
sudo ./bin/acs-engine deploy --resource-group $RESOURCE_GROUP_NAME --azure-env $ENVIRONMENT_NAME --location $REGION_NAME --subscription-id $TENANT_SUBSCRIPTION_ID --client-id $SPN_CLIENT_ID --client-secret $SPN_CLIENT_SECRET --auth-method client_secret --api-model $AZURESTACK_CONFIGURATION | |
fi | |
echo "Templates output directory is $PWD/_output/$MASTER_DNS_PREFIX" | |
echo "Ending deploying Kubernetes cluster." |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment