ashb · May 10, 2017 08:50
diff --git a/hacky.sh b/hacky.sh
 ensure_kube_apiserver_config() {
  local dry_run="$2"
  # If we want to make changes to the apiserver manifest we do it via an
  # Azure CustomLinuxCommand extension
  # https://github.com/Azure/custom-script-extension-linux which will run the
  # command for us without having to log in. The down side to this is that it
  # takes a while to "deploy" the extension, which is espeically a pain if
  # the config is already correct.
  #
  # To speed it up and only deploy the extension if something is changed we
  # look at the spec of the API server using `kubectl`.

  local client_id
  client_id="$(jq -r -e '.parameters.authOIDCClientId.value' "$customization_params_file")" || {
    info "Skipping APIServer config: no authOIDCClientId.value found in customization file"
    return
  }

  desired_args=(
    "--oidc-client-id" "$client_id"
    "--oidc-issuer-url" "https://login.microsoftonline.com/<creds>/v2.0"
    "--oidc-username-claim" "preferred_username"
    "--oidc-groups-claim" "groups"
  )

  info "Checking kube-apiserver config..."
  local current_config master_vms

  list_deployment_vms --master-only --names <<<"$1"
  master_vms=$(list_deployment_vms --master-only --names <<<"$1") || {
    err "Error getting master VMs."
    exit 1
  }
  for vm_id in $master_vms
  do
    # There might not be a kube-apiserver running (yet) on that node, so we
    # can't pipe the output directly into filter_spec_command_flags.
    current_config="$(kubectl --namespace=kube-system get "pods/kube-apiserver-$(tr '[:upper:]' '[:lower:]' <<<"$vm_id")" -o json)" || true
    local status=${PIPESTATUS[0]}
    if [[ $status != 0 ]] || <<<"$current_config" filter_spec_command_flags --are-changes-needed -- "${desired_args[@]}"; then
      [[ -n "$dry_run" ]] && {
        info "API Server config on $vm_id needs updating (dry run)"
        continue
      }

      info "Updating API Server config on $vm_id"
      az vm extension set \
        -g "$rg_name" \
        --vm-name "$vm_id" \
        --publisher Microsoft.Azure.Extensions \
        -n CustomScript \
        --protected-settings "$(extension_json_params "${desired_args[@]}")"
    fi
  done
 }

 extension_json_params() {
  # Build the json parameters we feed into the custom extension. We could
  # upload a file to an azure storage blob to make this nicer, but we haven't
  # done that anywhere else so for now it's a big-massive-log bash -c ''
  # command.

  # NOTE: This uses wildducktheories/y2j. If any environment blocks pulling
  # images from the docker hub this might fail!

  {
    echo '#!/bin/bash
 set -ex -o pipefail
 '
    # Now include the function body, and the flags we want
    # `type` will tell us what it is, and print the definition. We want just the definition
    type filter_spec_command_flags | tail -n +2
    echo "desired_args=("
    printf " %q" "$@"
    echo ")"

    # Then the rest of our script
    # shellcheck disable=SC1004
    echo '
 apiserver_manifest="$(</etc/kubernetes/manifests/kube-apiserver.yaml docker run --rm -i wildducktheories/y2j y2j)"

 if filter_spec_command_flags --are-changes-needed -- "${desired_args[@]}" <<<"${apiserver_manifest}"; then
    echo "Changes needed in kube-apiserver.yaml"
    tmpfile=$(mktemp)
    filter_spec_command_flags -- "${desired_args[@]}" <<<"${apiserver_manifest}" \
      | docker run --rm -i wildducktheories/y2j j2y > $tmpfile
    mv $tmpfile /etc/kubernetes/manifests/kube-apiserver.yaml
    echo "Restarting kubelet"
    service kubelet restart
 else
    echo "No change needed in kube-apiserver.yaml"
 fi
 '
  } | jq --slurp -R '{
    commandToExecute: "/bin/bash -c \(. | @sh)"
  }'
 }

 # Expects as input on stdin a pod spec IN JSON (not the default Yaml) and takes
 # arguments of the list of options we want to see in the command line. Returns
 # the new pod spec also in JSON.
 filter_spec_command_flags() {
  # Check for cmd flags first

  # Save stdout so we can re-open it later
  exec 3>&1

  local final_filter="" filters="" jq_args=()
  while [[ "$#" -gt 0 ]]; do
    case "${1:-}" in
    --are-changes-needed)
      # Exit with 0 exit code if the command needs to be updated
      filters='. as $orig | '
      final_filter=' | . != $orig'
      jq_args=(-e)
      # We won't want any output from this function
      exec >/dev/null
      ;;
    --)
      shift
      break
      ;;
    *)
      >&2 echo "Unknown mode to filter_spec_command_flags: \"$1\""
      exit 2
      ;;
    esac
    shift
  done

  [[ $(($# % 2)) -eq 0 ]] || {
      echo >&2 "filter_spec_command_flags needs an even number of remaining arguments!"
      return 1
  }
  while [[ $# -gt 0 ]]; do
    filters+=".spec.containers[].command |= update_or_append(\"$1\"; \"$2\") | "
    shift
    shift
  done

  # Trim off the " | " from the end using bash parameter substitution.
  filters="${filters% | }"

  jq "${jq_args[@]}" '
    def update_or_append($flag;$value):
        # Do we already have a setting of this value, or do we need to add a new one
        if any(startswith("\($flag)=")) then
            map(
                if startswith($flag) then
                    "\($flag)=\($value)"
                else
                  .
                end)
        else
            . + ["\($flag)=\($value)"]
        end;

    '"$filters$final_filter"
  local status=$?
  exec 1>&3 3>&-
  return $status
 }
	ensure_kube_apiserver_config() {
	local dry_run="$2"
	# If we want to make changes to the apiserver manifest we do it via an
	# Azure CustomLinuxCommand extension
	# https://github.com/Azure/custom-script-extension-linux which will run the
	# command for us without having to log in. The down side to this is that it
	# takes a while to "deploy" the extension, which is espeically a pain if
	# the config is already correct.
	#
	# To speed it up and only deploy the extension if something is changed we
	# look at the spec of the API server using `kubectl`.

	local client_id
	client_id="$(jq -r -e '.parameters.authOIDCClientId.value' "$customization_params_file")" \|\| {
	info "Skipping APIServer config: no authOIDCClientId.value found in customization file"
	return
	}

	desired_args=(
	"--oidc-client-id" "$client_id"
	"--oidc-issuer-url" "https://login.microsoftonline.com/<creds>/v2.0"
	"--oidc-username-claim" "preferred_username"
	"--oidc-groups-claim" "groups"
	)

	info "Checking kube-apiserver config..."
	local current_config master_vms

	list_deployment_vms --master-only --names <<<"$1"
	master_vms=$(list_deployment_vms --master-only --names <<<"$1") \|\| {
	err "Error getting master VMs."
	exit 1
	}
	for vm_id in $master_vms
	do
	# There might not be a kube-apiserver running (yet) on that node, so we
	# can't pipe the output directly into filter_spec_command_flags.
	current_config="$(kubectl --namespace=kube-system get "pods/kube-apiserver-$(tr '[:upper:]' '[:lower:]' <<<"$vm_id")" -o json)" \|\| true
	local status=${PIPESTATUS[0]}
	if [[ $status != 0 ]] \|\| <<<"$current_config" filter_spec_command_flags --are-changes-needed -- "${desired_args[@]}"; then
	[[ -n "$dry_run" ]] && {
	info "API Server config on $vm_id needs updating (dry run)"
	continue
	}

	info "Updating API Server config on $vm_id"
	az vm extension set \
	-g "$rg_name" \
	--vm-name "$vm_id" \
	--publisher Microsoft.Azure.Extensions \
	-n CustomScript \
	--protected-settings "$(extension_json_params "${desired_args[@]}")"
	fi
	done
	}

	extension_json_params() {
	# Build the json parameters we feed into the custom extension. We could
	# upload a file to an azure storage blob to make this nicer, but we haven't
	# done that anywhere else so for now it's a big-massive-log bash -c ''
	# command.

	# NOTE: This uses wildducktheories/y2j. If any environment blocks pulling
	# images from the docker hub this might fail!

	{
	echo '#!/bin/bash
	set -ex -o pipefail
	'
	# Now include the function body, and the flags we want
	# `type` will tell us what it is, and print the definition. We want just the definition
	type filter_spec_command_flags \| tail -n +2
	echo "desired_args=("
	printf " %q" "$@"
	echo ")"

	# Then the rest of our script
	# shellcheck disable=SC1004
	echo '
	apiserver_manifest="$(</etc/kubernetes/manifests/kube-apiserver.yaml docker run --rm -i wildducktheories/y2j y2j)"

	if filter_spec_command_flags --are-changes-needed -- "${desired_args[@]}" <<<"${apiserver_manifest}"; then
	echo "Changes needed in kube-apiserver.yaml"
	tmpfile=$(mktemp)
	filter_spec_command_flags -- "${desired_args[@]}" <<<"${apiserver_manifest}" \
	\| docker run --rm -i wildducktheories/y2j j2y > $tmpfile
	mv $tmpfile /etc/kubernetes/manifests/kube-apiserver.yaml
	echo "Restarting kubelet"
	service kubelet restart
	else
	echo "No change needed in kube-apiserver.yaml"
	fi
	'
	} \| jq --slurp -R '{
	commandToExecute: "/bin/bash -c \(. \| @sh)"
	}'
	}

	# Expects as input on stdin a pod spec IN JSON (not the default Yaml) and takes
	# arguments of the list of options we want to see in the command line. Returns
	# the new pod spec also in JSON.
	filter_spec_command_flags() {
	# Check for cmd flags first

	# Save stdout so we can re-open it later
	exec 3>&1

	local final_filter="" filters="" jq_args=()
	while [[ "$#" -gt 0 ]]; do
	case "${1:-}" in
	--are-changes-needed)
	# Exit with 0 exit code if the command needs to be updated
	filters='. as $orig \| '
	final_filter=' \| . != $orig'
	jq_args=(-e)
	# We won't want any output from this function
	exec >/dev/null
	;;
	--)
	shift
	break
	;;
	*)
	>&2 echo "Unknown mode to filter_spec_command_flags: \"$1\""
	exit 2
	;;
	esac
	shift
	done

	[[ $(($# % 2)) -eq 0 ]] \|\| {
	echo >&2 "filter_spec_command_flags needs an even number of remaining arguments!"
	return 1
	}
	while [[ $# -gt 0 ]]; do
	filters+=".spec.containers[].command \|= update_or_append(\"$1\"; \"$2\") \| "
	shift
	shift
	done

	# Trim off the " \| " from the end using bash parameter substitution.
	filters="${filters% \| }"

	jq "${jq_args[@]}" '
	def update_or_append($flag;$value):
	# Do we already have a setting of this value, or do we need to add a new one
	if any(startswith("\($flag)=")) then
	map(
	if startswith($flag) then
	"\($flag)=\($value)"
	else
	.
	end)
	else
	. + ["\($flag)=\($value)"]
	end;

	'"$filters$final_filter"
	local status=$?
	exec 1>&3 3>&-
	return $status
	}