Skip to content

Instantly share code, notes, and snippets.

@viglesiasce

viglesiasce/bootstrap_template.yaml

Last active July 30, 2019 06:12
Show Gist options
  • Save viglesiasce/46b3c30dbfdd392cf6e41c3472f656aa to your computer and use it in GitHub Desktop.
Save viglesiasce/46b3c30dbfdd392cf6e41c3472f656aa to your computer and use it in GitHub Desktop.
Download ZIP
Traffic Director Container-Optimized OS
Raw
bootstrap_template.yaml
node:
id: "ENVOY_NODE_ID"
cluster: cluster # unused
locality:
zone: "ENVOY_ZONE"
metadata:
TRAFFICDIRECTOR_INTERCEPTION_PORT: "ENVOY_PORT"
TRAFFICDIRECTOR_NETWORK_NAME: "VPC_NETWORK_NAME"
TRAFFICDIRECTOR_GCP_PROJECT_NUMBER: "CONFIG_PROJECT_NUMBER"
TRAFFICDIRECTOR_ENABLE_TRACING: "TRACING_ENABLED"
TRAFFICDIRECTOR_ACCESS_LOG_PATH: "ACCESSLOG_PATH"
dynamic_resources:
lds_config: {ads: {}}
cds_config: {ads: {}}
ads_config:
api_type: GRPC
grpc_services:
- google_grpc:
target_uri: trafficdirector.googleapis.com:443
stat_prefix: trafficdirector
channel_credentials:
ssl_credentials:
root_certs:
filename: XDS_SERVER_CERT
call_credentials:
google_compute_engine: {}
cluster_manager:
load_stats_config:
api_type: GRPC
grpc_services:
- google_grpc:
target_uri: trafficdirector.googleapis.com:443
stat_prefix: trafficdirector
channel_credentials:
ssl_credentials:
root_certs:
filename: XDS_SERVER_CERT
call_credentials:
google_compute_engine: {}
admin:
access_log_path: /dev/stdout
address:
socket_address:
address: 127.0.0.1 # Admin page is only accessible locally.
port_value: ENVOY_ADMIN_PORT
Raw
iptables.sh
#!/bin/bash
#
# Copyright 2017, 2018 Istio Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#
# Initialization script responsible for setting up port forwarding for Istio sidecar.
function usage() {
echo "${0} -p PORT -u UID -g GID [-m mode] [-b ports] [-d ports] [-i CIDR] [-x CIDR] [-k interfaces] [-h]"
echo ''
# shellcheck disable=SC2016
echo ' -p: Specify the envoy port to which redirect all TCP traffic (default $ENVOY_PORT = 15001)'
echo ' -u: Specify the UID of the user for which the redirection is not'
echo ' applied. Typically, this is the UID of the proxy container'
# shellcheck disable=SC2016
echo ' (default to uid of $ENVOY_USER, uid of istio_proxy, or 1337)'
echo ' -g: Specify the GID of the user for which the redirection is not'
echo ' applied. (same default value as -u param)'
echo ' -m: The mode used to redirect inbound connections to Envoy, either "REDIRECT" or "TPROXY"'
# shellcheck disable=SC2016
echo ' (default to $ISTIO_INBOUND_INTERCEPTION_MODE)'
echo ' -b: Comma separated list of inbound ports for which traffic is to be redirected to Envoy (optional). The'
echo ' wildcard character "*" can be used to configure redirection for all ports. An empty list will disable'
# shellcheck disable=SC2016
echo ' all inbound redirection (default to $ISTIO_INBOUND_PORTS)'
echo ' -d: Comma separated list of inbound ports to be excluded from redirection to Envoy (optional). Only applies'
# shellcheck disable=SC2016
echo ' when all inbound traffic (i.e. "*") is being redirected (default to $ISTIO_LOCAL_EXCLUDE_PORTS)'
echo ' -i: Comma separated list of IP ranges in CIDR form to redirect to envoy (optional). The wildcard'
echo ' character "*" can be used to redirect all outbound traffic. An empty list will disable all outbound'
# shellcheck disable=SC2016
echo ' redirection (default to $ISTIO_SERVICE_CIDR)'
echo ' -x: Comma separated list of IP ranges in CIDR form to be excluded from redirection. Only applies when all '
# shellcheck disable=SC2016
echo ' outbound traffic (i.e. "*") is being redirected (default to $ISTIO_SERVICE_EXCLUDE_CIDR).'
echo ' -k: Comma separated list of virtual interfaces whose inbound traffic (from VM)'
echo ' will be treated as outbound (optional)'
# shellcheck disable=SC2016
echo ''
# shellcheck disable=SC2016
echo 'Using environment variables in $ISTIO_SIDECAR_CONFIG (default: /var/lib/istio/envoy/sidecar.env)'
}
function dump {
iptables-save
ip6tables-save
}
trap dump EXIT
# Use a comma as the separator for multi-value arguments.
IFS=,
# The cluster env can be used for common cluster settings, pushed to all VMs in the cluster.
# This allows separating per-machine settings (the list of inbound ports, local path overrides) from cluster wide
# settings (CIDR range)
ISTIO_CLUSTER_CONFIG=${ISTIO_CLUSTER_CONFIG:-/var/lib/istio/envoy/cluster.env}
if [ -r "${ISTIO_CLUSTER_CONFIG}" ]; then
# shellcheck disable=SC1090
. "${ISTIO_CLUSTER_CONFIG}"
fi
ISTIO_SIDECAR_CONFIG=${ISTIO_SIDECAR_CONFIG:-/var/lib/istio/envoy/sidecar.env}
if [ -r "${ISTIO_SIDECAR_CONFIG}" ]; then
# shellcheck disable=SC1090
. "${ISTIO_SIDECAR_CONFIG}"
fi
# TODO: load all files from a directory, similar with ufw, to make it easier for automated install scripts
# Ideally we should generate ufw (and similar) configs as well, in case user already has an iptables solution.
PROXY_PORT=${ENVOY_PORT:-15001}
PROXY_UID=
PROXY_GID=
INBOUND_INTERCEPTION_MODE=${ISTIO_INBOUND_INTERCEPTION_MODE}
INBOUND_TPROXY_MARK=${ISTIO_INBOUND_TPROXY_MARK:-1337}
INBOUND_TPROXY_ROUTE_TABLE=${ISTIO_INBOUND_TPROXY_ROUTE_TABLE:-133}
INBOUND_PORTS_INCLUDE=${ISTIO_INBOUND_PORTS-}
INBOUND_PORTS_EXCLUDE=${ISTIO_LOCAL_EXCLUDE_PORTS-}
OUTBOUND_IP_RANGES_INCLUDE=${ISTIO_SERVICE_CIDR-}
OUTBOUND_IP_RANGES_EXCLUDE=${ISTIO_SERVICE_EXCLUDE_CIDR-}
KUBEVIRT_INTERFACES=
POD_IP=$(hostname --ip-address)
# Check if pod's ip is ipv4 or ipv6, in case of ipv6 set variable
# to program ip6tables
if [ "$POD_IP" != "${POD_IP#*:[0-9a-fA-F]}" ]; then
ENABLE_INBOUND_IPV6=$POD_IP
fi
-
while getopts ":p:u:g:m:b:d:i:x:k:h" opt; do
case ${opt} in
p)
PROXY_PORT=${OPTARG}
;;
u)
PROXY_UID=${OPTARG}
;;
g)
PROXY_GID=${OPTARG}
;;
m)
INBOUND_INTERCEPTION_MODE=${OPTARG}
;;
b)
INBOUND_PORTS_INCLUDE=${OPTARG}
;;
d)
INBOUND_PORTS_EXCLUDE=${OPTARG}
;;
i)
OUTBOUND_IP_RANGES_INCLUDE=${OPTARG}
;;
x)
OUTBOUND_IP_RANGES_EXCLUDE=${OPTARG}
;;
k)
KUBEVIRT_INTERFACES=${OPTARG}
;;
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG" >&2
usage
exit 1
;;
esac
done
# Traffic Director GCE VM deployment override: UID (and GID) is optional. If -u argument is not given, no default accounts will be exempted from redirection.
if [ -z "${TRAFFIC_DIRECTOR_GCE_VM_DEPLOYMENT_OVERRIDE-}" ]; then
# TODO: more flexibility - maybe a whitelist of users to be captured for output instead of a blacklist.
if [ -z "${PROXY_UID}" ]; then
# Default to the UID of ENVOY_USER and root
if ! PROXY_UID=$(id -u "${ENVOY_USER:-istio-proxy}"); then
PROXY_UID="1337"
fi
# If ENVOY_UID is not explicitly defined (as it would be in k8s env), we add root to the list,
# for ca agent.
PROXY_UID=${PROXY_UID},0
fi
# for TPROXY as its uid and gid are same
if [ -z "${PROXY_GID}" ]; then
PROXY_GID=${PROXY_UID}
fi
fi
# Remove the old chains, to generate new configs.
iptables -t nat -D PREROUTING -p tcp -j ISTIO_INBOUND 2>/dev/null
iptables -t mangle -D PREROUTING -p tcp -j ISTIO_INBOUND 2>/dev/null
iptables -t nat -D OUTPUT -p tcp -j ISTIO_OUTPUT 2>/dev/null
# Flush and delete the istio chains.
iptables -t nat -F ISTIO_OUTPUT 2>/dev/null
iptables -t nat -X ISTIO_OUTPUT 2>/dev/null
iptables -t nat -F ISTIO_INBOUND 2>/dev/null
iptables -t nat -X ISTIO_INBOUND 2>/dev/null
iptables -t mangle -F ISTIO_INBOUND 2>/dev/null
iptables -t mangle -X ISTIO_INBOUND 2>/dev/null
iptables -t mangle -F ISTIO_DIVERT 2>/dev/null
iptables -t mangle -X ISTIO_DIVERT 2>/dev/null
iptables -t mangle -F ISTIO_TPROXY 2>/dev/null
iptables -t mangle -X ISTIO_TPROXY 2>/dev/null
# Must be last, the others refer to it
iptables -t nat -F ISTIO_REDIRECT 2>/dev/null
iptables -t nat -X ISTIO_REDIRECT 2>/dev/null
iptables -t nat -F ISTIO_IN_REDIRECT 2>/dev/null
iptables -t nat -X ISTIO_IN_REDIRECT 2>/dev/null
if [ "${1:-}" = "clean" ]; then
echo "Only cleaning, no new rules added"
exit 0
fi
# Dump out our environment for debugging purposes.
echo "Environment:"
echo "------------"
echo "ENVOY_PORT=${ENVOY_PORT-}"
echo "ISTIO_INBOUND_INTERCEPTION_MODE=${ISTIO_INBOUND_INTERCEPTION_MODE-}"
echo "ISTIO_INBOUND_TPROXY_MARK=${ISTIO_INBOUND_TPROXY_MARK-}"
echo "ISTIO_INBOUND_TPROXY_ROUTE_TABLE=${ISTIO_INBOUND_TPROXY_ROUTE_TABLE-}"
echo "ISTIO_INBOUND_PORTS=${ISTIO_INBOUND_PORTS-}"
echo "ISTIO_LOCAL_EXCLUDE_PORTS=${ISTIO_LOCAL_EXCLUDE_PORTS-}"
echo "ISTIO_SERVICE_CIDR=${ISTIO_SERVICE_CIDR-}"
echo "ISTIO_SERVICE_EXCLUDE_CIDR=${ISTIO_SERVICE_EXCLUDE_CIDR-}"
echo
echo "Variables:"
echo "----------"
echo "PROXY_PORT=${PROXY_PORT}"
echo "INBOUND_CAPTURE_PORT=${INBOUND_CAPTURE_PORT:-$PROXY_PORT}"
echo "PROXY_UID=${PROXY_UID}"
echo "INBOUND_INTERCEPTION_MODE=${INBOUND_INTERCEPTION_MODE}"
echo "INBOUND_TPROXY_MARK=${INBOUND_TPROXY_MARK}"
echo "INBOUND_TPROXY_ROUTE_TABLE=${INBOUND_TPROXY_ROUTE_TABLE}"
echo "INBOUND_PORTS_INCLUDE=${INBOUND_PORTS_INCLUDE}"
echo "INBOUND_PORTS_EXCLUDE=${INBOUND_PORTS_EXCLUDE}"
echo "OUTBOUND_IP_RANGES_INCLUDE=${OUTBOUND_IP_RANGES_INCLUDE}"
echo "OUTBOUND_IP_RANGES_EXCLUDE=${OUTBOUND_IP_RANGES_EXCLUDE}"
echo "KUBEVIRT_INTERFACES=${KUBEVIRT_INTERFACES}"
echo "ENABLE_INBOUND_IPV6=${ENABLE_INBOUND_IPV6}"
echo
INBOUND_CAPTURE_PORT=${INBOUND_CAPTURE_PORT:-$PROXY_PORT}
set -o errexit
set -o nounset
set -o pipefail
set -x # echo on
# Create a new chain for redirecting outbound traffic to the common Envoy port.
# In both chains, '-j RETURN' bypasses Envoy and '-j ISTIO_REDIRECT'
# redirects to Envoy.
iptables -t nat -N ISTIO_REDIRECT
iptables -t nat -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_PORT}"
# Use this chain also for redirecting inbound traffic to the common Envoy port
# when not using TPROXY.
iptables -t nat -N ISTIO_IN_REDIRECT
iptables -t nat -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-port "${INBOUND_CAPTURE_PORT}"
# Handling of inbound ports. Traffic will be redirected to Envoy, which will process and forward
# to the local service. If not set, no inbound port will be intercepted by istio iptables.
if [ -n "${INBOUND_PORTS_INCLUDE}" ]; then
if [ "${INBOUND_INTERCEPTION_MODE}" = "TPROXY" ] ; then
# When using TPROXY, create a new chain for routing all inbound traffic to
# Envoy. Any packet entering this chain gets marked with the ${INBOUND_TPROXY_MARK} mark,
# so that they get routed to the loopback interface in order to get redirected to Envoy.
# In the ISTIO_INBOUND chain, '-j ISTIO_DIVERT' reroutes to the loopback
# interface.
# Mark all inbound packets.
iptables -t mangle -N ISTIO_DIVERT
iptables -t mangle -A ISTIO_DIVERT -j MARK --set-mark "${INBOUND_TPROXY_MARK}"
iptables -t mangle -A ISTIO_DIVERT -j ACCEPT
# Route all packets marked in chain ISTIO_DIVERT using routing table ${INBOUND_TPROXY_ROUTE_TABLE}.
ip -f inet rule add fwmark "${INBOUND_TPROXY_MARK}" lookup "${INBOUND_TPROXY_ROUTE_TABLE}"
# In routing table ${INBOUND_TPROXY_ROUTE_TABLE}, create a single default rule to route all traffic to
# the loopback interface.
ip -f inet route add local default dev lo table "${INBOUND_TPROXY_ROUTE_TABLE}" || ip route show table all
# Create a new chain for redirecting inbound traffic to the common Envoy
# port.
# In the ISTIO_INBOUND chain, '-j RETURN' bypasses Envoy and
# '-j ISTIO_TPROXY' redirects to Envoy.
iptables -t mangle -N ISTIO_TPROXY
iptables -t mangle -A ISTIO_TPROXY ! -d 127.0.0.1/32 -p tcp -j TPROXY --tproxy-mark "${INBOUND_TPROXY_MARK}/0xffffffff" --on-port "${PROXY_PORT}"
table=mangle
else
table=nat
fi
iptables -t ${table} -N ISTIO_INBOUND
iptables -t ${table} -A PREROUTING -p tcp -j ISTIO_INBOUND
if [ "${INBOUND_PORTS_INCLUDE}" == "*" ]; then
# Makes sure SSH is not redirected
iptables -t ${table} -A ISTIO_INBOUND -p tcp --dport 22 -j RETURN
# Apply any user-specified port exclusions.
if [ -n "${INBOUND_PORTS_EXCLUDE}" ]; then
for port in ${INBOUND_PORTS_EXCLUDE}; do
iptables -t ${table} -A ISTIO_INBOUND -p tcp --dport "${port}" -j RETURN
done
fi
# Redirect remaining inbound traffic to Envoy.
if [ "${INBOUND_INTERCEPTION_MODE}" = "TPROXY" ]; then
# If an inbound packet belongs to an established socket, route it to the
# loopback interface.
iptables -t mangle -A ISTIO_INBOUND -p tcp -m socket -j ISTIO_DIVERT || echo "No socket match support"
# Otherwise, it's a new connection. Redirect it using TPROXY.
iptables -t mangle -A ISTIO_INBOUND -p tcp -j ISTIO_TPROXY
else
iptables -t nat -A ISTIO_INBOUND -p tcp -j ISTIO_IN_REDIRECT
fi
else
# User has specified a non-empty list of ports to be redirected to Envoy.
for port in ${INBOUND_PORTS_INCLUDE}; do
if [ "${INBOUND_INTERCEPTION_MODE}" = "TPROXY" ]; then
iptables -t mangle -A ISTIO_INBOUND -p tcp --dport "${port}" -m socket -j ISTIO_DIVERT || echo "No socket match support"
iptables -t mangle -A ISTIO_INBOUND -p tcp --dport "${port}" -m socket -j ISTIO_DIVERT || echo "No socket match support"
iptables -t mangle -A ISTIO_INBOUND -p tcp --dport "${port}" -j ISTIO_TPROXY
else
iptables -t nat -A ISTIO_INBOUND -p tcp --dport "${port}" -j ISTIO_IN_REDIRECT
fi
done
fi
fi
# TODO: change the default behavior to not intercept any output - user may use http_proxy or another
# iptables wrapper (like ufw). Current default is similar with 0.1
# Create a new chain for selectively redirecting outbound packets to Envoy.
iptables -t nat -N ISTIO_OUTPUT
# Jump to the ISTIO_OUTPUT chain from OUTPUT chain for all tcp traffic.
iptables -t nat -A OUTPUT -p tcp -j ISTIO_OUTPUT
if [ -z "${DISABLE_REDIRECTION_ON_LOCAL_LOOPBACK-}" ]; then
# Redirect app calls to back itself via Envoy when using the service VIP or endpoint
# address, e.g. appN => Envoy (client) => Envoy (server) => appN.
iptables -t nat -A ISTIO_OUTPUT -o lo ! -d 127.0.0.1/32 -j ISTIO_REDIRECT
fi
for uid in ${PROXY_UID}; do
# Avoid infinite loops. Don't redirect Envoy traffic directly back to
# Envoy for non-loopback traffic.
iptables -t nat -A ISTIO_OUTPUT -m owner --uid-owner ${uid} -j RETURN
done
for gid in ${PROXY_GID}; do
# Avoid infinite loops. Don't redirect Envoy traffic directly back to
# Envoy for non-loopback traffic.
iptables -t nat -A ISTIO_OUTPUT -m owner --gid-owner ${gid} -j RETURN
done
# Skip redirection for Envoy-aware applications and
# container-to-container traffic both of which explicitly use
# localhost.
iptables -t nat -A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
# Apply outbound IP exclusions. Must be applied before inclusions.
if [ -n "${OUTBOUND_IP_RANGES_EXCLUDE}" ]; then
for cidr in ${OUTBOUND_IP_RANGES_EXCLUDE}; do
iptables -t nat -A ISTIO_OUTPUT -d "${cidr}" -j RETURN
done
fi
for internalInterface in ${KUBEVIRT_INTERFACES}; do
iptables -t nat -I PREROUTING 1 -i "${internalInterface}" -j RETURN
done
# Apply outbound IP inclusions.
if [ "${OUTBOUND_IP_RANGES_INCLUDE}" == "*" ]; then
# Wildcard specified. Redirect all remaining outbound traffic to Envoy.
iptables -t nat -A ISTIO_OUTPUT -j ISTIO_REDIRECT
for internalInterface in ${KUBEVIRT_INTERFACES}; do
iptables -t nat -I PREROUTING 1 -i "${internalInterface}" -j ISTIO_REDIRECT
done
elif [ -n "${OUTBOUND_IP_RANGES_INCLUDE}" ]; then
# User has specified a non-empty list of cidrs to be redirected to Envoy.
for cidr in ${OUTBOUND_IP_RANGES_INCLUDE}; do
for internalInterface in ${KUBEVIRT_INTERFACES}; do
iptables -t nat -I PREROUTING 1 -i "${internalInterface}" -d "${cidr}" -j ISTIO_REDIRECT
done
iptables -t nat -A ISTIO_OUTPUT -d "${cidr}" -j ISTIO_REDIRECT
done
# All other traffic is not redirected.
iptables -t nat -A ISTIO_OUTPUT -j RETURN
fi
# If ENABLE_INBOUND_IPV6 is unset (default unset), restrict IPv6 traffic.
set +o nounset
if [ -n "${ENABLE_INBOUND_IPV6}" ]; then
# TODO: support receiving IPv6 traffic in the same way as IPv4.
# Allow all ipv6 traffic inbound and outboud, whitebox mode for now
# Remove the old chains, to generate new configs.
ip6tables -t nat -D PREROUTING -p tcp -j ISTIO_INBOUND 2>/dev/null || true
ip6tables -t mangle -D PREROUTING -p tcp -j ISTIO_INBOUND 2>/dev/null || true
ip6tables -t nat -D OUTPUT -p tcp -j ISTIO_OUTPUT 2>/dev/null || true
# Flush and delete the istio chains.
ip6tables -t nat -F ISTIO_OUTPUT 2>/dev/null || true
ip6tables -t nat -X ISTIO_OUTPUT 2>/dev/null || true
ip6tables -t nat -F ISTIO_INBOUND 2>/dev/null || true
ip6tables -t nat -X ISTIO_INBOUND 2>/dev/null || true
ip6tables -t mangle -F ISTIO_INBOUND 2>/dev/null || true
ip6tables -t mangle -X ISTIO_INBOUND 2>/dev/null || true
ip6tables -t mangle -F ISTIO_DIVERT 2>/dev/null || true
ip6tables -t mangle -X ISTIO_DIVERT 2>/dev/null || true
ip6tables -t mangle -F ISTIO_TPROXY 2>/dev/null || true
ip6tables -t mangle -X ISTIO_TPROXY 2>/dev/null || true
# Must be last, the others refer to it
ip6tables -t nat -F ISTIO_REDIRECT 2>/dev/null || true
ip6tables -t nat -X ISTIO_REDIRECT 2>/dev/null|| true
ip6tables -t nat -F ISTIO_IN_REDIRECT 2>/dev/null || true
ip6tables -t nat -X ISTIO_IN_REDIRECT 2>/dev/null || true
# Create a new chain for redirecting outbound traffic to the common Envoy port.
# In both chains, '-j RETURN' bypasses Envoy and '-j ISTIO_REDIRECT'
# redirects to Envoy.
ip6tables -t nat -N ISTIO_REDIRECT
ip6tables -t nat -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-port "${PROXY_PORT}"
# Use this chain also for redirecting inbound traffic to the common Envoy port
# when not using TPROXY.
ip6tables -t nat -N ISTIO_IN_REDIRECT
ip6tables -t nat -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-port "${INBOUND_CAPTURE_PORT}"
# Handling of inbound ports. Traffic will be redirected to Envoy, which will process and forward
# to the local service. If not set, no inbound port will be intercepted by istio iptables.
if [ -n "${INBOUND_PORTS_INCLUDE}" ]; then
table=nat
ip6tables -t ${table} -N ISTIO_INBOUND
ip6tables -t ${table} -A PREROUTING -p tcp -j ISTIO_INBOUND
if [ "${INBOUND_PORTS_INCLUDE}" == "*" ]; then
# Makes sure SSH is not redirected
ip6tables -t ${table} -A ISTIO_INBOUND -p tcp --dport 22 -j RETURN
# Apply any user-specified port exclusions.
if [ -n "${INBOUND_PORTS_EXCLUDE}" ]; then
for port in ${INBOUND_PORTS_EXCLUDE}; do
ip6tables -t ${table} -A ISTIO_INBOUND -p tcp --dport "${port}" -j RETURN
done
fi
else
# User has specified a non-empty list of ports to be redirected to Envoy.
for port in ${INBOUND_PORTS_INCLUDE}; do
ip6tables -t nat -A ISTIO_INBOUND -p tcp --dport "${port}" -j ISTIO_IN_REDIRECT
done
fi
fi
# Create a new chain for selectively redirecting outbound packets to Envoy.
ip6tables -t nat -N ISTIO_OUTPUT
# Jump to the ISTIO_OUTPUT chain from OUTPUT chain for all tcp traffic.
ip6tables -t nat -A OUTPUT -p tcp -j ISTIO_OUTPUT
# Redirect app calls to back itself via Envoy when using the service VIP or endpoint
# address, e.g. appN => Envoy (client) => Envoy (server) => appN.
ip6tables -t nat -A ISTIO_OUTPUT -o lo ! -d ::1/128 -j ISTIO_REDIRECT
for uid in ${PROXY_UID}; do
# Avoid infinite loops. Don't redirect Envoy traffic directly back to
# Envoy for non-loopback traffic.
ip6tables -t nat -A ISTIO_OUTPUT -m owner --uid-owner ${uid} -j RETURN
done
for gid in ${PROXY_GID}; do
# Avoid infinite loops. Don't redirect Envoy traffic directly back to
# Envoy for non-loopback traffic.
ip6tables -t nat -A ISTIO_OUTPUT -m owner --gid-owner ${gid} -j RETURN
done
# Skip redirection for Envoy-aware applications and
# container-to-container traffic both of which explicitly use
# localhost.
ip6tables -t nat -A ISTIO_OUTPUT -d ::1/128 -j RETURN
# Apply outbound IPv6 inclusions.
# TODO Need to figure out differentiation between IPv4 and IPv6 ranges
# for now process only "*"
if [ "${OUTBOUND_IP_RANGES_INCLUDE}" == "*" ]; then
# Wildcard specified. Redirect all remaining outbound traffic to Envoy.
ip6tables -t nat -A ISTIO_OUTPUT -j ISTIO_REDIRECT
fi
for internalInterface in ${KUBEVIRT_INTERFACES}; do
ip6tables -t nat -I PREROUTING 1 -i "${internalInterface}" -j RETURN
done
else
# Drop all inbound traffic except established connections.
ip6tables -F INPUT || true
ip6tables -A INPUT -m state --state ESTABLISHED -j ACCEPT || true
ip6tables -A INPUT -i lo -d ::1 -j ACCEPT || true
ip6tables -A INPUT -j REJECT || true
fi
Raw
run.sh
#!/bin/bash
# Starts Envoy and configures traffic interception on GCE VM to access
# GCP Traffic Director.
#
# Usage steps:
# - Configure environment in "sidecar.env".
# - Prepare Envoy binary, put it under the same directory as this script.
# - Run this script as root: sudo /bin/bash -x </path/to/this/script>
#
# Notes:
# - The Envoy process will be started in the background. Only one Envoy
# process can exist at any given time; before restarting, please kill the
# existing Envoy process.
# - If Envoy is restarted by a different user, you might need to delete the
# file "/dev/shm/envoy_shared_memory_0". For more info, please refer:
# https://github.com/envoyproxy/envoy/issues/2106
# - This script must be run as root, as required by iptables and starting
# Envoy process using different UID.
set -o errexit
set -o nounset
set -o pipefail
if [[ "${EUID}" -ne 0 ]]; then
echo "$0 must be run with root privileges. Try sudo $0" >&2
exit 1
fi
# Location of compulsory files.
readonly PACKAGE_DIRECTORY=$(pwd)
readonly CONFIG_FILE="${PACKAGE_DIRECTORY}/sidecar.env"
readonly IPTABLES_SCRIPT="${PACKAGE_DIRECTORY}/iptables.sh"
readonly BOOTSTRAP_TEMPLATE="${PACKAGE_DIRECTORY}/bootstrap_template.yaml"
readonly BOOTSTRAP_INSTANCE="${PACKAGE_DIRECTORY}/bootstrap_instance.yaml"
readonly ENVOY_BINARY="${PACKAGE_DIRECTORY}/envoy"
function initialize() {
# Read configuration.
source "${CONFIG_FILE}"
if [[ "${EXCLUDE_ENVOY_USER_FROM_INTERCEPT}" == "true" ]] && [[ -z "${ENVOY_USER}" ]]; then
echo "ENVOY_USER must be set in $CONFIG_FILE." >&2
exit 1
fi
if [[ ! `docker pull getenvoy/envoy:stable` ]]; then
echo "Cannot find Envoy image: ${ENVOY_BINARY}." >&2
exit 1
fi
if [[ -z "${VPC_NETWORK_NAME}" ]]; then
echo -n "WARNING: VPC_NETWORK_NAME is not set in the $CONFIG_FILE. " >&2
echo "Will attempt to auto-derive, but it is recommended to set it explicitly." >&2
fi
if [[ -z "${GCP_PROJECT_NUMBER}" ]]; then
echo -n "WARNING: GCP_PROJECT_NUMBER is not set in the $CONFIG_FILE. " >&2
echo "Will attempt to auto-derive, but it is recommended to set it explicitly." >&2
fi
chmod +x "${IPTABLES_SCRIPT}"
}
function enable_interception() {
# Run iptables.sh.
echo "Enabling traffic interception for the '${SERVICE_CIDR}' destination."
TRAFFIC_DIRECTOR_GCE_VM_DEPLOYMENT_OVERRIDE='true'
DISABLE_REDIRECTION_ON_LOCAL_LOOPBACK='true'
export TRAFFIC_DIRECTOR_GCE_VM_DEPLOYMENT_OVERRIDE
export DISABLE_REDIRECTION_ON_LOCAL_LOOPBACK
if [[ -z "${EXCLUDE_ENVOY_USER_FROM_INTERCEPT}" ]]; then
"${IPTABLES_SCRIPT}" \
-i "${SERVICE_CIDR}" \
-p "${ENVOY_PORT}" > /dev/null 2>&1
else # Exclude ENVOY_USER by -u option.
envoy_user_id=$(id -u "${ENVOY_USER}")
"${IPTABLES_SCRIPT}" \
-i "${SERVICE_CIDR}" \
-p "${ENVOY_PORT}" \
-u "${envoy_user_id}" > /dev/null 2>&1
fi
}
function disable_interception() {
echo "Disabling traffic interception for the '${SERVICE_CIDR}' destination."
#bash -xe "${IPTABLES_SCRIPT}" clean
}
function get_zone() {
# Query the GCP instance metadata server to find the zone of the current VM.
# Delete any iptables rules created by previous executions.
disable_interception > /dev/null
envoy_zone=''
query_url='http://metadata.google.internal/computeMetadata/v1/instance/zone'
query_header='Metadata-Flavor: Google'
query_result=$(curl "${query_url}" -H "${query_header}" -sS || true)
if [[ -z "${query_result}" ]]; then
echo 'Failed to get zone from GCP metadata server.' >&2
else
# The response is in the format: "projects/[PROJECT_NUMBER]/zones/[ZONE]"
envoy_zone=$(echo "${query_result}" | cut -d'/' -f4 || true)
if [[ -z "${envoy_zone}" ]]; then
echo 'Failed to parse the result from GCP metadata server.' >&2
fi
fi
}
function prepare_envoy() {
# Get Envoy GCP zone.
get_zone
# Generate Envoy bootstrap.
envoy_node_id="$(uuidgen)~$(hostname -i)"
# Set optional variables if they are not defined previously.
if [[ -z "${TRACING_ENABLED+x}" ]]; then
TRACING_ENABLED="false"
fi
if [[ -z "${ACCESSLOG_PATH+x}" ]]; then
ACCESSLOG_PATH=""
fi
if [[ -z "${ENVOY_ADMIN_PORT+x}" ]]; then
ENVOY_ADMIN_PORT="15000"
fi
cat "${BOOTSTRAP_TEMPLATE}" \
| sed -e "s|ENVOY_NODE_ID|${envoy_node_id}|g" \
| sed -e "s|ENVOY_ZONE|${envoy_zone}|g" \
| sed -e "s|VPC_NETWORK_NAME|${VPC_NETWORK_NAME}|g" \
| sed -e "s|CONFIG_PROJECT_NUMBER|${GCP_PROJECT_NUMBER}|g" \
| sed -e "s|ENVOY_PORT|${ENVOY_PORT}|g" \
| sed -e "s|ENVOY_ADMIN_PORT|${ENVOY_ADMIN_PORT}|g" \
| sed -e "s|XDS_SERVER_CERT|${XDS_SERVER_CERT}|g" \
| sed -e "s|TRACING_ENABLED|${TRACING_ENABLED}|g" \
| sed -e "s|ACCESSLOG_PATH|${ACCESSLOG_PATH}|g" \
> "${BOOTSTRAP_INSTANCE}"
# Grant permissions to ${ENVOY_USER}.
mkdir -p "${LOG_DIR}"
chown "${ENVOY_USER}":"${ENVOY_USER}" "${LOG_DIR}"
#chown "${ENVOY_USER}": "${ENVOY_BINARY}"
chown "${ENVOY_USER}":"${ENVOY_USER}" "${BOOTSTRAP_INSTANCE}"
}
function start_envoy() {
echo "Starting Envoy process."
docker run -v ${BOOTSTRAP_INSTANCE}:/etc/envoy/config.yaml \
-v ${LOG_DIR}:${LOG_DIR} \
-d \
--network host \
--user 2000:2000 \
--name envoy \
--rm \
getenvoy/envoy:stable \
--config-path /etc/envoy/config.yaml \
--log-level ${LOG_LEVEL} \
--allow-unknown-fields
}
function stop_envoy() {
echo "Stopping Envoy process."
docker stop envoy
}
function check_status() {
ENVOY_IS_RUNNING=0
if [[ $(pgrep -c envoy) -gt 0 ]]; then
echo "OK: Envoy seems to be running."
ENVOY_IS_RUNNING=1
else
echo "NOT ENABLED: Envoy does not seem to be running."
fi
if [[ $(iptables -t nat -S | grep -c ISTIO) -gt 0 ]]; then
echo "OK: Traffic interception seems to be enabled."
else
echo "NOT ENABLED: Traffic interception does not seem to be enabled."
fi
}
initialize
case ${1-usage} in
'start')
check_status > /dev/null
if [[ "$ENVOY_IS_RUNNING" -eq 1 ]]; then
echo "Envoy is already running. Will not start another one." >&2
echo "You can stop it by running 'sudo $0 stop'." >&2
else
prepare_envoy
start_envoy
fi
enable_interception
check_status
;;
'stop')
disable_interception
stop_envoy
;;
'status')
check_status
;;
*)
echo "Usage: $0 <start|stop|status>" >&2
;;
esac
Raw
sidecar.env
# Environment variables used to configure Envoy startup.
ENVOY_USER='envoy'
VPC_NETWORK_NAME=''
GCP_PROJECT_NUMBER=''
EXCLUDE_ENVOY_USER_FROM_INTERCEPT='true'
SERVICE_CIDR='10.128.0.10/32'
ENVOY_PORT='15001'
# Envoy proxy port to listen on for the admin interface.
ENVOY_ADMIN_PORT='15000'
# Location for Envoy output.
LOG_DIR='/var/log/envoy/'
# Envoy log level. Must be one of:
# [trace][debug][info][warning][error][critical][off]
LOG_LEVEL='info'
# Location of public server certificate for GCP Traffic Director
# (trafficdirector.googleapis.com) over HTTPS.
XDS_SERVER_CERT='/etc/ssl/certs/ca-certificates.crt'
# If set to "true", enables generation of tracing for inbound and outbound
# requests.
TRACING_ENABLED='false'
# If not empty, defines a path on a file-system local to the sidecar proxy,
# where access logs for all inbound and outbound requests will be written.
ACCESSLOG_PATH='/var/log/envoy/'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment