Skip to content

Instantly share code, notes, and snippets.

@WoozyMasta
Last active November 24, 2022 13:02
Show Gist options
  • Save WoozyMasta/9aee21a0a233fef844a8925e2d938781 to your computer and use it in GitHub Desktop.
Save WoozyMasta/9aee21a0a233fef844a8925e2d938781 to your computer and use it in GitHub Desktop.
Create and update multiple container registry cache proxies in CRI-O with Daemonset without direct access to k8s node
apiVersion: v1
kind: ConfigMap
metadata:
name: registry-config
namespace: kube-system
data:
unqualified-search-registries.conf: |
unqualified-search-registries = [
"docker.io",
"quay.io",
"gcr.io",
"k8s.gcr.io",
"ghcr.io",
"mcr.microsoft.com",
"registry.gitlab.com",
]
docker.io.conf: |
[[registry]]
prefix = "docker.io"
location = "docker.io"
[[registry.mirror]]
prefix = "docker.io"
location = "cache.tld"
quay.io.conf: |
[[registry]]
prefix = "quay.io"
location = "quay.io"
[[registry.mirror]]
prefix = "quay.io"
location = "cache.tld"
gcr.io.conf: |
[[registry]]
prefix = "gcr.io"
location = "gcr.io"
[[registry.mirror]]
prefix = "gcr.io"
location = "cache.tld"
k8s.gcr.conf: |
[[registry]]
prefix = "k8s.gcr.io"
location = "k8s.gcr.io"
[[registry.mirror]]
prefix = "k8s.gcr.io"
location = "cache.tld"
ghcr.io.conf: |
[[registry]]
prefix = "ghcr.io"
location = "ghcr.io"
[[registry.mirror]]
prefix = "ghcr.io"
location = "cache.tld"
mcr.microsoft.com.conf: |
[[registry]]
prefix = "mcr.microsoft.com"
location = "mcr.microsoft.com"
[[registry.mirror]]
prefix = "mcr.microsoft.com"
location = "cache.tld"
registry.gitlab.com.conf: |
[[registry]]
prefix = "registry.gitlab.com"
location = "registry.gitlab.com"
[[registry.mirror]]
prefix = "registry.gitlab.com"
location = "cache.tld"
entrypoint.sh: |
#!/bin/bash
set -eu
: "${WORK_DIR:=${1:-/configmaps}}"
: "${CONFIG_DIR:=${2:-/registries.conf.d}}"
: "${RETRY_TIMEOUTL_SEC:=300}"
: "${CRIO_BIN:=/usr/bin/crio}"
fail() { >&2 printf '[%s]\tERROR:\t%s\n' "$(date -uIs)" "$@"; exit 1; }
updateConfig() {
local src_cfg dst_f src_md5 dst_md5 j=0 i=0
for src_cfg in "$WORK_DIR"/*.conf; do
[ ! -f "$src_cfg" ] && continue
i=$((i+1)); dst_f="$CONFIG_DIR/${src_cfg##*/}"
if [ ! -f "$dst_f" ]; then
cp "$src_cfg" "$dst_f" && j=$((j+1))
>&2 printf '[%s]\tInstalled:\t%s\n' "$(date -uIs)" "${src_cfg##*/}"
else
src_md5=($(md5sum "$src_cfg"));
dst_md5=($(md5sum "$dst_f"))
[ "${src_md5[0]}" = "${dst_md5[0]}" ] && continue
cp "$src_cfg" "$dst_f" && j=$((j+1))
>&2 printf '[%s]\tUpdated:\t%s\n' "$(date -uIs)" "${src_cfg##*/}"
fi
done
>&2 printf '[%s]\tDone:\t[%d/%d] changes\n' "$(date -uIs)" "$j" "$i"
return "$j"
}
updateHosts() {
while read -r ip host; do
[ -z "${host:-}" ] && continue
grep -qE "$ip\s+$host" /mount/hosts && continue
printf '# populated from script\n%s\t%s\n' "$ip" "$host" >> /mount/hosts
printf '[%s]\tHosts:%s\n' "$(date -uIs)" "Added $ip --> $host"
done < <(grep -v '^$\|^#' "$WORK_DIR/hosts")
}
[ ! -d "$WORK_DIR" ] && fail "Missed $WORK_DIR (\$WORK_DIR or \$1)"
[ ! -d "$CONFIG_DIR" ] && fail "Missed $CONFIG_DIR (\$CONFIG_DIR or \$2)"
while :; do
[ -f "$WORK_DIR/hosts" ] && updateHosts
if ! updateConfig; then
if kill -s SIGHUP "$(pgrep $CRIO_BIN)"; then
printf '[%s]\tReload:%s\n' "$(date -uIs)" "$CRIO_BIN update config"
else
fail "$CRIO_BIN not a valid bin for get pid (\$CRIO_BIN)"
fi
fi
sleep ${RETRY_TIMEOUTL_SEC}s
done
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: registry-config
namespace: kube-system
spec:
selector:
matchLabels:
name: registry-config
template:
metadata:
labels:
name: registry-config
spec:
hostPID: true
tolerations:
- key: runner
operator: Equal
value: "true"
effect: NoSchedule
- key: build
operator: Equal
value: "true"
effect: NoSchedule
- key: build
operator: Equal
value: large
effect: NoSchedule
- key: build
operator: Equal
value: medium
effect: NoSchedule
- key: build
operator: Equal
value: small
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
containers:
- name: registry-config
image: cache.tld/bash:latest
securityContext:
privileged: true
runAsUser: 0
command:
- bash
- /configmaps/entrypoint.sh
- /configmaps
- /registries.conf.d
resources:
limits:
cpu: 50m
memory: 50Mi
requests:
cpu: 50m
memory: 50Mi
volumeMounts:
- name: registries-conf
mountPath: /registries.conf.d
readOnly: false
- name: configmaps
mountPath: /configmaps
terminationGracePeriodSeconds: 30
volumes:
- name: registries-conf
hostPath:
path: /etc/containers/registries.conf.d
- name: configmaps
configMap:
name: registry-config
defaultMode: 0755
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment