Skip to content

Instantly share code, notes, and snippets.

@Andor

Andor/cloud-init-template.yaml

Created May 12, 2021 10:10
Show Gist options
  • Save Andor/20c3afe3fd8a9d1db447d767764e51b5 to your computer and use it in GitHub Desktop.
Save Andor/20c3afe3fd8a9d1db447d767764e51b5 to your computer and use it in GitHub Desktop.
Download ZIP
GKE Gitlab-runner with cloud-init
Raw
cloud-init-template.yaml
#cloud-config
package_update: true
package_upgrade: false # we will upgrade it with runcmd
fqdn: "${fqdn}"
apt:
primary:
- arches: [default]
search_dns: True
sources:
gitlab.list:
# https://packages.gitlab.com/install/repositories/runner/gitlab-runner/config_file.list?os=ubuntu&dist=bionic&source=script
# https://packages.gitlab.com/gpg.key
source: deb https://packages.gitlab.com/runner/gitlab-runner/ubuntu/ bionic main
key: |
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBF5dI2sBEACyGx5isuXqEV2zJGIx8rlJFCGw6A9g5Zk/9Hj50UpXNuOXlvQl
7vq91m2CAh88Jad7OiMHIJJhX3ZJEOf/pUx/16QKumsaEyBk9CegxUG9jAQXsjL3
WLyP0/l27UzNrOAFB+IUGjsoP+32gsSPiF5P485mirIJNojIAFzDQl3Uo4FbvqYU
9AIRk5kV4nEYz1aKXAovIUsyqrztMtwlAG2xqdwVpGD2A4/w8I143qPGjjhEQmf4
/EeS4CP9ztyLAx+01t2Acwa7Bygsb5KQPuT25UlevuxdDy/Rd5Zn/Lzwr2GQqjUs
6GbM0t1HYjh57e4V+p0qMf6jxXfrDCbehgzFvGS0cx/d7hWHm5sXZIt3gxpjBQU2
8MQWtrR8Y3nTBkCHwOKsXdsdD+YHxTq/yuvxl1Bcyshp29cGWv1es3wn2Z6i9tWe
asGfVewJZiXFSEqSBGguEmLyCAZcWgXvHOV2kc66wG4d4TGIxmoo9GBqEtBftCVH
MGDHt7zeg2hg6EIsx8/nj1duO5nBnbnik5iG8Xv46e/aw2p4DfTdfxHpjvyJudyN
+UI5eSuuuXhyTZWedd5K1Q3+0CmACJ39t/NA6g7cZaw3boFKw3fTWIgOVTvC3y5v
d7wsuyGUk9xNhHLcu6HjB4VPGzcTwQWMFf6+I4qGAUykU5mjTJchQeqmQwARAQAB
tEJHaXRMYWIgQi5WLiAocGFja2FnZSByZXBvc2l0b3J5IHNpZ25pbmcga2V5KSA8
cGFja2FnZXNAZ2l0bGFiLmNvbT6JAlQEEwEKAD4WIQT2QD9lRKOIY9qgtuA/AWGK
UTEvPwUCXl0jawIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA/
AWGKUTEvP3/+EACEpR4JwFz7fAZxKoTzq1xkv7JiVC1jDnfZ6U6tumMDSIBLqlZX
Jv/lixuXC/GCnyiduqqpO14RCkHrCECzNeu7+lt+eiBUpOKvDgkNid6FLMoulu1w
hDhQWss6+40dIWwa5i8maIFg6WOwIiI24PW9T+ywrf2Gfv9mB1YP3ob+8Cx1EVb/
sf5mu1SGHvq2PqNvPeyY3W5vU7rB0Ax5Kcn3e0Z+tUSC8fV7TCg9hm9o2Ou928K4
hmvdFfR0t47cXt1wmZ/pjrWcezVqeIrMJyWtje4hgcO3TSXsfvedEdYn8Q/BgVRw
9KL4DkR1HSemSsPB4YyOwLscjV6p5OCPm0PhPPXUGIdImcQH7jYuEXNi5nnN5dX4
197ooTB2UCk8r0QtnhcQUE2ph46mylcksbR0nKhGh5bYW3jfd0X+MP36reo+EFQ7
Sw35f7P7QvZqnEE8rd5fX3GImKm38xJi+9bGb4IH8WuslUZUMapgQqqBfw1k5+mP
BBqKWSdEsP1i7LBv9jVOaauMYQPLZcodx5prgjrB89V1hCKu+ZQl/hzoCwmeSruD
LUqX/RFeleZO2VeKXh1a/VQ69ThqZ7gyXcrvHopPPGTr9IESoV9/qcZWplEccP9b
FuY9t6HuSpcL7SlbsRVQ0NBQrsQeZR2J0YgvRc3JWgZAfcE5MK2jcoWKCLkCDQRe
XSNrARAApHc0R4tfPntr5bhTuXU/iVLyxlAlzdEv1XsdDC8YBYehT72Jpvpphtq7
sKVsuC59l8szojgO/gW//yKSuc3Gm5h58+HpIthjviGcvZXf/JcN7Pps0UGkLeQN
2+IRZgbA6CAAPh2njE60v5iXgS91bxlSJi8GVHq1h28kbKQeqUYthu9yA2+8J4Fz
ivYV2VImKLSxbQlc86tl6rMKKIIOph+N4WujJgd5HZ80n2qp1608X3+9CXvtBasX
VCI2ZqCuWjffVCOQzsqRbJ6LQyMbgti/23F4Yqjqp+8eyiDNL6MyWJCBbtkW3Imi
FHfR0sQIM6I7fk0hvt9ljx9SG6az/s3qWK5ceQ7XbJgCAVS4yVixfgIjWvNE5ggE
QNOmeF9r76t0+0xsdMYJR6lxdaQI8AAYaoMXTkCXX2DrASOjjEP65Oq/d42xpSf9
tG6XIq+xtRQyFWSMc+HfTlEHbfGReAEBlJBZhNoAwpuDckOC08vw7v2ybS5PYjJ4
5Kzdwej0ga03Wg9hrAFd/lVa5eO4pzMLuexLplhpIbJjYwCUGS4cc/LQ2jq4fue5
oxDpWPN+JrBH8oyqy91b10e70ohHppN8dQoCa79ySgMxDim92oHCkGnaVyULYDqJ
zy0zqbi3tJu639c4pbcggxtAAr0I3ot8HPhKiNJRA6u8HTm//xEAEQEAAYkCPAQY
AQoAJhYhBPZAP2VEo4hj2qC24D8BYYpRMS8/BQJeXSNrAhsMBQkDwmcAAAoJED8B
YYpRMS8/vzQP/iO0poRR9ZYjonP5GGIARRnF+xpWCRTZVSHLcAfS0ujZ7ekXoeeS
JNMJ/7T4Yk1EJ9MTFZ83Jj4UybKO3Rw+/iPmcPpqUQGaEReYLlx7SyxmsOBXf+Q9
PtyUmGO47tL+eAPInYyxsWGib/EeOw4KQrfByAIPWu0aeNeXadzxBLIkqD863H5q
nTDrXOw6SLprlGt2zlc+XQKDv3DZez6wTcp205xdaNs55Bfk9pmKUS/ey3ZP7GvC
CDEGxuWulVSKL2DYtq0sEZD7pQYSy8gBTqXLQAyfmPDcxe9Lczhk3UYrUUomN1/w
+VE09q75yNqkaHdckVt+aYAHMgQ0ilmwTg6+OlEK+ZQkUT94viB6YW7B0M4uzols
9FSDxXea/uKn75jTSkA3GAXf7O5hqbkDDctJbtO2pPdLDxbXN95iZ9xpgRE3exGl
ucjgV5XGpLO4XXf0GTzug/TJAtNljJ/44+6meO0WwOwLMMhAJVxcp1fpbtgRmrcJ
8bAsCkV5EO8SeQZDu2C8I9tMGlJ1VLTAfv6Lv2Z89B1AOOweGz4I48i9lux+HdXd
HewnA37zx0XNjNQmqiG85UWUusnDxF0Je2jEhGIpHK/KdyI1BfNzX3d5HVoM1VE3
THtRZHnetoMek8L5x/ciYQNIt40rQ6MHtPEo1ZC4346DP6eJmeX1DGGI
=91uZ
-----END PGP PUBLIC KEY BLOCK-----
write_files:
- path: /etc/apt/preferences.d/pin-gitlab-runner.pref
owner: root:root
permissions: '0644'
content: |
Explanation: Prefer GitLab provided packages over the Debian native ones
Package: gitlab-runner
Pin: origin packages.gitlab.com
Pin-Priority: 1001
- path: /etc/gitlab-runner/gcs-creds.json
owner: root:root
permissions: '0640'
encoding: b64
content: "${cache-key}"
- path: /etc/gitlab-runner/config.toml
owner: root:root
permissions: '0640'
content: |
concurrent = ${concurrent}
check_interval = 0
log_format = "text"
- path: /etc/gitlab-runner/gcp-credentials.json
owner: root:root
permissions: '0640'
encoding: b64
content: "${machine-key}"
- path: /etc/systemd/system/gitlab-runner.service.d/override.conf
owner: root:root
permissions: '0644'
content: |
[Service]
Environment=GOOGLE_APPLICATION_CREDENTIALS=/etc/gitlab-runner/gcp-credentials.json
runcmd:
- export DEBIAN_FRONTEND=noninteractive
- apt-get remove -q -y --purge snapd lxd lxcfs liblxc-common liblxc1 unattended-upgrades tmux landscape-common language-selector-common snapd
- apt-get autoremove -q -y
- apt-get upgrade -q -y
- apt-get install --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y docker.io gitlab-runner git tzdata less
- ln -fs /usr/share/zoneinfo/Europe/Helsinki /etc/localtime
- curl -L https://gitlab-docker-machine-downloads.s3.amazonaws.com/v0.16.2-gitlab.4/docker-machine > /usr/bin/docker-machine && chmod +x /usr/bin/docker-machine
- >
gitlab-runner register
--non-interactive
--url=https://gitlab.com/
--registration-token=${registration-token}
--name=${fqdn}
--executor=docker+machine
--docker-image=debian:buster-slim
--docker-pull-policy=always
--docker-privileged
--machine-idle-nodes=1
--machine-idle-time=600
--machine-machine-driver=google
--machine-machine-name="gitlab-autoscale-%s"
--machine-machine-options="google-project=${machine-project}"
--machine-machine-options="google-machine-type=${machine-type}"
--machine-machine-options="google-zone=${machine-zone}"
--machine-machine-options="google-machine-image=ubuntu-os-cloud/global/images/family/ubuntu-minimal-1804-lts"
--machine-machine-options="google-tags=gitlab-runner-slave"
--machine-machine-options="google-use-internal-ip=${use-internal-ip}"
--machine-machine-options="google-preemptible=true"
--machine-machine-options="google-min-cpu-platform=Intel Cascade Lake"
--machine-machine-options="google-disk-size=30"
--machine-machine-options="google-disk-type=pd-ssd"
--machine-off-peak-periods="* * 0-9,16-23 * * mon-fri *"
--machine-off-peak-periods="* * * * * sat,sun *"
--machine-off-peak-timezone=Europe/Helsinki
--machine-off-peak-idle-count=0
--machine-off-peak-idle-time=120
--cache-type=gcs
--cache-shared
--cache-path=gitlab-ci/runner-cache
--cache-gcs-credentials-file=/etc/gitlab-runner/gcs-creds.json
--cache-gcs-bucket-name=${cache-bucket}
- systemctl enable gitlab-runner
- systemctl disable docker
power_state:
mode: reboot
condition: True
Raw
main.tf
provider "google" {
}
provider "random" {
}
provider "template" {
}
data "google_compute_zones" "zones" {
project = var.machine-project
region = var.machine-region
}
data "google_compute_image" "image" {
family = "ubuntu-minimal-2004-lts"
project = "ubuntu-os-cloud"
}
resource "random_string" "instance" {
count = var.instances-count
length = 12
special = false
upper = false
}
# add this option when gitlab-runner itself will be running inside google cloud
# --machine-machine-options="google-use-internal-ip=true"
locals {
name-prefix = "gitlab-runner-"
instances-names = [
for i in range(0, var.instances-count) : {
name = "${local.name-prefix}${random_string.instance[i].result}"
dns_name = "${local.name-prefix}${random_string.instance[i].result}.${var.dns-zone.dns_name}"
fqdn = "${local.name-prefix}${random_string.instance[i].result}.${replace(var.dns-zone.dns_name, "/\\.$/", "")}"
}
]
instances = [
for i in range(0, var.instances-count) : {
name = local.instances-names[i].name
dns_name = local.instances-names[i].dns_name
user-data = templatefile("${path.module}/cloud-init.template.yaml", {
fqdn = local.instances-names[i].fqdn
concurrent = var.concurrent
registration-token = var.registration-token
cache-bucket = var.cache-bucket
cache-key = var.cache-key
machine-key = var.machine-key
machine-project = var.machine-project
machine-type = var.machine-type
machine-zone = data.google_compute_zones.zones.names[i]
use-internal-ip = true
})
zone = data.google_compute_zones.zones.names[i]
}
]
}
resource "google_compute_instance" "instance" {
count = length(local.instances)
name = local.instances[count.index].name
machine_type = var.runner-machine-type
project = var.runner-project
zone = local.instances[count.index].zone
boot_disk {
initialize_params {
size = "20"
image = data.google_compute_image.image.name
}
}
network_interface {
network = "default"
access_config {
// TODO: remove external IP from gitlab-runner and docker-machine instances
}
}
metadata = {
user-data = local.instances[count.index].user-data
server-role = "gitlab-runner"
env = "infra"
}
service_account {
# copy-pasted form manually created instance
scopes = [
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
"https://www.googleapis.com/auth/monitoring.write",
]
}
lifecycle {
ignore_changes = [
boot_disk[0].initialize_params,
]
}
}
Raw
variables.tf
variable "runner-machine-type" {
default = "f1-micro"
}
variable "runner-project" {
}
variable "instances-count" {
}
variable "dns-zone" {
}
variable "cache-bucket" {
}
variable "cache-key" {
}
variable "registration-token" {
}
variable "concurrent" {
}
variable "machine-project" {
description = "google project to run machines by docker+machine executor"
}
variable "machine-key" {
description = "google auth credentials to use by docker+machine executor"
}
variable "machine-region" {
}
# n2-standard-2: 2 cores (2.8GHz - 3.9GHz) 8GiB memory $0.0235/hour aka $17.16/month
# n2d-standard-2: 2 cores (2.25GHz-2.7GHz-3.3 GHz) 8GiB memory $0.0204/h aka $14.89/m
# c2-standard-4: 4 cores (3.1GHz - 3.9GHz) 16GiB memory $0.0505/hour aka $36.865/month
variable "machine-type" {
default = "n2-standard-2"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment