Created
March 20, 2024 08:24
-
-
Save wujku/91bdb7e7026f6365b97b5f34fd999386 to your computer and use it in GitHub Desktop.
Advanced VCL with Varnish sharding example (Magento 2)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6 | |
vcl 4.0; | |
import directors; | |
import std; | |
# The minimal Varnish version is 6.0 | |
# For SSL offloading, pass the following header in your proxy server or load balancer: 'X-Forwarded-Proto: https' | |
# Define probe used for heartbeat | |
probe heartbeat { | |
.request = "HEAD /heartbeat HTTP/1.1" | |
"Connection: close" | |
"Host: shard"; | |
.interval = 2s; | |
.window = 4; | |
.threshold = 2; | |
} | |
backend magento-cache1 { | |
.host = "192.168.1.2"; | |
.port = "6081"; | |
.probe = heartbeat; | |
} | |
backend magento-cache2 { | |
.host = "192.168.1.9"; | |
.port = "6081"; | |
.probe = heartbeat; | |
} | |
backend magento-app1 { | |
.host = "192.168.1.3"; | |
.port = "80"; | |
.first_byte_timeout = 600s; | |
.probe = { | |
.url = "/health_check.php"; | |
.timeout = 4s; | |
.interval = 5s; | |
.window = 10; | |
.threshold = 5; | |
} | |
} | |
backend magento-app2 { | |
.host = "192.168.1.8"; | |
.port = "80"; | |
.first_byte_timeout = 600s; | |
.probe = { | |
.url = "/health_check.php"; | |
.timeout = 4s; | |
.interval = 5s; | |
.window = 10; | |
.threshold = 5; | |
} | |
} | |
backend magento-admin1 { | |
.host = "192.168.1.4"; | |
.port = "80"; | |
.first_byte_timeout = 600s; | |
.probe = { | |
.url = "/health_check.php"; | |
.timeout = 4s; | |
.interval = 5s; | |
.window = 10; | |
.threshold = 5; | |
} | |
} | |
acl debug { | |
"1.2.3.4; # Host to expose debug headers | |
} | |
acl purge { | |
# Allow application hosts to purge cache | |
"192.168.1.3"; | |
"192.168.1.8"; | |
"192.168.1.4"; | |
"localhost"; | |
} | |
acl acl_cluster { | |
# Allow cache hosts to heartbeat request | |
"192.168.1.2"; | |
"192.168.1.9"; | |
} | |
sub vcl_init { | |
new cluster = directors.shard(); | |
cluster.add_backend(magento-cache1); | |
cluster.add_backend(magento-cache2); | |
cluster.set_rampup(10s); | |
cluster.set_warmup(0.1); | |
cluster.reconfigure(); | |
new lb = directors.round_robin(); | |
lb.add_backend(magento-app1); | |
lb.add_backend(magento-app2); | |
new admin = directors.round_robin(); | |
admin.add_backend(magento-admin1); | |
} | |
sub vcl_recv { | |
if (req.restarts > 0) { | |
set req.hash_always_miss = true; | |
} | |
# Answer to heartbeats only from cluster members | |
if (remote.ip ~ acl_cluster) { | |
if (req.http.Host == "shard") { | |
if (req.url == "/heartbeat") { | |
return (synth(200)); | |
} | |
return (synth(404)); | |
} | |
} | |
if (!client.ip ~ debug) { | |
set req.http.x-debug = "debug"; | |
} | |
if (req.method == "PURGE") { | |
if (client.ip !~ purge) { | |
return (synth(405, "Method not allowed")); | |
} | |
# To use the X-Pool header for purging varnish during automated deployments, make sure the X-Pool header | |
# has been added to the response in your backend server config. This is used, for example, by the | |
# capistrano-magento2 gem for purging old content from varnish during it's deploy routine. | |
if (!req.http.X-Magento-Tags-Pattern && !req.http.X-Pool) { | |
return (synth(400, "X-Magento-Tags-Pattern or X-Pool header required")); | |
} | |
if (req.http.X-Magento-Tags-Pattern) { | |
ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern); | |
} | |
if (req.http.X-Pool) { | |
ban("obj.http.X-Pool ~ " + req.http.X-Pool); | |
} | |
return (synth(200, "Purged")); | |
} | |
# VarnishManager | |
if (req.method == "BAN") { | |
if (client.ip !~ purge) { | |
return (synth(405, "Method not allowed")); | |
} | |
if (req.http.X-Regex) { | |
ban("req.url ~ " + req.http.X-Regex); | |
return(synth(200, "Regex ban added")); | |
} | |
ban("req.url == " + req.url); | |
return(synth(200, "Url ban added")); | |
} | |
if (req.url ~ "admin_") { | |
set req.backend_hint = admin.backend(); | |
} else if (req.url ~ "/rest/V1/") { | |
set req.backend_hint = admin.backend(); | |
} else { | |
# Let shard director to pick the backend (shard) | |
set req.backend_hint = cluster.backend(by=URL); | |
set req.http.x-shard = req.backend_hint; | |
# Do we have the content? If yes, reroute to the backend | |
# If not, pass the request to the appropriate node | |
if (req.http.x-shard == server.identity) { | |
set req.backend_hint = lb.backend(); | |
} else { | |
return(pass); | |
} | |
} | |
if (req.method != "GET" && | |
req.method != "HEAD" && | |
req.method != "PUT" && | |
req.method != "POST" && | |
req.method != "TRACE" && | |
req.method != "OPTIONS" && | |
req.method != "DELETE") { | |
/* Non-RFC2616 or CONNECT which is weird. */ | |
return (pipe); | |
} | |
# We only deal with GET and HEAD by default | |
if (req.method != "GET" && req.method != "HEAD") { | |
return (pass); | |
} | |
# Bypass shopping cart, checkout and search requests | |
if (req.url ~ "/checkout" || req.url ~ "/catalogsearch") { | |
return (pass); | |
} | |
# Bypass health check requests | |
if (req.url ~ "/pub/health_check.php") { | |
return (pass); | |
} | |
# Set initial grace period usage status | |
set req.http.grace = "none"; | |
# normalize url in case of leading HTTP scheme and domain | |
set req.url = regsub(req.url, "^http[s]?://", ""); | |
# collect all cookies | |
std.collect(req.http.Cookie); | |
# Compression filter. See https://www.varnish-cache.org/trac/wiki/FAQ/Compression | |
if (req.http.Accept-Encoding) { | |
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") { | |
# No point in compressing these | |
unset req.http.Accept-Encoding; | |
} elsif (req.http.Accept-Encoding ~ "gzip") { | |
set req.http.Accept-Encoding = "gzip"; | |
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") { | |
set req.http.Accept-Encoding = "deflate"; | |
} else { | |
# unknown algorithm | |
unset req.http.Accept-Encoding; | |
} | |
} | |
# Remove all marketing get parameters to minimize the cache objects | |
if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") { | |
set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", ""); | |
set req.url = regsub(req.url, "[?|&]+$", ""); | |
} | |
# Static files caching | |
if (req.url ~ "^/(pub/)?(media|static)/") { | |
# Static files should not be cached by default | |
return (pass); | |
# But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines | |
#unset req.http.Https; | |
#unset req.http.X-Forwarded-Proto; | |
#unset req.http.Cookie; | |
} | |
# Authenticated GraphQL requests should not be cached by default | |
if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") { | |
return (pass); | |
} | |
return (hash); | |
} | |
sub vcl_hash { | |
if (req.http.cookie ~ "X-Magento-Vary=") { | |
hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1")); | |
} | |
# For multi site configurations to not cache each other's content | |
if (req.http.host) { | |
hash_data(req.http.host); | |
} else { | |
hash_data(server.ip); | |
} | |
# To make sure http users don't see ssl warning | |
if (req.http.X-Forwarded-Proto) { | |
hash_data(req.http.X-Forwarded-Proto); | |
} | |
if (req.url ~ "/graphql") { | |
call process_graphql_headers; | |
} | |
} | |
sub process_graphql_headers { | |
if (req.http.Store) { | |
hash_data(req.http.Store); | |
} | |
if (req.http.Content-Currency) { | |
hash_data(req.http.Content-Currency); | |
} | |
} | |
sub vcl_backend_response { | |
set beresp.http.node = "magento-cache1"; | |
set beresp.grace = 3d; | |
if (beresp.http.content-type ~ "text") { | |
set beresp.do_esi = true; | |
} | |
if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") { | |
set beresp.do_gzip = true; | |
} | |
if (beresp.http.X-Magento-Debug) { | |
set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control; | |
} | |
# cache only successfully responses and 404s | |
if (beresp.status != 200 && beresp.status != 404) { | |
set beresp.ttl = 0s; | |
set beresp.uncacheable = true; | |
return (deliver); | |
} elsif (beresp.http.Cache-Control ~ "private") { | |
set beresp.uncacheable = true; | |
set beresp.ttl = 86400s; | |
return (deliver); | |
} | |
# validate if we need to cache it and prevent from setting cookie | |
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) { | |
unset beresp.http.set-cookie; | |
} | |
# If page is not cacheable then bypass varnish for 2 minutes as Hit-For-Pass | |
if (beresp.ttl <= 0s || | |
beresp.http.Surrogate-control ~ "no-store" || | |
(!beresp.http.Surrogate-Control && | |
beresp.http.Cache-Control ~ "no-cache|no-store") || | |
beresp.http.Vary == "*") { | |
# Mark as Hit-For-Pass for the next 2 minutes | |
set beresp.ttl = 120s; | |
set beresp.uncacheable = true; | |
} | |
return (deliver); | |
} | |
sub vcl_deliver { | |
#if (resp.http.X-Magento-Debug) { | |
if (resp.http.x-varnish ~ " ") { | |
set resp.http.X-Magento-Cache-Debug = "HIT"; | |
set resp.http.Grace = req.http.grace; | |
} else { | |
set resp.http.X-Magento-Cache-Debug = "MISS"; | |
} | |
#} else { | |
# unset resp.http.Age; | |
#} | |
# Not letting browser to cache non-static files. | |
if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") { | |
set resp.http.Pragma = "no-cache"; | |
set resp.http.Expires = "-1"; | |
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0"; | |
} | |
if (!req.http.x-debug) { | |
# unset resp.http.X-Magento-Debug; | |
# unset resp.http.X-Varnish; | |
unset resp.http.X-Magento-Tags; | |
unset resp.http.X-Powered-By; | |
unset resp.http.Server; | |
unset resp.http.Via; | |
unset resp.http.Link; | |
} | |
} | |
sub vcl_hit { | |
if (obj.ttl >= 0s) { | |
# Hit within TTL period | |
return (deliver); | |
} | |
if (std.healthy(req.backend_hint)) { | |
if (obj.ttl + 300s > 0s) { | |
# Hit after TTL expiration, but within grace period | |
set req.http.grace = "normal (healthy server)"; | |
return (deliver); | |
} else { | |
# Hit after TTL and grace expiration | |
return (restart); | |
} | |
} else { | |
# server is not healthy, retrieve from cache | |
set req.http.grace = "unlimited (unhealthy server)"; | |
return (deliver); | |
} | |
} | |
sub vcl_backend_error { | |
set beresp.http.Content-Type = "text/html; charset=utf-8"; | |
synthetic(std.fileread("/etc/varnish/maintenance_template.html")); | |
return (deliver); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
https://info.varnish-software.com/blog/creating-self-routing-varnish-cluster
https://ibm.github.io/varnish-operator/vcl-configuration.html