Updated Magento 2 Varnish 6 VCL, in cooperation with Varnish Software, see Xkey soft purge version here; https://gist.github.com/peterjaap/7f7bf11aa7d089792e8fcc2fb34760fa

varnish6.vcl

# A number of these changes come form the following PR's; , combines changes in https://github.com/magento/magento2/pull/29360, https://github.com/magento/magento2/pull/28944 and https://github.com/magento/magento2/pull/28894, https://github.com/magento/magento2/pull/35228, https://github.com/magento/magento2/pull/36524, https://github.com/magento/magento2/pull/34323
# VCL version 5.0 is not supported so it should be 4.0 even though actually used Varnish version is 6

# See the Xkey version here: https://gist.github.com/peterjaap/7f7bf11aa7d089792e8fcc2fb34760fa

vcl 4.1;

import std;
# The minimal Varnish version is 6.0
# For SSL offloading, pass the following header in your proxy server or load balancer: '/* {{ ssl_offloaded_header }} */: https'

backend default {
    .host = "/* {{ host }} */";
    .port = "/* {{ port }} */";
    .first_byte_timeout = 600s;
 #   .probe = {
 #       .url = "/health_check.php";
 #       .timeout = 2s;
 #       .interval = 5s;
 #       .window = 10;
 #       .threshold = 5;
 #  }
}

acl purge {
/* {{ ips }} */
}

sub vcl_recv {
    # Add support for Prismic preview functionality
    if (req.http.Cookie ~ "io.prismic.preview") {
        return (pass);
    }
    
    # Bypass generated sitemap files
    if (req.url ~ "^/sitemaps/") {
        return (pass);
    }

    # Remove empty query string parameters
    # e.g.: www.example.com/index.html?    
    if (req.url ~ "\?$") {
        set req.url = regsub(req.url, "\?$", "");
    }

    # Remove port number from host header
    set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
    
    # Sorts query string parameters alphabetically for cache normalization purposes    
    set req.url = std.querysort(req.url);
    
    # Remove the proxy header to mitigate the httpoxy vulnerability
    # See https://httpoxy.org/    
    unset req.http.proxy;
    
    # Add X-Forwarded-Proto header when using https
    if (!req.http.X-Forwarded-Proto && (std.port(server.ip) == 443 || std.port(server.ip) == 8443)) {
        set req.http.X-Forwarded-Proto = "https";
    }
    
    # Reduce grace to the configured setting if the backend is healthy
    # In case of an unhealthy backend, the original grace is used
    if (std.healthy(req.backend_hint)) {
        set req.grace = /* {{ grace_period }} */s;
    }
    
    # Purge logic to remove objects from the cache
    # Tailored to Magento's cache invalidation mechanism
    # The X-Magento-Tags-Pattern value is matched to the tags in the X-Magento-Tags header
    # If X-Magento-Tags-Pattern is not set, a URL-based purge is executed
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        if (!req.http.X-Magento-Tags-Pattern) {
            return (purge);
        }
        ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        return (synth(200, "Purged"));
    }

    if (req.method != "GET" &&
        req.method != "HEAD" &&
        req.method != "PUT" &&
        req.method != "POST" &&
        req.method != "PATCH" &&
        req.method != "TRACE" &&
        req.method != "OPTIONS" &&
        req.method != "DELETE") {
          return (pipe);
    }

    # We only deal with GET and HEAD by default
    if (req.method != "GET" && req.method != "HEAD") {
        return (pass);
    }

    # Bypass health check requests
    if (req.url ~ "^/(pub/)?(health_check.php)$") {
        return (pass);
    }

    # Collapse multiple cookie headers into one
    std.collect(req.http.Cookie, ";");

    # Remove all marketing get parameters to minimize the cache objects
    if (req.url ~ "(\?|&)(_branch_match_id|srsltid|_bta_c|_bta_tid|_ga|_gl|_ke|_kx|campid|cof|customid|cx|dclid|dm_i|ef_id|epik|fbclid|gad_source|gbraid|gclid|gclsrc|gdffi|gdfms|gdftrk|hsa_acc|hsa_ad|hsa_cam|hsa_grp|hsa_kw|hsa_mt|hsa_net|hsa_src|hsa_tgt|hsa_ver|ie|igshid|irclickid|matomo_campaign|matomo_cid|matomo_content|matomo_group|matomo_keyword|matomo_medium|matomo_placement|matomo_source|mc_cid|mc_eid|mkcid|mkevt|mkrid|mkwid|msclkid|mtm_campaign|mtm_cid|mtm_content|mtm_group|mtm_keyword|mtm_medium|mtm_placement|mtm_source|nb_klid|ndclid|origin|pcrid|piwik_campaign|piwik_keyword|piwik_kwd|pk_campaign|pk_keyword|pk_kwd|redirect_log_mongo_id|redirect_mongo_id|rtid|sb_referer_host|ScCid|si|siteurl|s_kwcid|sms_click|sms_source|sms_uph|toolid|trk_contact|trk_module|trk_msg|trk_sid|ttclid|twclid|utm_campaign|utm_content|utm_creative_format|utm_id|utm_marketing_tactic|utm_medium|utm_source|utm_source_platform|utm_term|wbraid|yclid|zanpid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
        set req.url = regsuball(req.url, "(_branch_match_id|srsltid|_bta_c|_bta_tid|_ga|_gl|_ke|_kx|campid|cof|customid|cx|dclid|dm_i|ef_id|epik|fbclid|gad_source|gbraid|gclid|gclsrc|gdffi|gdfms|gdftrk|hsa_acc|hsa_ad|hsa_cam|hsa_grp|hsa_kw|hsa_mt|hsa_net|hsa_src|hsa_tgt|hsa_ver|ie|igshid|irclickid|matomo_campaign|matomo_cid|matomo_content|matomo_group|matomo_keyword|matomo_medium|matomo_placement|matomo_source|mc_cid|mc_eid|mkcid|mkevt|mkrid|mkwid|msclkid|mtm_campaign|mtm_cid|mtm_content|mtm_group|mtm_keyword|mtm_medium|mtm_placement|mtm_source|nb_klid|ndclid|origin|pcrid|piwik_campaign|piwik_keyword|piwik_kwd|pk_campaign|pk_keyword|pk_kwd|redirect_log_mongo_id|redirect_mongo_id|rtid|sb_referer_host|ScCid|si|siteurl|s_kwcid|sms_click|sms_source|sms_uph|toolid|trk_contact|trk_module|trk_msg|trk_sid|ttclid|twclid|utm_campaign|utm_content|utm_creative_format|utm_id|utm_marketing_tactic|utm_medium|utm_source|utm_source_platform|utm_term|wbraid|yclid|zanpid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+(){}%.]+&?", "");
        set req.url = regsub(req.url, "[?|&]+$", "");
    }

    # Static files caching
    if (req.url ~ "^/(pub/)?(media|static)/") {
        # Static files should not be cached by default
        return (pass);

        # But if you use a few locales and don't use CDN you can enable caching static files by commenting previous line (#return (pass);) and uncommenting next 3 lines
        #unset req.http.Https;
        #unset req.http./* {{ ssl_offloaded_header }} */;
        #unset req.http.Cookie;
    }

    # Don't cache the authenticated GraphQL requests
    if (req.url ~ "/graphql" && req.http.Authorization ~ "^Bearer") {
        return (pass);
    }

    return (hash);
}

sub vcl_hash {
    if (req.url !~ "/graphql" && req.http.cookie ~ "X-Magento-Vary=") {
        hash_data(regsub(req.http.cookie, "^.*?X-Magento-Vary=([^;]+);*.*$", "\1"));
    }

    # To make sure http users don't see ssl warning
    hash_data(req.http./* {{ ssl_offloaded_header }} */);

    /* {{ design_exceptions_code }} */

    if (req.url ~ "/graphql") {
        if (req.http.X-Magento-Cache-Id) {
            hash_data(req.http.X-Magento-Cache-Id);
        } else {
            # if no X-Magento-Cache-Id (which already contains Store & Currency) is not set, use the HTTP headers
            hash_data(req.http.Store);
            hash_data(req.http.Content-Currency);
        }
    }
}

sub vcl_backend_response {
	# Serve stale content for three days after object expiration
	# Perform asynchronous revalidation while stale content is served
    set beresp.grace = 3d;

    # All text-based content can be parsed as ESI
    if (beresp.http.content-type ~ "text") {
        set beresp.do_esi = true;
    }

    # Allow GZIP compression on all JavaScript files and all text-based content
    if (bereq.url ~ "\.js$" || beresp.http.content-type ~ "text") {
        set beresp.do_gzip = true;
    }
    
    # Add debug headers
    if (beresp.http.X-Magento-Debug) {
        set beresp.http.X-Magento-Cache-Control = beresp.http.Cache-Control;
    }

    # Only cache HTTP 200 and HTTP 404 responses
    if (beresp.status != 200 && beresp.status != 404) {
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
        return (deliver);
    }
    
    # Don't cache if the request cache ID doesn't match the response cache ID for graphql requests
    if (bereq.url ~ "/graphql" && bereq.http.X-Magento-Cache-Id && bereq.http.X-Magento-Cache-Id != beresp.http.X-Magento-Cache-Id) {
       set beresp.ttl = 120s;
       set beresp.uncacheable = true;
       return (deliver);
    }

    # Remove the Set-Cookie header for cacheable content
    # Only for HTTP GET & HTTP HEAD requests
    if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
        unset beresp.http.Set-Cookie;
    }
}

sub vcl_deliver {
    if (obj.uncacheable) {
        set resp.http.X-Magento-Cache-Debug = "UNCACHEABLE";
    } else if (obj.hits) {
        set resp.http.X-Magento-Cache-Debug = "HIT";
        set resp.http.Grace = req.http.grace;
    } else {
        set resp.http.X-Magento-Cache-Debug = "MISS";
    }

    # Not letting browser to cache non-static files.
    if (resp.http.Cache-Control !~ "private" && req.url !~ "^/(pub/)?(media|static)/") {
        set resp.http.Pragma = "no-cache";
        set resp.http.Expires = "-1";
        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
    }
    
    # Prevent browser caching for customer and checkout pages
    if (req.url ~ "^/(customer|checkout)(/|$)") {
        set resp.http.Cache-Control = "no-store, no-cache, must-revalidate";
    }

    if (!resp.http.X-Magento-Debug) {
        unset resp.http.Age;
    }
    unset resp.http.X-Magento-Debug;
    unset resp.http.X-Magento-Tags;
    unset resp.http.X-Powered-By;
    unset resp.http.Server;
    unset resp.http.X-Varnish;
    unset resp.http.Via;
    unset resp.http.Link;
}

@ThijsFeryn agreed!

@peterjaap This would be the updated purge logic, now that X-Pool is stripped off:

    # Purge logic to remove objects from the cache
    # Tailored to Magento's cache invalidation mechanism
    # The X-Magento-Tags-Pattern value is matched to the tags in the X-Magento-Tags header
    # If X-Magento-Tags-Pattern is not set, a URL-based purge is executed
    if (req.method == "PURGE") {
        if (client.ip !~ purge) {
            return (synth(405, "Method not allowed"));
        }
        if (!req.http.X-Magento-Tags-Pattern) {
            return (purge);
        }
        ban("obj.http.X-Magento-Tags ~ " + req.http.X-Magento-Tags-Pattern);
        return (synth(200, "Purged"));
    }

Can you update your gist accordingly?

@peterjaap std.collect(req.http.Cookie); should be changed to std.collect(req.http.Cookie, ";");.

Can you please adjust the gist?

@ThijsFeryn I did both updates; https://gist.github.com/peterjaap/006169c5d95eeffde3a1cc062de1b514/revisions

@peterjaap I'd like to propose adding the following to vcl_recv to normalize trailing slashes in URLs:

# Remove trailing slashes in URLs
if (req.url ~ "^(/[^\?]+?)/(\?.*)?$") {
    set req.url = regsub(req.url, "^(/[^\?]+?)/(\?.*)?$", "\1\2");
}

Currently, if someone visits a URL like /shirts/, Magento receives the request and redirects to /shirts, which seems unnecessary if we've got the page cached. By removing trailing slash, both /shirts/ and /shirts hit the same cache object. What do you think?

@peterjaap why did you remove this?

 #   .probe = {
 #       .url = "/health_check.php";
 #       .timeout = 2s;
 #       .interval = 5s;
 #       .window = 10;
 #       .threshold = 5;
 #  }

Following discussions on bfache magento/magento2#38376, is it an option to remove no-store from https://gist.github.com/peterjaap/006169c5d95eeffde3a1cc062de1b514#file-varnish6-vcl-L207

Following discussions on bfache magento/magento2#38376, is it an option to remove no-store from https://gist.github.com/peterjaap/006169c5d95eeffde3a1cc062de1b514#file-varnish6-vcl-L207

@peterjaap can you take a look at this and validate the request by @jissereitsma ?

If the suggested fix is valid, I'll create a PR on https://github.com/elgentos/magento2-varnish-extended and I'll write a VTC to ensure our VCL file behaves as expected.

Keep me in the loop!

@jissereitsma agreed!

@ThijsFeryn this is already in (well, out) the VCL in the Varnish extended extension :) See https://github.com/elgentos/magento2-varnish-extended/blob/d77d3b415eec282b0f6127aff30886d0de1c9718/etc/varnish6.vcl#L285

Improvement proposal: Fix for Split Cache (Absolute URLs & Trailing Slashes)

Hi, thanks for sharing this configuration, it's a great baseline for Magento 2.

I noticed a potential "Split Cache" issue with the current vcl_recv logic:

1. Absolute URLs: Varnish treats absolute URLs (e.g., GET https://example.com/) and relative URLs (e.g., GET /) as different cache objects. This often happens with internal purge requests, specific proxies, or crawlers.
2. Trailing Slashes: It also treats /category/ and /category as distinct entries, leading to duplicate cache storage for the same content.

To prevent cache duplication and ensure better hit rates, I suggest adding URL normalization right at the beginning of sub vcl_recv (before std.querysort).

Here is the snippet to add:

    # Normalize absolute URLs (remove scheme and host)
    # Converts "[https://www.example.com/foo](https://www.example.com/foo)" to "/foo"
    if (req.url ~ "^http[s]?://") {
        set req.url = regsub(req.url, "^http[s]?://[^/]+", "");
    }

    # Normalize Trailing Slashes
    # Converts "/foo/" to "/foo" (keeps "/" root intact and ignores query strings)
    if (req.url ~ "^/[^?]+\/$") {
        set req.url = regsub(req.url, "\/$", "");
    }

Improvement proposal: Fix for Split Cache (Absolute URLs & Trailing Slashes)

Hi, thanks for sharing this configuration, it's a great baseline for Magento 2.

I noticed a potential "Split Cache" issue with the current vcl_recv logic:

1. Absolute URLs: Varnish treats absolute URLs (e.g., GET https://example.com/) and relative URLs (e.g., GET /) as different cache objects. This often happens with internal purge requests, specific proxies, or crawlers.
2. Trailing Slashes: It also treats /category/ and /category as distinct entries, leading to duplicate cache storage for the same content.

To prevent cache duplication and ensure better hit rates, I suggest adding URL normalization right at the beginning of sub vcl_recv (before std.querysort).

Here is the snippet to add:

    # Normalize absolute URLs (remove scheme and host)
    # Converts "[https://www.example.com/foo](https://www.example.com/foo)" to "/foo"
    if (req.url ~ "^http[s]?://") {
        set req.url = regsub(req.url, "^http[s]?://[^/]+", "");
    }

    # Normalize Trailing Slashes
    # Converts "/foo/" to "/foo" (keeps "/" root intact and ignores query strings)
    if (req.url ~ "^/[^?]+\/$") {
        set req.url = regsub(req.url, "\/$", "");
    }

@Amadeco The trailing slash solution is a sensible one. I'll implement it. However, I have questions about the scheme & host ending up in req.url: can you provide a curl command to simulate this behavior?

Some Magento installations rely on the forward slash /some/category/ and some use extensions forward slash everything, this would break that integration as it could end up in endless loops internally!

So, should be configurable imho

Some Magento installations rely on the forward slash /some/category/ and some use extensions forward slash everything, this would break that integration as it could end up in endless loops internally!

So, should be configurable imho

@JeroenBoersma @Amadeco In that case, I suggest leaving the URL intact from a proxying perspective, but remove (or add) the trailing slash in vcl_hash to reduce cache duplication. The current template already has some custom hashing logic. I could add the following at the end of vcl_hash:

hash_data(req.http.Host);
hash_data(regsub(req.url, "\/$", ""));
return(lookup);

This would bypass built-in vcl_hash logic as a fallback by adding return(lookup), but the hash_data() functions I'd add would cover this.

Thoughts?

@JeroenBoersma @Amadeco In that case, I suggest leaving the URL intact from a proxying perspective, but remove (or add) the trailing slash in vcl_hash to reduce cache duplication. The current template already has some custom hashing logic. I could add the following at the end of vcl_hash:

hash_data(req.http.Host);
hash_data(regsub(req.url, "\/$", ""));
return(lookup);

This would bypass built-in vcl_hash logic as a fallback by adding return(lookup), but the hash_data() functions I'd add would cover this.

LGTM!

@ThijsFeryn agreed