diff --git a/docs/en/02_Developer_Guides/08_Performance/02_HTTP_Cache_Headers.md b/docs/en/02_Developer_Guides/08_Performance/02_HTTP_Cache_Headers.md
index 163248ecc..e883110b9 100644
--- a/docs/en/02_Developer_Guides/08_Performance/02_HTTP_Cache_Headers.md
+++ b/docs/en/02_Developer_Guides/08_Performance/02_HTTP_Cache_Headers.md
@@ -218,7 +218,7 @@ when calculating a cache key, usually in addition to the full URL.
 By default, SilverStripe will output a `Vary` header with the following content: 
 
 ```
-Vary: X-Requested-With, X-Forwarded-Protocol
+Vary: X-Forwarded-Protocol
 ```
 
 To change the value of the `Vary` header, you can change this value by specifying the header in configuration.
@@ -227,3 +227,6 @@ To change the value of the `Vary` header, you can change this value by specifyin
 SilverStripe\Control\HTTP:
   vary: ""
 ```
+
+Note that if you use `Director::is_ajax()` on cached pages
+then you should add `X-Requested-With` to the vary header.
\ No newline at end of file
diff --git a/docs/en/04_Changelogs/4.2.0.md b/docs/en/04_Changelogs/4.2.0.md
index 8543e735e..9147bd5b4 100644
--- a/docs/en/04_Changelogs/4.2.0.md
+++ b/docs/en/04_Changelogs/4.2.0.md
@@ -357,6 +357,9 @@ class PageController extends ContentController
    Note this is different from `Vary: Accept-Encoding`,
    which is important for compression (e.g. gzip), and usually added by
    other layers such as Apache's mod_gzip.
+ * Removed `Vary: X-Requested-With` since it's only applicable when varying
+   content based on the client context, mostly for returning different XHR responses
+   as determined through `Director::is_ajax()`.
  * No longer sets `Last-Modified` date in HTTP response headers in `DataObject::__construct()`.
    Uses `ETag` calculation based on response body which is more accurate,
    and resilient against partial and object caching which can produce stale `Last-Modified` values. 
diff --git a/src/Control/Director.php b/src/Control/Director.php
index 6e7aef593..8b85e1fee 100644
--- a/src/Control/Director.php
+++ b/src/Control/Director.php
@@ -993,6 +993,9 @@ class Director implements TemplateGlobalProvider
      * Checks if the current HTTP-Request is an "Ajax-Request" by checking for a custom header set by
      * jQuery or whether a manually set request-parameter 'ajax' is present.
      *
+     * Note that if you plan to use this to alter your HTTP response on a cached page,
+     * you should add X-Requested-With to the Vary header.
+     *
      * @param HTTPRequest $request
      * @return bool
      */
diff --git a/src/Control/Middleware/HTTPCacheControlMiddleware.php b/src/Control/Middleware/HTTPCacheControlMiddleware.php
index efd97dc66..3ea5f1546 100644
--- a/src/Control/Middleware/HTTPCacheControlMiddleware.php
+++ b/src/Control/Middleware/HTTPCacheControlMiddleware.php
@@ -136,7 +136,6 @@ class HTTPCacheControlMiddleware implements HTTPMiddleware, Resettable
      * @var array
      */
     private static $defaultVary = [
-        "X-Requested-With" => true,
         "X-Forwarded-Protocol" => true,
     ];