diff --git a/core/lib/Drupal/Core/Access/AccessResult.php b/core/lib/Drupal/Core/Access/AccessResult.php
index a5b748d..dfdfb0c 100644
--- a/core/lib/Drupal/Core/Access/AccessResult.php
+++ b/core/lib/Drupal/Core/Access/AccessResult.php
@@ -401,6 +401,7 @@ public function orIf(AccessResultInterface $other) {
     //    cached access result will be used, even though for that other role, B
     //    is forbidden!
     $merge_other = FALSE;
+    $bothCacheable = $this->isCacheable() && $other instanceof CacheableInterface && $other->isCacheable();
     if ($this->isForbidden() || $other->isForbidden()) {
       $result = static::forbidden();
       if (!$this->isForbidden() || (!$this->isCacheable() && $other->isForbidden())) {
@@ -409,7 +410,6 @@ public function orIf(AccessResultInterface $other) {
     }
     elseif ($this->isAllowed() || $other->isAllowed()) {
       $result = static::allowed();
-      $bothCacheable = $this->isCacheable() && $other instanceof CacheableInterface && $other->isCacheable();
       if (!$this->isAllowed() || (!$this->isCacheable() && $other->isAllowed()) || $bothCacheable) {
         $merge_other = TRUE;
       }
diff --git a/core/lib/Drupal/Core/Field/FormatterBase.php b/core/lib/Drupal/Core/Field/FormatterBase.php
index 5ad0d81..cd7f6fc 100644
--- a/core/lib/Drupal/Core/Field/FormatterBase.php
+++ b/core/lib/Drupal/Core/Field/FormatterBase.php
@@ -104,7 +104,7 @@ public function view(FieldItemListInterface $items) {
 
       $addition = array_merge($info, $elements);
     }
-    // Otherwise, we are probably dealing with access and cacheability metadata
+    // Otherwise, we are probably dealing with access cacheability metadata
     // only. Just set the properties so that the necessary cache contexts and
     // tags are bubbled.
     elseif ($elements) {
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
index 7f05230..de0f170 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
@@ -69,18 +69,20 @@ protected function getEntitiesToView(array &$build, EntityReferenceFieldItemList
         }
 
         // Check entity access if needed.
-        $access = $this->defaultAccess($item)->orIf($entity->access('view', NULL, TRUE));
-        $access_cacheability = BubbleableMetadata::createFromAccessResult($access);
-        if ($access->isAllowed()) {
-          // Add the referring item, in case the formatter needs it.
-          $entity->_referringItem = $items[$delta];
-          // Add the cacheability metadata, to be applied with
-          // ::setAccessCacheability() by the formatter.
-          $entity->_accessCacheability = $access_cacheability;
-          $entities[$delta] = $entity;
-        }
-        else {
-          $inaccessible_entities_cacheability = $inaccessible_entities_cacheability->merge($access_cacheability);
+        if ($this->needsAccessCheck($item)) {
+          $access = $this->defaultAccess($item)->orIf($entity->access('view', NULL, TRUE));
+          $access_cacheability = BubbleableMetadata::createFromAccessResult($access);
+          if ($access->isAllowed()) {
+            // Add the referring item, in case the formatter needs it.
+            $entity->_referringItem = $items[$delta];
+            // Add the cacheability metadata, to be applied with
+            // ::setAccessCacheability() by the formatter.
+            $entity->_accessCacheability = $access_cacheability;
+            $entities[$delta] = $entity;
+          }
+          else {
+            $inaccessible_entities_cacheability = $inaccessible_entities_cacheability->merge($access_cacheability);
+          }
         }
       }
     }
diff --git a/core/modules/file/src/Plugin/Field/FieldFormatter/FileFormatterBase.php b/core/modules/file/src/Plugin/Field/FieldFormatter/FileFormatterBase.php
index 76a45de..d914919 100644
--- a/core/modules/file/src/Plugin/Field/FieldFormatter/FileFormatterBase.php
+++ b/core/modules/file/src/Plugin/Field/FieldFormatter/FileFormatterBase.php
@@ -24,6 +24,19 @@ protected function needsEntityLoad(EntityReferenceItem $item) {
   }
 
   /**
+   * Returns whether entity access should be checked.
+   *
+   * @param \Drupal\Core\Field\Plugin\Field\FieldType\EntityReferenceItem $item
+   *    The item to check.
+   *
+   * @return bool
+   *   TRUE if entity access should be checked.
+   */
+  protected function needsAccessCheck(EntityReferenceItem $item) {
+    return TRUE;
+  }
+
+  /**
    * {@inheritdoc}
    */
   protected function defaultAccess(EntityReferenceItem $item) {