core/lib/Drupal/Core/Entity/Entity/EntityViewDisplay.php            | 1 +
 core/lib/Drupal/Core/Field/FormatterBase.php                        | 6 +++---
 .../Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php    | 5 ++++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/lib/Drupal/Core/Entity/Entity/EntityViewDisplay.php b/core/lib/Drupal/Core/Entity/Entity/EntityViewDisplay.php
index bed6f24..accdd92 100644
--- a/core/lib/Drupal/Core/Entity/Entity/EntityViewDisplay.php
+++ b/core/lib/Drupal/Core/Entity/Entity/EntityViewDisplay.php
@@ -246,6 +246,7 @@ public function buildMultiple(array $entities) {
             $build_list[$id][$name] = $formatter->view($items);
           }
           // Apply the field access cacheability metadata to the render array.
+          // @todo Use RendererInterface::addDependency() when https://www.drupal.org/node/2444231 lands
           BubbleableMetadata::createFromRenderArray($build_list[$id][$name])
             ->merge(BubbleableMetadata::createFromAccessResult($field_access))
             ->applyTo($build_list[$id][$name]);
diff --git a/core/lib/Drupal/Core/Field/FormatterBase.php b/core/lib/Drupal/Core/Field/FormatterBase.php
index 51e19e3..fa064ba 100644
--- a/core/lib/Drupal/Core/Field/FormatterBase.php
+++ b/core/lib/Drupal/Core/Field/FormatterBase.php
@@ -104,9 +104,9 @@ public function view(FieldItemListInterface $items) {
 
       $addition = array_merge($info, $elements);
     }
-    // Otherwise, we're probably dealing with access cacheability metadata only.
-    // Just set the properties so that the necessary cache contexts and tags are
-    // bubbled.
+    // Otherwise, we are dealing with access cacheability metadata only (i.e.
+    // only the #cache property is set). Pass this, so that the necessary cache
+    // contexts and tags are bubbled.
     elseif ($elements) {
       $addition = $elements;
     }
diff --git a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
index 7f05230..cac7341 100644
--- a/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
+++ b/core/lib/Drupal/Core/Field/Plugin/Field/FieldFormatter/EntityReferenceFormatterBase.php
@@ -69,7 +69,10 @@ protected function getEntitiesToView(array &$build, EntityReferenceFieldItemList
         }
 
         // Check entity access if needed.
-        $access = $this->defaultAccess($item)->orIf($entity->access('view', NULL, TRUE));
+        $access = $this->defaultAccess($item);
+        if (!$access->isAllowed()) {
+          $access = $this->defaultAccess($item)->orIf($entity->access('view', NULL, TRUE));
+        }
         $access_cacheability = BubbleableMetadata::createFromAccessResult($access);
         if ($access->isAllowed()) {
           // Add the referring item, in case the formatter needs it.