diff --git a/core/modules/file/src/FileAccessControlHandler.php b/core/modules/file/src/FileAccessControlHandler.php
index c0d5145fe8..ef05a4d050 100644
--- a/core/modules/file/src/FileAccessControlHandler.php
+++ b/core/modules/file/src/FileAccessControlHandler.php
@@ -25,18 +25,8 @@ protected function checkAccess(EntityInterface $entity, $operation, AccountInter
         // Always allow access to file in public file system.
         return AccessResult::allowed();
       }
-      elseif ($references = $this->getFileReferences($entity)) {
-        foreach ($references as $field_name => $entity_map) {
-          foreach ($entity_map as $referencing_entity_type => $referencing_entities) {
-            /** @var \Drupal\Core\Entity\EntityInterface $referencing_entity */
-            foreach ($referencing_entities as $referencing_entity) {
-              $entity_and_field_access = $referencing_entity->access('view', $account, TRUE)->andIf($referencing_entity->$field_name->access('view', $account, TRUE));
-              if ($entity_and_field_access->isAllowed()) {
-                return $entity_and_field_access;
-              }
-            }
-          }
-        }
+      elseif ($access = $this->getOperationAccessOnReference($entity, $account, 'view')) {
+        return $access;
       }
       elseif ($entity->getOwnerId() == $account->id()) {
         // This case handles new nodes, or detached files. The user who uploaded
@@ -60,9 +50,12 @@ protected function checkAccess(EntityInterface $entity, $operation, AccountInter
     if ($operation == 'delete' || $operation == 'update') {
       $account = $this->prepareUser($account);
       $file_uid = $entity->get('uid')->target_id;
-      // Only admins or the file owner can delete and update the file entity.
+      // Check if the user has permission on a reference.
       // @todo Create a new permission to handle this?
-      if ($account->hasPermission('administer nodes') || ($account->id() == $file_uid)) {
+      if ($access = $this->getOperationAccessOnReference($entity, $account, $operation)) {
+        return $access;
+      }
+      if ($account->id() == $file_uid) {
         return AccessResult::allowed()->cachePerPermissions();
       }
       return AccessResult::forbidden();
@@ -72,6 +65,68 @@ protected function checkAccess(EntityInterface $entity, $operation, AccountInter
     return AccessResult::neutral();
   }
 
+  /**
+   * Checks if the account has access to perform an operation on a file.
+   *
+   * For the 'delete' operation the account must be able to delete all entities
+   * that reference the file.
+   *
+   * For other operations the account must be able to perform the operatoin on
+   * at least one reference or have the admin permission for the entity type of
+   * at least one of the references.
+   *
+   * @param \Drupal\file\FileInterface $entity
+   *   The file entity.
+   * @param \Drupal\Core\Session\AccountInterface $account
+   *   The user account.
+   * @param string $operation
+   *   The operation.
+   *
+   * @return null|\Drupal\Core\Access\AccessResultInterface
+   *   Either the access if it can be determined or the NULL if access cannot be
+   *   determined.
+   */
+  protected function getOperationAccessOnReference(FileInterface $entity, AccountInterface $account, $operation) {
+    $references = $this->getFileReferences($entity);
+    foreach ($references as $field_name => $entity_map) {
+      foreach ($entity_map as $referencing_entity_type => $referencing_entities) {
+        /** @var \Drupal\Core\Entity\EntityInterface $referencing_entity */
+        foreach ($referencing_entities as $referencing_entity) {
+          if ($operation === 'delete') {
+            // There is no 'delete' permission on fields so only check entity
+            // access.
+            $entity_access = $referencing_entity->access($operation, $account, TRUE);
+            if ($entity_access->isForbidden()) {
+              return AccessResult::forbidden()->cachePerPermissions();
+            }
+            elseif (!$entity_access->isAllowed()) {
+              // If the access is not allowed or forbidden don't return an
+              // access.
+              return NULL;
+            }
+          }
+          else {
+            $entity_and_field_access = $referencing_entity->access($operation, $account, TRUE)->andIf($referencing_entity->$field_name->access($operation, $account, TRUE));
+            if ($entity_and_field_access->isAllowed()) {
+              return $entity_and_field_access;
+            }
+            if ($admin_permission = \Drupal::entityTypeManager()->getDefinition($referencing_entity_type)->getAdminPermission()) {
+              if ($account->hasPermission($admin_permission)) {
+                return AccessResult::allowed()->cachePerPermissions();
+              }
+            }
+          }
+        }
+      }
+    }
+    if ($operation === 'delete') {
+      // For delete permission the user must be able to delete all references
+      // to delete the file.
+      return AccessResult::allowed()->cachePerPermissions();
+    }
+    return NULL;
+  }
+
   /**
    * Wrapper for file_get_file_references().
    *