diff --git a/core/lib/Drupal/Core/Entity/Entity/Access/FormModeEntityAccessControlHandler.php b/core/lib/Drupal/Core/Entity/Entity/Access/FormModeEntityAccessControlHandler.php
new file mode 100644
index 0000000..81fec25
--- /dev/null
+++ b/core/lib/Drupal/Core/Entity/Entity/Access/FormModeEntityAccessControlHandler.php
@@ -0,0 +1,23 @@
+<?php
+
+namespace Drupal\Core\Entity\Entity\Access;
+
+use Drupal\Core\Access\AccessResult;
+use Drupal\Core\Entity\EntityAccessControlHandler;
+use Drupal\Core\Entity\EntityInterface;
+use Drupal\Core\Session\AccountInterface;
+
+/**
+ * Provides an entity access control handler for form modes.
+ */
+class FormModeEntityAccessControlHandler extends EntityAccessControlHandler {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function checkAccess(EntityInterface $entity, $operation, AccountInterface $account) {
+    /** @var \Drupal\Core\Entity\EntityFormModeInterface $entity */
+    return AccessResult::allowedIfHasPermission($account, 'administer display modes');
+  }
+
+}
diff --git a/core/lib/Drupal/Core/Entity/Entity/Access/ViewModeEntityAccessControlHandler.php b/core/lib/Drupal/Core/Entity/Entity/Access/ViewModeEntityAccessControlHandler.php
new file mode 100644
index 0000000..d5def19
--- /dev/null
+++ b/core/lib/Drupal/Core/Entity/Entity/Access/ViewModeEntityAccessControlHandler.php
@@ -0,0 +1,23 @@
+<?php
+
+namespace Drupal\Core\Entity\Entity\Access;
+
+use Drupal\Core\Access\AccessResult;
+use Drupal\Core\Entity\EntityAccessControlHandler;
+use Drupal\Core\Entity\EntityInterface;
+use Drupal\Core\Session\AccountInterface;
+
+/**
+ * Provides an entity access control handler for modes.
+ */
+class ViewModeEntityAccessControlHandler extends EntityAccessControlHandler {
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function checkAccess(EntityInterface $entity, $operation, AccountInterface $account) {
+    /** @var \Drupal\Core\Entity\EntityViewModeInterface $entity */
+    return AccessResult::allowedIfHasPermission($account, 'administer display modes');
+  }
+
+}
diff --git a/core/lib/Drupal/Core/Entity/Entity/EntityFormMode.php b/core/lib/Drupal/Core/Entity/Entity/EntityFormMode.php
index e5d94a9..9bcecd6 100644
--- a/core/lib/Drupal/Core/Entity/Entity/EntityFormMode.php
+++ b/core/lib/Drupal/Core/Entity/Entity/EntityFormMode.php
@@ -29,6 +29,9 @@
  *     "id" = "id",
  *     "label" = "label"
  *   },
+ *   handlers = {
+ *     "access" = "\Drupal\Core\Entity\Entity\Access\FormModeEntityAccessControlHandler",
+ *   },
  *   config_export = {
  *     "id",
  *     "label",
diff --git a/core/lib/Drupal/Core/Entity/Entity/EntityViewMode.php b/core/lib/Drupal/Core/Entity/Entity/EntityViewMode.php
index 3181607..c2fbd4e 100644
--- a/core/lib/Drupal/Core/Entity/Entity/EntityViewMode.php
+++ b/core/lib/Drupal/Core/Entity/Entity/EntityViewMode.php
@@ -31,6 +31,9 @@
  *     "id" = "id",
  *     "label" = "label"
  *   },
+ *   handlers = {
+ *     "access" = "\Drupal\Core\Entity\Entity\Access\ViewModeEntityAccessControlHandler",
+ *   },
  *   config_export = {
  *     "id",
  *     "label",
diff --git a/core/modules/field_ui/field_ui.module b/core/modules/field_ui/field_ui.module
index 79b3f15..cb82036 100644
--- a/core/modules/field_ui/field_ui.module
+++ b/core/modules/field_ui/field_ui.module
@@ -87,7 +87,6 @@ function field_ui_entity_type_build(array &$entity_types) {
   $form_mode->setFormClass('add', 'Drupal\field_ui\Form\EntityFormModeAddForm');
   $form_mode->setFormClass('edit', 'Drupal\field_ui\Form\EntityDisplayModeEditForm');
   $form_mode->setFormClass('delete', 'Drupal\field_ui\Form\EntityDisplayModeDeleteForm');
-  $form_mode->set('admin_permission', 'administer display modes');
   $form_mode->setLinkTemplate('delete-form', '/admin/structure/display-modes/form/manage/{entity_form_mode}/delete');
   $form_mode->setLinkTemplate('edit-form', '/admin/structure/display-modes/form/manage/{entity_form_mode}');
   $form_mode->setLinkTemplate('add-form', '/admin/structure/display-modes/form/add/{entity_type_id}');
@@ -98,7 +97,6 @@ function field_ui_entity_type_build(array &$entity_types) {
   $view_mode->setFormClass('add', 'Drupal\field_ui\Form\EntityDisplayModeAddForm');
   $view_mode->setFormClass('edit', 'Drupal\field_ui\Form\EntityDisplayModeEditForm');
   $view_mode->setFormClass('delete', 'Drupal\field_ui\Form\EntityDisplayModeDeleteForm');
-  $view_mode->set('admin_permission', 'administer display modes');
   $view_mode->setLinkTemplate('delete-form', '/admin/structure/display-modes/view/manage/{entity_view_mode}/delete');
   $view_mode->setLinkTemplate('edit-form', '/admin/structure/display-modes/view/manage/{entity_view_mode}');
   $view_mode->setLinkTemplate('add-form', '/admin/structure/display-modes/view/add/{entity_type_id}');