diff -u b/core/modules/field/field.module b/core/modules/field/field.module
--- b/core/modules/field/field.module
+++ b/core/modules/field/field.module
@@ -336,12 +336,6 @@
 
 /**
  * Implements hook_ENTITY_TYPE_insert() for 'field_config'.
- *
- * Allow other view modes to update their configuration for the new field.
- * Otherwise, configuration for view modes won't get updated until the mode
- * is used for the first time, creating noise in config diffs.
- *
- * @see https://www.drupal.org/node/2915036
  */
 function field_field_config_insert(FieldConfigInterface $field) {
   if ($field->isSyncing()) {
@@ -349,6 +343,9 @@
     return;
   }
 
+  // Allow other view modes to update their configuration for the new field.
+  // Otherwise, configuration for view modes won't get updated until the mode
+  // is used for the first time, creating noise in config diffs.
   /** @var \Drupal\field\EntityDisplayRebuilder $display_rebuilder */
   $display_rebuilder = \Drupal::service('field.entity_display_rebuilder');
   $display_rebuilder->rebuildEntityTypeDisplays($field->getTargetEntityTypeId());
diff -u b/core/modules/field/src/EntityDisplayRebuilder.php b/core/modules/field/src/EntityDisplayRebuilder.php
--- b/core/modules/field/src/EntityDisplayRebuilder.php
+++ b/core/modules/field/src/EntityDisplayRebuilder.php
@@ -2,9 +2,8 @@
 
 namespace Drupal\field;
 
-use Drupal\Core\Entity\Entity\EntityFormDisplay;
-use Drupal\Core\Entity\Entity\EntityViewDisplay;
 use Drupal\Core\Entity\EntityDisplayRepositoryInterface;
+use Drupal\Core\Entity\EntityTypeBundleInfoInterface;
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 
 /**
@@ -27,32 +26,26 @@
   protected $entityDisplayRepository;
 
   /**
+   * The entity type bundle info.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeBundleInfoInterface
+   */
+  protected $entityTypeBundleInfo;
+
+  /**
    * Constructs a new EntityDisplayRebuilder.
    *
    * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
    *   The entity manager.
    * @param \Drupal\Core\Entity\EntityDisplayRepositoryInterface $entity_display_repository
    *   The entity display repository.
+   * @param \Drupal\Core\Entity\EntityTypeBundleInfoInterface $entity_type_bundle_info
+   *   The entity type bundle info.
    */
-  public function __construct(EntityTypeManagerInterface $entity_type_manager, EntityDisplayRepositoryInterface $entity_display_repository) {
+  public function __construct(EntityTypeManagerInterface $entity_type_manager, EntityDisplayRepositoryInterface $entity_display_repository, EntityTypeBundleInfoInterface $entity_type_bundle_info) {
     $this->entityTypeManager = $entity_type_manager;
     $this->entityDisplayRepository = $entity_display_repository;
-  }
-
-  /**
-   * Rebuild displays for all Entity Types.
-   *
-   * @param bool $include_default_displays
-   *   Add 'default' displays. Defaults to FALSE.
-   */
-  public function rebuildAllEntityTypeDisplays($include_default_displays = FALSE) {
-
-    // Loop through all entity types.
-    $entity_types = $this->entityTypeManager->getDefinitions();
-    foreach ($entity_types as $entity_type_id => $entity_type_definition) {
-      $this->rebuildEntityTypeDisplays($entity_type_id, $include_default_displays);
-    }
-
+    $this->entityTypeBundleInfo = $entity_type_bundle_info;
   }
 
   /**
@@ -62,37 +55,38 @@
    *   The entity type machine name.
    * @param bool $include_default_displays
    *   Add 'default' displays. Defaults to FALSE.
+   *
+   * @throws \Drupal\Core\Entity\EntityStorageException
+   *   In case of failures an exception is thrown.
    */
   public function rebuildEntityTypeDisplays($entity_type_id, $include_default_displays = FALSE) {
-    // Loop through all bundles.
-    $entity_bundles = entity_get_bundles($entity_type_id);
-    foreach ($entity_bundles as $bundle => $bundle_definition) {
 
-      // Get the displays.
-      $view_modes = $this->entityDisplayRepository->getViewModes($entity_type_id);
-      $form_modes = $this->entityDisplayRepository->getFormModes($entity_type_id);
-
-      // Explicitly add default displays.
-      if ($include_default_displays) {
-        $view_modes['default'] = [];
-        $form_modes['default'] = [];
-      }
+    // Get the displays.
+    $view_modes = $this->entityDisplayRepository->getViewModes($entity_type_id);
+    $form_modes = $this->entityDisplayRepository->getFormModes($entity_type_id);
+    if ($include_default_displays) {
+      $view_modes['default'] = [];
+      $form_modes['default'] = [];
+    }
 
-      // Update view modes displays.
-      foreach ($view_modes as $view_mode_id => $view_mode) {
-        if ($display = EntityViewDisplay::load($entity_type_id . '.' . $bundle . '.' . $view_mode_id)) {
-          $display->save();
-        }
+    // Loop through all bundles and save all displays.
+    $entity_bundles = $this->entityTypeBundleInfo->getBundleInfo($entity_type_id);
+    foreach ($entity_bundles as $bundle => $bundle_definition) {
+      // Save view mode displays.
+      $view_mode_ids = array_map(function ($view_mode) use ($entity_type_id, $bundle) {
+        return "$entity_type_id.$bundle.$view_mode";
+      }, array_keys($view_modes));
+      foreach ($this->entityTypeManager->getStorage('entity_view_display')->loadMultiple($view_mode_ids) as $display) {
+        $display->save();
       }
-      // Update form modes displays.
-      foreach ($form_modes as $form_mode_id => $form_mode) {
-        if ($display = EntityFormDisplay::load($entity_type_id . '.' . $bundle . '.' . $form_mode_id)) {
-          $display->save();
-        }
+      // Save form mode displays.
+      $form_mode_ids = array_map(function ($form_mode) use ($entity_type_id, $bundle) {
+        return "$entity_type_id.$bundle.$form_mode";
+      }, array_keys($form_modes));
+      foreach ($this->entityTypeManager->getStorage('entity_form_display')->loadMultiple($form_mode_ids) as $display) {
+        $display->save();
       }
-
     }
-
   }
 
 }
diff -u b/core/modules/field/tests/src/Kernel/DisplayModeUpdateTest.php b/core/modules/field/tests/src/Kernel/DisplayModeUpdateTest.php
--- b/core/modules/field/tests/src/Kernel/DisplayModeUpdateTest.php
+++ b/core/modules/field/tests/src/Kernel/DisplayModeUpdateTest.php
@@ -2,10 +2,12 @@
 
 namespace Drupal\Tests\field\Kernel;
 
-use Drupal\field\Entity\FieldConfig;
-use Drupal\field\Entity\FieldStorageConfig;
+use Drupal\Core\Entity\Entity\EntityFormDisplay;
+use Drupal\Core\Entity\Entity\EntityFormMode;
 use Drupal\Core\Entity\Entity\EntityViewDisplay;
 use Drupal\Core\Entity\Entity\EntityViewMode;
+use Drupal\field\Entity\FieldConfig;
+use Drupal\field\Entity\FieldStorageConfig;
 
 /**
  * Tests exposing field definitions for configurable fields.
@@ -15,49 +17,82 @@
 class DisplayModeUpdateTest extends FieldKernelTestBase {
 
   /**
-   * Ensure display modes are updated when fields are created or deleted.
+   * {@inheritdoc}
    */
-  public function testDisplayModeUpdate() {
-    $default_config_name = 'core.entity_view_display.entity_test.entity_test.foobar';
-    $foobar_config_name = 'core.entity_view_display.entity_test.entity_test.foobar';
-
-    entity_test_create_bundle('entity_test');
+  protected function setUp() {
+    parent::setUp();
 
-    // Create default display.
+    // Create 'default' view-display.
     EntityViewMode::create([
       'id' => 'entity_test.default',
       'targetEntityType' => 'entity_test',
       'status' => TRUE,
       'enabled' => TRUE,
     ])->save();
+    EntityViewDisplay::create([
+      'targetEntityType' => 'entity_test',
+      'bundle' => 'entity_test',
+      'mode' => 'default',
+      'status' => TRUE,
+    ])->save();
 
-    $display = EntityViewDisplay::create([
+    // Create 'default' form-display.
+    EntityFormMode::create([
+      'id' => 'entity_test.default',
+      'targetEntityType' => 'entity_test',
+      'status' => TRUE,
+      'enabled' => TRUE,
+    ])->save();
+    EntityFormDisplay::create([
       'targetEntityType' => 'entity_test',
       'bundle' => 'entity_test',
       'mode' => 'default',
       'status' => TRUE,
-    ]);
-    $display->save();
+    ])->save();
 
-    // Create foobar display.
+    // Create a view-mode 'foobar', create view-display that uses it.
     EntityViewMode::create([
       'id' => 'entity_test.foobar',
       'targetEntityType' => 'entity_test',
       'status' => TRUE,
       'enabled' => TRUE,
     ])->save();
+    EntityViewDisplay::create([
+      'targetEntityType' => 'entity_test',
+      'bundle' => 'entity_test',
+      'mode' => 'foobar',
+      'status' => TRUE,
+    ])->save();
 
-    $display = EntityViewDisplay::create([
+    // Create a new form-mode 'foobar', create form-display that uses it.
+    EntityFormMode::create([
+      'id' => 'entity_test.foobar',
+      'targetEntityType' => 'entity_test',
+      'status' => TRUE,
+      'enabled' => TRUE,
+    ])->save();
+    EntityFormDisplay::create([
       'targetEntityType' => 'entity_test',
       'bundle' => 'entity_test',
       'mode' => 'foobar',
       'status' => TRUE,
-    ]);
-    $display->save();
+    ])->save();
+  }
+
+  /**
+   * Ensure display modes are updated when fields are created.
+   */
+  public function testDisplayModeUpdateAfterFieldCreation() {
+    $default_view_display = 'core.entity_view_display.entity_test.entity_test.default';
+    $default_form_display = 'core.entity_form_display.entity_test.entity_test.default';
+    $foobar_view_display = 'core.entity_view_display.entity_test.entity_test.foobar';
+    $foobar_form_display = 'core.entity_form_display.entity_test.entity_test.foobar';
 
     // Sanity test: field has not been created yet, so should not exist in display.
-    $this->assertArrayNotHasKey('field_test', \Drupal::config($default_config_name)->get('hidden'));
-    $this->assertArrayNotHasKey('field_test', \Drupal::config($foobar_config_name)->get('hidden'));
+    $this->assertArrayNotHasKey('field_test', \Drupal::config($default_view_display)->get('hidden'));
+    $this->assertArrayNotHasKey('field_test', \Drupal::config($default_form_display)->get('hidden'));
+    $this->assertArrayNotHasKey('field_test', \Drupal::config($foobar_view_display)->get('hidden'));
+    $this->assertArrayNotHasKey('field_test', \Drupal::config($foobar_form_display)->get('hidden'));
 
     $field_storage = FieldStorageConfig::create([
       'field_name' => 'field_test',
@@ -73,20 +108,10 @@
     ])->save();
 
     // Ensure field is added to display modes.
-    $this->assertArrayHasKey('field_test', \Drupal::config($default_config_name)->get('hidden'));
-    $this->assertArrayHasKey('field_test', \Drupal::config($foobar_config_name)->get('hidden'));
-
-    $entityFieldManager = \Drupal::service('entity_field.manager');
-
-    $fields = $entityFieldManager->getFieldDefinitions('entity_test', 'entity_test');
-
-    if (isset($fields['field_test'])) {
-      $fields['field_test']->delete();
-    }
-
-    // Ensure field is removed from display modes.
-    $this->assertArrayNotHasKey('field_test', \Drupal::config($default_config_name)->get('hidden'));
-    $this->assertArrayNotHasKey('field_test', \Drupal::config($foobar_config_name)->get('hidden'));
+    $this->assertArrayHasKey('field_test', \Drupal::config($default_view_display)->get('hidden'));
+    $this->assertArrayHasKey('field_test', \Drupal::config($default_form_display)->get('hidden'));
+    $this->assertArrayHasKey('field_test', \Drupal::config($foobar_view_display)->get('hidden'));
+    $this->assertArrayHasKey('field_test', \Drupal::config($foobar_form_display)->get('hidden'));
 
   }
 
only in patch2:
unchanged:
--- a/core/modules/system/tests/modules/entity_test/entity_test.module
+++ b/core/modules/system/tests/modules/entity_test/entity_test.module
@@ -231,11 +231,9 @@ function entity_test_entity_form_mode_info_alter(&$form_modes) {
   $entity_info = \Drupal::entityManager()->getDefinitions();
   foreach ($entity_info as $entity_type => $info) {
     if ($entity_info[$entity_type]->getProvider() == 'entity_test') {
-      $form_modes[$entity_type] = [
-        'compact' => [
-          'label' => t('Compact version'),
-          'status' => TRUE,
-        ],
+      $form_modes[$entity_type]['compact'] = [
+        'label' => t('Compact version'),
+        'status' => TRUE,
       ];
     }
   }