diff --git a/config/schema/entity_browser.schema.yml b/config/schema/entity_browser.schema.yml
index 7416799..cf2c9ec 100644
--- a/config/schema/entity_browser.schema.yml
+++ b/config/schema/entity_browser.schema.yml
@@ -169,6 +169,9 @@ field.widget.settings.entity_browser_entity_reference:
     field_widget_remove:
       type: boolean
       label: 'Field widget remove'
+    field_widget_replace:
+      type: boolean
+      label: 'Field widget replace'
     open:
       type: boolean
       label: 'Open'
@@ -211,6 +214,9 @@ field.widget.settings.entity_browser_file:
     field_widget_remove:
       type: boolean
       label: 'Field widget remove'
+    field_widget_replace:
+      type: boolean
+      label: 'Field widget replace'
     open:
       type: boolean
       label: 'Open'
diff --git a/modules/example/config/install/core.entity_form_display.node.entity_browser_test.default.yml b/modules/example/config/install/core.entity_form_display.node.entity_browser_test.default.yml
index de7d83c..0bf7b37 100644
--- a/modules/example/config/install/core.entity_form_display.node.entity_browser_test.default.yml
+++ b/modules/example/config/install/core.entity_form_display.node.entity_browser_test.default.yml
@@ -44,6 +44,7 @@ content:
       open: false
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
     third_party_settings: {  }
   field_files1:
     type: entity_browser_entity_reference
@@ -55,6 +56,7 @@ content:
       open: false
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
     third_party_settings: {  }
   field_files_over_ajax:
     weight: 36
@@ -63,6 +65,7 @@ content:
       field_widget_display: label
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
       selection_mode: selection_edit
       open: false
       field_widget_display_settings: {  }
@@ -74,6 +77,7 @@ content:
       entity_browser: test_files
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
       open: false
     third_party_settings: {  }
     type: entity_browser_file
@@ -87,6 +91,7 @@ content:
       open: false
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
     third_party_settings: {  }
   path:
     type: path
diff --git a/src/Plugin/Field/FieldWidget/EntityReferenceBrowserWidget.php b/src/Plugin/Field/FieldWidget/EntityReferenceBrowserWidget.php
index 592a553..66bbf74 100644
--- a/src/Plugin/Field/FieldWidget/EntityReferenceBrowserWidget.php
+++ b/src/Plugin/Field/FieldWidget/EntityReferenceBrowserWidget.php
@@ -136,6 +136,7 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
       'field_widget_display' => 'label',
       'field_widget_edit' => TRUE,
       'field_widget_remove' => TRUE,
+      'field_widget_replace' => FALSE,
       'field_widget_display_settings' => [],
       'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
     ] + parent::defaultSettings();
@@ -196,12 +197,18 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
     ];
 
     $element['field_widget_remove'] = [
-      '#title' => $this->t('Display Remove / Replace button'),
-      '#description' => $this->t('In required single-valued fields, this button will show "Replace" instead of the default "Remove". In that scenario, once clicked the current selection will be emptied and the browser will be opened to allow a new selection to take place.'),
+      '#title' => $this->t('Display Remove button'),
       '#type' => 'checkbox',
       '#default_value' => $this->getSetting('field_widget_remove'),
     ];
 
+    $element['field_widget_replace'] = [
+      '#title' => $this->t('Display Replace button'),
+      '#description' => $this->t('Note that the Replace button will only be displayed if there is a single entity in the current selection.'),
+      '#type' => 'checkbox',
+      '#default_value' => $this->getSetting('field_widget_replace'),
+    ];
+
     $element['open'] = [
       '#title' => $this->t('Show widget details as open by default'),
       '#description' => $this->t('If marked, the fieldset container that wraps the browser on the entity form will be loaded initially expanded.'),
@@ -426,17 +433,19 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
     // entities are added or by one of the "Remove" buttons. Depending on that
     // we need to figure out where root of the widget is in the form structure
     // and use this information to return correct part of the form.
+    $parents = [];
     if (!empty($trigger['#ajax']['event']) && $trigger['#ajax']['event'] == 'entity_browser_value_updated') {
       $parents = array_slice($trigger['#array_parents'], 0, -1);
     }
     elseif ($trigger['#type'] == 'submit' && strpos($trigger['#name'], '_remove_')) {
       $parents = array_slice($trigger['#array_parents'], 0, -static::$deleteDepth);
-      if (!empty($trigger['#attributes']['class']) && in_array('replace-button', $trigger['#attributes']['class'])) {
-        // We need to re-open the browser. Instead of just passing "TRUE", send
-        // to the JS the unique part of the button's name that needs to be
-        // clicked.
-        $reopen_browser = $parents[0];
-      }
+    }
+    elseif ($trigger['#type'] == 'submit' && strpos($trigger['#name'], '_replace_')) {
+      $parents = array_slice($trigger['#array_parents'], 0, -static::$deleteDepth);
+      // We need to re-open the browser. Instead of just passing "TRUE", send
+      // to the JS the unique part of the button's name that needs to be clicked
+      // on to relaunch the browser.
+      $reopen_browser = $parents[0];
     }
 
     $parents = NestedArray::getValue($form, $parents);
@@ -514,18 +523,15 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
       $classes[] = 'sortable';
     }
 
-    // The "Remove" button doesn't make sense when the field is required and
-    // cardinality === 1. We use a "Replace" button instead in those cases,
-    // which will clean the current selection and re-open the browser.
-    $cardinality = $this->fieldDefinition->getFieldStorageDefinition()->getCardinality();
-    $field_required = $this->fieldDefinition->isRequired();
-    $show_replace = ($field_required && ($cardinality === 1));
+    // The "Replace" button will only be shown if this setting is enabled in the
+    // widget, and there is only one entity in the current selection.
+    $replace_button_access = $this->getSetting('field_widget_replace') && (count($entities) === 1);
 
     return [
       '#theme_wrappers' => ['container'],
       '#attributes' => ['class' => $classes],
       'items' => array_map(
-        function (ContentEntityInterface $entity, $row_id) use ($field_widget_display, $details_id, $field_parents, $show_replace) {
+        function (ContentEntityInterface $entity, $row_id) use ($field_widget_display, $details_id, $field_parents, $replace_button_access) {
           $display = $field_widget_display->view($entity);
           $edit_button_access = $this->getSetting('field_widget_edit') && $entity->access('update', $this->currentUser);
           if ($entity->getEntityTypeId() == 'file') {
@@ -547,7 +553,7 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
             'display' => $display,
             'remove_button' => [
               '#type' => 'submit',
-              '#value' => $show_replace ? $this->t('Replace') : $this->t('Remove'),
+              '#value' => $this->t('Remove'),
               '#ajax' => [
                 'callback' => [get_class($this), 'updateWidgetCallback'],
                 'wrapper' => $details_id,
@@ -558,10 +564,27 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
               '#attributes' => [
                 'data-entity-id' => $entity->getEntityTypeId() . ':' . $entity->id(),
                 'data-row-id' => $row_id,
-                'class' => [$show_replace ? 'replace-button' : 'remove-button'],
+                'class' => ['remove-button'],
               ],
               '#access' => (bool) $this->getSetting('field_widget_remove'),
             ],
+            'replace_button' => [
+              '#type' => 'submit',
+              '#value' => $this->t('Replace'),
+              '#ajax' => [
+                'callback' => [get_class($this), 'updateWidgetCallback'],
+                'wrapper' => $details_id,
+              ],
+              '#submit' => [[get_class($this), 'removeItemSubmit']],
+              '#name' => $this->fieldDefinition->getName() . '_replace_' . $entity->id() . '_' . $row_id . '_' . md5(json_encode($field_parents)),
+              '#limit_validation_errors' => [array_merge($field_parents, [$this->fieldDefinition->getName()])],
+              '#attributes' => [
+                'data-entity-id' => $entity->getEntityTypeId() . ':' . $entity->id(),
+                'data-row-id' => $row_id,
+                'class' => ['replace-button'],
+              ],
+              '#access' => $replace_button_access,
+            ],
             'edit_button' => [
               '#type' => 'submit',
               '#value' => $this->t('Edit'),
@@ -578,6 +601,9 @@ class EntityReferenceBrowserWidget extends WidgetBase implements ContainerFactor
                   ],
                 ],
               ],
+              '#attributes' => [
+                'class' => ['edit-button'],
+              ],
               '#access' => $edit_button_access,
             ],
           ];
diff --git a/src/Plugin/Field/FieldWidget/FileBrowserWidget.php b/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
index 6db9c8d..f9eea20 100644
--- a/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
+++ b/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
@@ -253,6 +253,10 @@ class FileBrowserWidget extends EntityReferenceBrowserWidget {
         $edit_button_access = $can_edit && $entity->access('update', $this->currentUser);
       }
 
+      // The "Replace" button will only be shown if this setting is enabled in
+      // the widget, and there is only one entity in the current selection.
+      $replace_button_access = $this->getSetting('field_widget_replace') && (count($entities) === 1);
+
       $entity_id = $entity->id();
 
       // Find the default description.
@@ -356,9 +360,27 @@ class FileBrowserWidget extends EntityReferenceBrowserWidget {
           '#attributes' => [
             'data-entity-id' => $entity->getEntityTypeId() . ':' . $entity->id(),
             'data-row-id' => $delta,
+            'class' => ['edit-button'],
           ],
           '#access' => $edit_button_access,
         ],
+        'replace_button' => [
+          '#type' => 'submit',
+          '#value' => $this->t('Replace'),
+          '#ajax' => [
+            'callback' => [get_class($this), 'updateWidgetCallback'],
+            'wrapper' => $details_id,
+          ],
+          '#submit' => [[get_class($this), 'removeItemSubmit']],
+          '#name' => $field_machine_name . '_replace_' . $entity_id . '_' . md5(json_encode($field_parents)),
+          '#limit_validation_errors' => [array_merge($field_parents, [$field_machine_name, 'target_id'])],
+          '#attributes' => [
+            'data-entity-id' => $entity->getEntityTypeId() . ':' . $entity->id(),
+            'data-row-id' => $delta,
+            'class' => ['replace-button'],
+          ],
+          '#access' => $replace_button_access,
+        ],
         'remove_button' => [
           '#type' => 'submit',
           '#value' => $this->t('Remove'),
@@ -372,6 +394,7 @@ class FileBrowserWidget extends EntityReferenceBrowserWidget {
           '#attributes' => [
             'data-entity-id' => $entity->getEntityTypeId() . ':' . $entity->id(),
             'data-row-id' => $delta,
+            'class' => ['remove-button'],
           ],
           '#access' => (bool) $widget_settings['field_widget_remove'],
         ],
diff --git a/tests/modules/entity_browser_ief_test/config/install/core.entity_form_display.media.ief_media_bundle.default.yml b/tests/modules/entity_browser_ief_test/config/install/core.entity_form_display.media.ief_media_bundle.default.yml
index 6288723..f2ab144 100644
--- a/tests/modules/entity_browser_ief_test/config/install/core.entity_form_display.media.ief_media_bundle.default.yml
+++ b/tests/modules/entity_browser_ief_test/config/install/core.entity_form_display.media.ief_media_bundle.default.yml
@@ -18,6 +18,7 @@ content:
       field_widget_display: label
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
       open: true
       selection_mode: selection_edit
       field_widget_display_settings: {  }
diff --git a/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.content_embed.default.yml b/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.content_embed.default.yml
index 5732c92..d09edbb 100644
--- a/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.content_embed.default.yml
+++ b/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.content_embed.default.yml
@@ -20,6 +20,7 @@ content:
       field_widget_display: label
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
       selection_mode: selection_append
       open: true
       field_widget_display_settings: {  }
diff --git a/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.nested_paragrah.default.yml b/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.nested_paragrah.default.yml
index 50848f7..2a817a9 100644
--- a/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.nested_paragrah.default.yml
+++ b/tests/modules/entity_browser_test_paragraphs/config/install/core.entity_form_display.paragraph.nested_paragrah.default.yml
@@ -19,6 +19,7 @@ content:
       field_widget_display: label
       field_widget_edit: true
       field_widget_remove: true
+      field_widget_replace: false
       selection_mode: selection_append
       open: true
       field_widget_display_settings: {  }
diff --git a/tests/src/FunctionalJavascript/EntityReferenceWidgetTest.php b/tests/src/FunctionalJavascript/EntityReferenceWidgetTest.php
index d88825d..0a412b1 100644
--- a/tests/src/FunctionalJavascript/EntityReferenceWidgetTest.php
+++ b/tests/src/FunctionalJavascript/EntityReferenceWidgetTest.php
@@ -73,6 +73,7 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
         'open' => TRUE,
         'field_widget_edit' => TRUE,
         'field_widget_remove' => TRUE,
+        'field_widget_replace' => FALSE,
         'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
         'field_widget_display' => 'label',
         'field_widget_display_settings' => [],
@@ -115,6 +116,7 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
         'open' => TRUE,
         'field_widget_edit' => FALSE,
         'field_widget_remove' => FALSE,
+        'field_widget_replace' => FALSE,
         'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
         'field_widget_display' => 'label',
         'field_widget_display_settings' => [],
@@ -132,6 +134,7 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
         'open' => TRUE,
         'field_widget_edit' => TRUE,
         'field_widget_remove' => TRUE,
+        'field_widget_replace' => FALSE,
         'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
         'field_widget_display' => 'label',
         'field_widget_display_settings' => [],
@@ -141,19 +144,31 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
     $remove_button = $assert_session->buttonExists('edit-field-entity-reference1-current-items-0-remove-button');
     $this->assertEquals('Remove', $remove_button->getValue());
     $this->assertTrue($remove_button->hasClass('remove-button'));
-    $this->assertFalse($remove_button->hasClass('replace-button'));
     $edit_button = $assert_session->buttonExists('edit-field-entity-reference1-current-items-0-edit-button');
     $this->assertEquals('Edit', $edit_button->getValue());
+    $this->assertTrue($edit_button->hasClass('edit-button'));
+    // Make sure the "Replace" button is not there.
+    $assert_session->buttonNotExists('edit-field-entity-reference1-current-items-0-replace-button');
 
     // Test the "Remove" button on the widget works.
     $page->pressButton('Remove');
     $this->waitForAjaxToFinish();
     $assert_session->pageTextNotContains('Target example node 1');
 
-    // Change the field cardinality to 1 and make it required, so we can check
-    // that the "Remove" button shows "Replace" instead.
-    $field_storage->setCardinality(1)->save();
-    $field->setRequired(TRUE)->save();
+    // Test the "Replace" button functionality.
+    $form_display->setComponent('field_entity_reference1', [
+      'type' => 'entity_browser_entity_reference',
+      'settings' => [
+        'entity_browser' => 'test_entity_browser_iframe_node_view',
+        'open' => TRUE,
+        'field_widget_edit' => TRUE,
+        'field_widget_remove' => TRUE,
+        'field_widget_replace' => TRUE,
+        'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
+        'field_widget_display' => 'label',
+        'field_widget_display_settings' => [],
+      ],
+    ])->save();
     // We'll need a third node to be able to make a new selection.
     $target_node2 = Node::create([
       'title' => 'Target example node 2',
@@ -161,10 +176,11 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
     ]);
     $target_node2->save();
     $this->drupalGet('node/' . $nid . '/edit');
-    $remove_button = $assert_session->buttonExists('edit-field-entity-reference1-current-items-0-remove-button');
-    $this->assertEquals('Replace', $remove_button->getValue());
-    $this->assertTrue($remove_button->hasClass('replace-button'));
-    $this->assertFalse($remove_button->hasClass('remove-button'));
+    // If there is only one entity in the current selection the button should
+    // show up.
+    $replace_button = $assert_session->buttonExists('edit-field-entity-reference1-current-items-0-replace-button');
+    $this->assertEquals('Replace', $replace_button->getValue());
+    $this->assertTrue($replace_button->hasClass('replace-button'));
     // Clicking on the button should empty the selection and automatically
     // open the browser again.
     $remove_button->click();
@@ -175,25 +191,20 @@ class EntityReferenceWidgetTest extends EntityBrowserJavascriptTestBase {
     $page->pressButton('Select entities');
     $session->switchToIFrame();
     $this->waitForAjaxToFinish();
+    // Even in the AJAX-built markup for the newly selected element, the replace
+    // button should be there.
+    $assert_session->elementExists('css', 'input[data-drupal-selector="edit-field-entity-reference1-current-items-0-replace-button"]');
+    // Adding a new node to the selection, however, should make it disappear.
+    $session->switchToIFrame('entity_browser_iframe_test_entity_browser_iframe_node_view');
+    $this->waitForAjaxToFinish();
+    $page->checkField('edit-entity-browser-select-node1');
+    $page->pressButton('Select entities');
+    $session->switchToIFrame();
+    $this->waitForAjaxToFinish();
+    $assert_session->elementNotExists('css', 'input[data-drupal-selector="edit-field-entity-reference1-current-items-0-replace-button"]');
     $page->pressButton('Save');
     $assert_session->pageTextContains('Article Referencing node 1 has been updated.');
 
-    // Go to the widget config settings and check that the "Remove" button
-    // option has the correct help text.
-    $this->drupalGet('/admin/structure/types/manage/article/form-display');
-    $assert_session->elementExists('css', '#edit-fields-field-entity-reference1-settings-edit')->click();
-    $this->waitForAjaxToFinish();
-    $assert_session->elementContains(
-      'css',
-      '.ajax-new-content .form-item-fields-field-entity-reference1-settings-edit-form-settings-field-widget-remove label',
-      'Display Remove / Replace button'
-    );
-    $assert_session->elementContains(
-      'css',
-      '.ajax-new-content .form-item-fields-field-entity-reference1-settings-edit-form-settings-field-widget-remove .description',
-      'In required single-valued fields, this button will show "Replace" instead of the default "Remove". In that scenario, once clicked the current selection will be emptied and the browser will be opened to allow a new selection to take place.'
-    );
-
     // Verify that if the user cannot edit the entity, the "Edit" button does
     // not show up, even if configured to.
     /** @var \Drupal\user\RoleInterface $role */
diff --git a/tests/src/FunctionalJavascript/ImageFieldTest.php b/tests/src/FunctionalJavascript/ImageFieldTest.php
index a08781a..ea1e223 100644
--- a/tests/src/FunctionalJavascript/ImageFieldTest.php
+++ b/tests/src/FunctionalJavascript/ImageFieldTest.php
@@ -70,6 +70,7 @@ class ImageFieldTest extends EntityBrowserJavascriptTestBase {
         'open' => TRUE,
         'field_widget_edit' => FALSE,
         'field_widget_remove' => TRUE,
+        'field_widget_replace' => FALSE,
         'selection_mode' => EntityBrowserElement::SELECTION_MODE_APPEND,
         'view_mode' => 'default',
         'preview_image_style' => 'thumbnail',