diff --git a/core/misc/icons/ee0000/ex.svg b/core/misc/icons/ee0000/ex.svg
new file mode 100644
index 0000000000..6b45a1d572
--- /dev/null
+++ b/core/misc/icons/ee0000/ex.svg
@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16"><path fill="#ee0000" d="M3.51 13.925c.194.194.512.195.706.001l3.432-3.431c.194-.194.514-.194.708 0l3.432 3.431c.192.194.514.193.707-.001l1.405-1.417c.191-.195.189-.514-.002-.709l-3.397-3.4c-.192-.193-.192-.514-.002-.708l3.401-3.43c.189-.195.189-.515 0-.709l-1.407-1.418c-.195-.195-.513-.195-.707-.001l-3.43 3.431c-.195.194-.516.194-.708 0l-3.432-3.431c-.195-.195-.512-.194-.706.001l-1.407 1.417c-.194.195-.194.515 0 .71l3.403 3.429c.193.195.193.514-.001.708l-3.4 3.399c-.194.195-.195.516-.001.709l1.406 1.419z"/></svg>
diff --git a/core/modules/media_library/css/media_library.theme.css b/core/modules/media_library/css/media_library.theme.css
index 2bad77cefb..197c3f0bd3 100644
--- a/core/modules/media_library/css/media_library.theme.css
+++ b/core/modules/media_library/css/media_library.theme.css
@@ -84,17 +84,31 @@
 }
 
 /* Generic media add form styles. */
-.media-library-add-form--without-input {
-  padding: 16px;
-  border: 1px solid #bfbfbf;
-  border-radius: 2px;
-  background: #fcfcfa;
+.media-library-add-form-wrapper {
+  outline: none;
 }
 
 .media-library-add-form--without-input .form-item {
   margin: 0 0 1em;
 }
 
+/**
+ * Remove outline from added media container.
+ *
+ * The added media container receives focus after adding new media, but doesn't
+ * need the outline.
+ */
+.media-library-add-form__added-media {
+  outline: none;
+}
+
+.media-library-add-form__input-wrapper {
+  padding: 16px;
+  border: 1px solid #bfbfbf;
+  border-radius: 2px;
+  background: #fcfcfa;
+}
+
 /* Style the media add upload form. */
 .media-library-add-form--upload.media-library-add-form--without-input .form-item-upload {
   margin-bottom: 0;
@@ -105,12 +119,12 @@
 }
 
 /* Style the media add oEmbed form. */
-.media-library-add-form-oembed-wrapper {
+.media-library-add-form--oembed .media-library-add-form__input-wrapper {
   display: flex;
 }
 
 @media screen and (max-width: 37.5em) {
-  .media-library-add-form-oembed-wrapper {
+  .media-library-add-form--oembed .media-library-add-form__input-wrapper {
     display: block;
   }
 }
@@ -466,6 +480,7 @@
 
 /* Style the wrappers around new media and files. */
 .media-library-add-form__media {
+  position: relative;
   display: flex;
   padding: 20px 0 20px 0;
   border-bottom: 1px solid #c0c0c0;
@@ -476,6 +491,12 @@
   padding-top: 0;
 }
 
+/* The first item doesn't have a top padding, change the location of the remove
+   button as well. */
+.media-library-add-form__media:first-child .media-library-add-form__remove-button[type="submit"] {
+  top: 5px;
+}
+
 /* Do not show the bottom border and padding for the last item. */
 .media-library-add-form__media:last-child {
   padding-bottom: 0;
@@ -504,6 +525,57 @@
   margin-left: 20px;
 }
 
+/* @todo Remove [type="submit"] when styles are moved to the seven theme in
+     https://www.drupal.org/project/drupal/issues/2980769 */
+.media-library-add-form__remove-button[type="submit"] {
+  position: absolute;
+  top: 25px;
+  right: 6px; /* LTR */
+  width: auto;
+  margin: 0;
+  padding: 2px 20px 2px 2px; /* LTR */
+  text-transform: lowercase;
+  color: transparent;
+  border: 0;
+  border-radius: 0;
+  background: transparent url(../../../misc/icons/787878/ex.svg) right 2px no-repeat; /* LTR */
+  font-weight: normal;
+  line-height: 16px;
+}
+[dir="rtl"] .media-library-add-form__remove-button[type="submit"] {
+  right: auto;
+  left: 13px;
+  padding: 2px 2px 2px 20px;
+  background-position: left 2px;
+}
+
+.media-library-add-form__remove-button:focus,
+.media-library-add-form__remove-button.button:disabled,
+.media-library-add-form__remove-button.button:disabled:active,
+.media-library-add-form__remove-button.button:focus {
+  color: #787878;
+  border: 0;
+  background: transparent url(../../../misc/icons/787878/ex.svg) right 2px no-repeat; /* LTR */
+}
+[dir="rtl"] .media-library-add-form__remove-button:focus,
+[dir="rtl"] .media-library-add-form__remove-button.button:disabled,
+[dir="rtl"] .media-library-add-form__remove-button.button:disabled:active,
+[dir="rtl"] .media-library-add-form__remove-button.button:focus {
+  background-position: left 2px;
+}
+
+.media-library-add-form__remove-button:hover,
+.media-library-add-form__remove-button.button:hover {
+  color: #e00;
+  border: 0;
+  background: transparent url(../../../misc/icons/ee0000/ex.svg) right 2px no-repeat; /* LTR */
+  box-shadow: none;
+}
+[dir="rtl"] .media-library-add-form__remove-button:hover,
+[dir="rtl"] .media-library-add-form__remove-button.button:hover {
+  background-position: left 2px;
+}
+
 /* @todo Remove or re-work in https://www.drupal.org/node/2985168 */
 .media-library-widget .media-library-item__name a,
 .media-library-view--widget .media-library-item__name a {
diff --git a/core/modules/media_library/src/Form/AddFormBase.php b/core/modules/media_library/src/Form/AddFormBase.php
index d6e76a5a03..7391879582 100644
--- a/core/modules/media_library/src/Form/AddFormBase.php
+++ b/core/modules/media_library/src/Form/AddFormBase.php
@@ -3,6 +3,7 @@
 namespace Drupal\media_library\Form;
 
 use Drupal\Core\Ajax\AjaxResponse;
+use Drupal\Core\Ajax\InvokeCommand;
 use Drupal\Core\Ajax\ReplaceCommand;
 use Drupal\Core\Entity\Entity\EntityFormDisplay;
 use Drupal\Core\Entity\EntityStorageInterface;
@@ -109,7 +110,9 @@ protected function getMediaType(FormStateInterface $form_state) {
    * {@inheritdoc}
    */
   public function buildForm(array $form, FormStateInterface $form_state) {
-    $form['#prefix'] = '<div id="media-library-add-form-wrapper">';
+    // @todo Remove the ID when we can use selectors to replace content via
+    //   AJAX in https://www.drupal.org/project/drupal/issues/2821793.
+    $form['#prefix'] = '<div id="media-library-add-form-wrapper" class="media-library-add-form-wrapper">';
     $form['#suffix'] = '</div>';
     $form['#attached']['library'][] = 'media_library/style';
 
@@ -139,6 +142,15 @@ public function buildForm(array $form, FormStateInterface $form_state) {
 
       $form['media'] = [
         '#type' => 'container',
+        '#attributes' => [
+          'class' => [
+            'media-library-add-form__added-media',
+          ],
+          // Add the tabindex '-1' to allow the focus to be shifted to the
+          // container.
+          'tabindex' => -1,
+          'aria-label' => $this->t('Added media items'),
+        ],
       ];
 
       foreach ($added_media as $delta => $media) {
@@ -185,6 +197,14 @@ public function buildForm(array $form, FormStateInterface $form_state) {
    *   The element containing the required fields sub-form.
    */
   protected function buildEntityFormElement(MediaInterface $media, array $form, FormStateInterface $form_state, $delta) {
+    // We need to make sure each button has a unique name attribute. The default
+    // name for button elements is 'op'. If the name is not unique, the
+    // triggering element is not set correctly and the wrong media item is
+    // removed.
+    // @see ::removeButtonSubmit()
+    $parents = $form['#parents'];
+    $id_suffix = $parents ? '-' . implode('-', $parents) : '';
+
     $element = [
       '#type' => 'container',
       '#attributes' => [
@@ -194,6 +214,7 @@ protected function buildEntityFormElement(MediaInterface $media, array $form, Fo
       ],
       'preview' => [
         '#type' => 'container',
+        '#weight' => 10,
         '#attributes' => [
           'class' => [
             'media-library-add-form__preview',
@@ -202,6 +223,7 @@ protected function buildEntityFormElement(MediaInterface $media, array $form, Fo
       ],
       'fields' => [
         '#type' => 'container',
+        '#weight' => 20,
         '#attributes' => [
           'class' => [
             'media-library-add-form__fields',
@@ -211,6 +233,24 @@ protected function buildEntityFormElement(MediaInterface $media, array $form, Fo
         // to build the entity form fields.
         '#parents' => ['media', $delta, 'fields'],
       ],
+      'remove_button' => [
+        '#type' => 'submit',
+        '#value' => $this->t('Remove'),
+        '#name' => 'media-' . $delta . '-remove-button' . $id_suffix,
+        '#weight' => 30,
+        '#attributes' => [
+          'class' => ['media-library-add-form__remove-button'],
+          'aria-label' => $this->t('Remove @label', ['@label' => $media->getName()]),
+        ],
+        '#ajax' => [
+          'callback' => '::updateFormCallback',
+          'wrapper' => 'media-library-add-form-wrapper',
+          'message' => $this->t('Removing @label.', ['@label' => $media->getName()]),
+        ],
+        '#submit' => ['::removeButtonSubmit'],
+        // Ensure errors in other media items do not prevent removal.
+        '#limit_validation_errors' => [],
+      ],
     ];
     // @todo Make the image style configurable in
     //   https://www.drupal.org/node/2988223
@@ -295,7 +335,8 @@ protected function processInputValues(array $source_field_values, array $form, F
     $media = array_map(function ($source_field_value) use ($media_type, $media_storage, $source_field_name) {
       return $this->createMediaFromValue($media_type, $media_storage, $source_field_name, $source_field_value);
     }, $source_field_values);
-    $form_state->set('media', $media)->setRebuild();
+    // Re-key the media items before setting them in the form state.
+    $form_state->set('media', array_values($media))->setRebuild();
   }
 
   /**
@@ -332,6 +373,33 @@ protected function prepareMediaEntityForSave(MediaInterface $media) {
     // Intentionally empty by default.
   }
 
+  /**
+   * Submit handler for the remove button.
+   *
+   * @param array $form
+   *   The form render array.
+   * @param \Drupal\Core\Form\FormStateInterface $form_state
+   *   The form state.
+   */
+  public function removeButtonSubmit(array $form, FormStateInterface $form_state) {
+    // Retrieve the delta of the media item from the parents of the remove
+    // button.
+    $triggering_element = $form_state->getTriggeringElement();
+    $delta = array_slice($triggering_element['#array_parents'], -2, 1)[0];
+
+    $added_media = $form_state->get('media');
+    $removed_media = $added_media[$delta];
+
+    // Update the list of added media items in the form state.
+    unset($added_media[$delta]);
+
+    // Re-key the media items before setting them in the form state.
+    $form_state->set('media', array_values($added_media))->setRebuild();
+
+    // Show a message to the user to confirm the media is removed.
+    $this->messenger()->addStatus($this->t('The media item %label has been removed.', ['%label' => $removed_media->label()]));
+  }
+
   /**
    * AJAX callback to update the entire form based on source field input.
    *
@@ -344,17 +412,43 @@ protected function prepareMediaEntityForSave(MediaInterface $media) {
    *   The form render array or an AJAX response object.
    */
   public function updateFormCallback(array &$form, FormStateInterface $form_state) {
+    $triggering_element = $form_state->getTriggeringElement();
+    $wrapper_id = $triggering_element['#ajax']['wrapper'];
+    $added_media = $form_state->get('media');
+
+    $response = new AjaxResponse();
+
     // When the source field input contains errors, replace the existing form to
     // let the user change the source field input. If the user input is valid,
     // the entire modal is replaced with the second step of the form to show the
     // form fields for each media item.
     if ($form_state::hasAnyErrors()) {
-      $response = new AjaxResponse();
       $response->addCommand(new ReplaceCommand('#media-library-add-form-wrapper', $form));
       return $response;
     }
 
-    return $form;
+    // Check if the remove button is clicked.
+    if (end($triggering_element['#parents']) === 'remove_button') {
+      // When the list of added media is empty, return to the media library and
+      // shift focus back to the first tabbable element (which should be the
+      // source field).
+      if (empty($added_media)) {
+        $response->addCommand(new ReplaceCommand('#media-library-add-form-wrapper', $this->buildMediaLibraryUi($form_state)));
+        $response->addCommand(new InvokeCommand('#media-library-add-form-wrapper :tabbable', 'focus'));
+      }
+      // When there are still more items, the AJAX system automatically sets
+      // focus to the next remove button.
+      else {
+        $response->addCommand(new ReplaceCommand("#$wrapper_id", $form));
+      }
+    }
+    // Update the form and shift focus to the added media items.
+    else {
+      $response->addCommand(new ReplaceCommand("#$wrapper_id", $form));
+      $response->addCommand(new InvokeCommand('.media-library-add-form__added-media', 'focus'));
+    }
+
+    return $response;
   }
 
   /**
@@ -420,6 +514,22 @@ public function updateWidget(array &$form, FormStateInterface $form_state) {
       return $media->id();
     }, $added_media);
 
+    $response = new AjaxResponse();
+    $response->addCommand(new UpdateSelectionCommand($media_ids));
+    $response->addCommand(new ReplaceCommand('#media-library-add-form-wrapper', $this->buildMediaLibraryUi($form_state)));
+    return $response;
+  }
+
+  /**
+   * Build the render array of the media library UI.
+   *
+   * @param \Drupal\Core\Form\FormStateInterface $form_state
+   *   The current form state.
+   *
+   * @return array
+   *   The render array for the media library.
+   */
+  protected function buildMediaLibraryUi(FormStateInterface $form_state) {
     // Get the render array for the media library. The media library state might
     // contain the 'media_library_content' when it has been opened from a
     // vertical tab. We need to remove that to make sure the render array
@@ -429,12 +539,7 @@ public function updateWidget(array &$form, FormStateInterface $form_state) {
     $state = $this->getMediaLibraryState($form_state);
     $state->remove('media_library_content');
     $state->set('_media_library_form_rebuild', TRUE);
-    $library_ui = $this->libraryUiBuilder->buildUi($state);
-
-    $response = new AjaxResponse();
-    $response->addCommand(new UpdateSelectionCommand($media_ids));
-    $response->addCommand(new ReplaceCommand('#media-library-add-form-wrapper', $library_ui));
-    return $response;
+    return $this->libraryUiBuilder->buildUi($state);
   }
 
   /**
diff --git a/core/modules/media_library/src/Form/FileUploadForm.php b/core/modules/media_library/src/Form/FileUploadForm.php
index 8614a2e2d8..283ae74e97 100644
--- a/core/modules/media_library/src/Form/FileUploadForm.php
+++ b/core/modules/media_library/src/Form/FileUploadForm.php
@@ -122,8 +122,16 @@ protected function buildInputElement(array $form, FormStateInterface $form_state
 
     $slots = $state->getAvailableSlots();
 
+    // Add a container to group the input elements for styling purposes.
+    $form['container'] = [
+      '#type' => 'container',
+      '#attributes' => [
+        'class' => ['media-library-add-form__input-wrapper'],
+      ],
+    ];
+
     $process = (array) $this->elementInfo->getInfoProperty('managed_file', '#process', []);
-    $form['upload'] = [
+    $form['container']['upload'] = [
       '#type' => 'managed_file',
       '#title' => $this->formatPlural($slots, 'Add file', 'Add files'),
       // @todo Move validation in https://www.drupal.org/node/2988215
@@ -136,7 +144,7 @@ protected function buildInputElement(array $form, FormStateInterface $form_state
 
     $file_upload_help = [
       '#theme' => 'file_upload_help',
-      '#upload_validators' => $form['upload']['#upload_validators'],
+      '#upload_validators' => $form['container']['upload']['#upload_validators'],
       '#cardinality' => $slots,
     ];
 
@@ -145,7 +153,7 @@ protected function buildInputElement(array $form, FormStateInterface $form_state
     // upload help in the same way, so any theming improvements made to file
     // fields would also be applied to this upload field.
     // @see \Drupal\file\Plugin\Field\FieldWidget\FileWidget::formElement()
-    $form['upload']['#description'] = $this->renderer->renderPlain($file_upload_help);
+    $form['container']['upload']['#description'] = $this->renderer->renderPlain($file_upload_help);
 
     return $form;
   }
diff --git a/core/modules/media_library/src/Form/OEmbedForm.php b/core/modules/media_library/src/Form/OEmbedForm.php
index 72b2b1c547..4af94334c0 100644
--- a/core/modules/media_library/src/Form/OEmbedForm.php
+++ b/core/modules/media_library/src/Form/OEmbedForm.php
@@ -91,10 +91,11 @@ protected function buildInputElement(array $form, FormStateInterface $form_state
     $media_type = $this->getMediaType($form_state);
     $providers = $media_type->getSource()->getProviders();
 
+    // Add a container to group the input elements for styling purposes.
     $form['container'] = [
       '#type' => 'container',
       '#attributes' => [
-        'class' => ['media-library-add-form-oembed-wrapper'],
+        'class' => ['media-library-add-form__input-wrapper'],
       ],
     ];
 
diff --git a/core/modules/media_library/tests/src/FunctionalJavascript/MediaLibraryTest.php b/core/modules/media_library/tests/src/FunctionalJavascript/MediaLibraryTest.php
index 16db27bfd8..7fbc75b2ce 100644
--- a/core/modules/media_library/tests/src/FunctionalJavascript/MediaLibraryTest.php
+++ b/core/modules/media_library/tests/src/FunctionalJavascript/MediaLibraryTest.php
@@ -822,6 +822,32 @@ public function testWidgetUpload() {
     $assert_session->assertWaitOnAjaxRequest();
     $assert_session->pageTextNotContains('Add or select media');
     $assert_session->pageTextContains($jpg_image->filename);
+
+    // Assert removing an uploaded media item before save works as expected.
+    $assert_session->elementExists('css', '.media-library-open-button[name^="field_unlimited_media"]')->click();
+    $assert_session->assertWaitOnAjaxRequest();
+    $assert_session->pageTextContains('Add or select media');
+    $page->clickLink('Type Three');
+    $assert_session->assertWaitOnAjaxRequest();
+    $page->attachFileToField('Add files', $this->container->get('file_system')->realpath($png_image->uri));
+    $assert_session->assertWaitOnAjaxRequest();
+    // Assert the focus is shifted to the added media items.
+    $this->assertJsCondition('jQuery(".media-library-add-form__added-media").is(":focus")');
+    // Assert the media item fields are shown and the vertical tabs are no
+    // longer shown.
+    $assert_session->elementExists('css', '.media-library-add-form__fields');
+    $assert_session->elementNotExists('css', '.media-library-menu');
+    // Press the 'Remove button' and assert the user is sent back to the media
+    // library.
+    $assert_session->elementExists('css', '.media-library-add-form__remove-button')->click();
+    $assert_session->assertWaitOnAjaxRequest();
+    // Assert the remove message is shown.
+    $assert_session->pageTextContains("The media item $png_image->filename has been removed.");
+    // Assert the focus is shifted to the first tabbable element of the add
+    // form, which should be the source field.
+    $this->assertJsCondition('jQuery("#media-library-add-form-wrapper :tabbable").is(":focus")');
+    $assert_session->elementNotExists('css', '.media-library-add-form__fields');
+    $assert_session->elementExists('css', '.media-library-menu');
   }
 
   /**
@@ -921,6 +947,33 @@ public function testWidgetOEmbed() {
     $assert_session->assertWaitOnAjaxRequest();
     $assert_session->pageTextNotContains('Add or select media');
     $assert_session->pageTextContains('Custom video title');
+
+    // Assert removing an added oEmbed media item before save works as expected.
+    $assert_session->elementExists('css', '.media-library-open-button[name^="field_unlimited_media"]')->click();
+    $assert_session->assertWaitOnAjaxRequest();
+    $assert_session->pageTextContains('Add or select media');
+    $page->clickLink('Type Five');
+    $assert_session->assertWaitOnAjaxRequest();
+    $page->fillField('Add Type Five via URL', $video_url);
+    $page->pressButton('Add');
+    $assert_session->assertWaitOnAjaxRequest();
+    // Assert the focus is shifted to the added media items.
+    $this->assertJsCondition('jQuery(".media-library-add-form__added-media").is(":focus")');
+    // Assert the media item fields are shown and the vertical tabs are no
+    // longer shown.
+    $assert_session->elementExists('css', '.media-library-add-form__fields');
+    $assert_session->elementNotExists('css', '.media-library-menu');
+    // Press the 'Remove button' and assert the user is sent back to the media
+    // library.
+    $assert_session->elementExists('css', '.media-library-add-form__remove-button')->click();
+    $assert_session->assertWaitOnAjaxRequest();
+    // Assert the remove message is shown.
+    $assert_session->pageTextContains("The media item $video_title has been removed.");
+    // Assert the focus is shifted to the first tabbable element of the add
+    // form, which should be the source field.
+    $this->assertJsCondition('jQuery("#media-library-add-form-wrapper :tabbable").is(":focus")');
+    $assert_session->elementNotExists('css', '.media-library-add-form__fields');
+    $assert_session->elementExists('css', '.media-library-menu');
   }
 
 }
diff --git a/core/modules/media_library/tests/src/Kernel/MediaLibraryAddFormTest.php b/core/modules/media_library/tests/src/Kernel/MediaLibraryAddFormTest.php
index 0d180bb880..833ede9a65 100644
--- a/core/modules/media_library/tests/src/Kernel/MediaLibraryAddFormTest.php
+++ b/core/modules/media_library/tests/src/Kernel/MediaLibraryAddFormTest.php
@@ -87,7 +87,7 @@ public function testMediaTypeAddForm() {
     ]));
     // Assert the media library UI only contains the add form for the image
     // media type.
-    $this->assertSame('managed_file', $this->buildLibraryUi('image')['content']['form']['upload']['#type']);
+    $this->assertSame('managed_file', $this->buildLibraryUi('image')['content']['form']['container']['upload']['#type']);
     $this->assertEmpty($this->buildLibraryUi('remote_video')['content']['form']);
 
     // Create a user that has access to create both media types.
@@ -97,7 +97,7 @@ public function testMediaTypeAddForm() {
     ]));
     // Assert the media library UI only contains the add form for both media
     // types.
-    $this->assertSame('managed_file', $this->buildLibraryUi('image')['content']['form']['upload']['#type']);
+    $this->assertSame('managed_file', $this->buildLibraryUi('image')['content']['form']['container']['upload']['#type']);
     $this->assertSame('url', $this->buildLibraryUi('remote_video')['content']['form']['container']['url']['#type']);
   }
 
