diff --git a/src/Plugin/Field/FieldWidget/FileBrowserWidget.php b/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
index e57dbc7..ee46ad2 100644
--- a/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
+++ b/src/Plugin/Field/FieldWidget/FileBrowserWidget.php
@@ -15,6 +15,7 @@ use Drupal\Core\Url;
 use Drupal\entity_browser\FieldWidgetDisplayManager;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\Validator\ConstraintViolationListInterface;
 
 /**
  * Entity browser file widget.
@@ -362,7 +363,7 @@ class FileBrowserWidget extends EntityReferenceBrowserWidget {
           ],
           '#submit' => [[get_class($this), 'removeItemSubmit']],
           '#name' => $field_machine_name . '_remove_' . $entity_id,
-          '#limit_validation_errors' => [],
+          '#limit_validation_errors' => [array_merge($field_parents, [$field_machine_name])],
           '#attributes' => ['data-entity-id' => $entity_id],
           '#access' => (bool) $widget_settings['field_widget_remove'],
         ],
@@ -480,4 +481,14 @@ class FileBrowserWidget extends EntityReferenceBrowserWidget {
     return $data;
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public function flagErrors(FieldItemListInterface $items, ConstraintViolationListInterface $violations, array $form, FormStateInterface $form_state) {
+    $clicked_button = end($form_state->getTriggeringElement()['#parents']);
+    if ($clicked_button !== 'remove_button') {
+      parent::flagErrors($items, $violations, $form, $form_state);
+    }
+  }
+
 }