diff -u b/src/DiffBuilderManager.php b/src/DiffBuilderManager.php
--- b/src/DiffBuilderManager.php
+++ b/src/DiffBuilderManager.php
@@ -184,13 +184,10 @@
 
     if ($all_bundles) {
       $storage = $this->entityTypeManager->getStorage('entity_form_display');
-      $visible = TRUE;
+      $visible = FALSE;
       if ($field_definition instanceof BaseFieldDefinition) {
-        foreach ($storage->loadByProperties(['targetEntityType' => $field_definition->getTargetEntityTypeId()]) as $value) {
+        foreach ($storage->loadByProperties(['targetEntityType' => $field_definition->getTargetEntityTypeId(), 'mode' => 'default']) as $value) {
           $visible = (bool) $value->getComponent($field_definition->getName());
-          if (!$visible) {
-            return ['type' => 'hidden'];
-          }
         }
       }
       else {
@@ -198,11 +195,11 @@
           if ($display = $storage->load($field_definition->getTargetEntityTypeId() . '.' . $value . '.default')) {
             $visible = (bool) $display->getComponent($field_definition->getName());
           }
-          if (!$visible) {
-            return ['type' => 'hidden'];
-          }
         }
       }
+      if (!$visible) {
+        return ['type' => 'hidden'];
+      }
     }
 
     $selected_plugin = $this->pluginsConfig->get('fields.' . $field_key);