diff --git a/core/modules/layout_builder/config/schema/layout_builder.schema.yml b/core/modules/layout_builder/config/schema/layout_builder.schema.yml
index 7bc4461891..ed0bbf7893 100644
--- a/core/modules/layout_builder/config/schema/layout_builder.schema.yml
+++ b/core/modules/layout_builder/config/schema/layout_builder.schema.yml
@@ -82,3 +82,11 @@ layout_plugin.settings.layout_twocol_section:
 
 layout_plugin.settings.layout_threecol_section:
   type: layout_builder_multi_width
+
+layout_builder.settings:
+  type: config_object
+  label: 'Layout builder settings'
+  mapping:
+    expose_all_field_blocks:
+      type: boolean
+      label: 'Expose all field blocks'
diff --git a/core/modules/layout_builder/src/Entity/LayoutBuilderEntityViewDisplay.php b/core/modules/layout_builder/src/Entity/LayoutBuilderEntityViewDisplay.php
index 70e86f6999..31396457d9 100644
--- a/core/modules/layout_builder/src/Entity/LayoutBuilderEntityViewDisplay.php
+++ b/core/modules/layout_builder/src/Entity/LayoutBuilderEntityViewDisplay.php
@@ -156,6 +156,10 @@ public function preSave(EntityStorageInterface $storage) {
         // When being disabled, remove all existing section data.
         $this->removeAllSections();
       }
+
+      // Invalidate the block cache in order to regenerate field block
+      // definitions.
+      \Drupal::service('plugin.manager.block')->clearCachedDefinitions();
     }
   }
 
diff --git a/core/modules/layout_builder/src/Plugin/Derivative/ExtraFieldBlockDeriver.php b/core/modules/layout_builder/src/Plugin/Derivative/ExtraFieldBlockDeriver.php
index 8de394c349..6e8b6c3dad 100644
--- a/core/modules/layout_builder/src/Plugin/Derivative/ExtraFieldBlockDeriver.php
+++ b/core/modules/layout_builder/src/Plugin/Derivative/ExtraFieldBlockDeriver.php
@@ -4,6 +4,7 @@
 
 use Drupal\Component\Plugin\Derivative\DeriverBase;
 use Drupal\Component\Plugin\PluginBase;
+use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Entity\EntityFieldManagerInterface;
 use Drupal\Core\Entity\EntityTypeBundleInfoInterface;
 use Drupal\Core\Entity\EntityTypeManagerInterface;
@@ -52,6 +53,13 @@ class ExtraFieldBlockDeriver extends DeriverBase implements ContainerDeriverInte
    */
   protected $entityTypeRepository;
 
+  /**
+   * The config factory.
+   *
+   * @var \Drupal\Core\Config\ConfigFactoryInterface
+   */
+  protected $configFactory;
+
   /**
    * Constructs new FieldBlockDeriver.
    *
@@ -63,12 +71,15 @@ class ExtraFieldBlockDeriver extends DeriverBase implements ContainerDeriverInte
    *   The entity type bundle info.
    * @param \Drupal\Core\Entity\EntityTypeRepositoryInterface $entity_type_repository
    *   The entity type repository.
+   * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
+   *   The config factory.
    */
-  public function __construct(EntityFieldManagerInterface $entity_field_manager, EntityTypeManagerInterface $entity_type_manager, EntityTypeBundleInfoInterface $entity_type_bundle_info, EntityTypeRepositoryInterface $entity_type_repository) {
+  public function __construct(EntityFieldManagerInterface $entity_field_manager, EntityTypeManagerInterface $entity_type_manager, EntityTypeBundleInfoInterface $entity_type_bundle_info, EntityTypeRepositoryInterface $entity_type_repository, ConfigFactoryInterface $config_factory) {
     $this->entityFieldManager = $entity_field_manager;
     $this->entityTypeManager = $entity_type_manager;
     $this->entityTypeBundleInfo = $entity_type_bundle_info;
     $this->entityTypeRepository = $entity_type_repository;
+    $this->configFactory = $config_factory;
   }
 
   /**
@@ -79,7 +90,8 @@ public static function create(ContainerInterface $container, $base_plugin_id) {
       $container->get('entity_field.manager'),
       $container->get('entity_type.manager'),
       $container->get('entity_type.bundle.info'),
-      $container->get('entity_type.repository')
+      $container->get('entity_type.repository'),
+      $container->get('config.factory')
     );
   }
 
@@ -88,14 +100,26 @@ public static function create(ContainerInterface $container, $base_plugin_id) {
    */
   public function getDerivativeDefinitions($base_plugin_definition) {
     $entity_type_labels = $this->entityTypeRepository->getEntityTypeLabels();
+    $enabled_bundle_ids = $this->bundleIdsWithLayoutBuilderDisplays();
+    $expose_all_fields = $this->configFactory->get('layout_builder.settings')->get('expose_all_field_blocks');
     foreach ($this->entityTypeManager->getDefinitions() as $entity_type_id => $entity_type) {
       // Only process fieldable entity types.
       if (!$entity_type->entityClassImplements(FieldableEntityInterface::class)) {
         continue;
       }
 
+      // If not loading everything, skip entity types that aren't included.
+      if (!$expose_all_fields && !isset($enabled_bundle_ids[$entity_type_id])) {
+        continue;
+      }
+
       $bundles = $this->entityTypeBundleInfo->getBundleInfo($entity_type_id);
       foreach ($bundles as $bundle_id => $bundle) {
+        // If not loading everything, skip bundle types that aren't included.
+        if (!$expose_all_fields && !in_array($bundle_id, $enabled_bundle_ids[$entity_type_id])) {
+          continue;
+        }
+
         $extra_fields = $this->entityFieldManager->getExtraFields($entity_type_id, $bundle_id);
         // Skip bundles without any extra fields.
         if (empty($extra_fields['display'])) {
@@ -123,4 +147,17 @@ public function getDerivativeDefinitions($base_plugin_definition) {
     return $this->derivatives;
   }
 
+  protected function bundleIdsWithLayoutBuilderDisplays() {
+    /** @var \Drupal\layout_builder\Entity\LayoutEntityDisplayInterface[] $displays */
+    $displays = $this->entityTypeManager->getStorage('entity_view_display')->loadByProperties([
+      'third_party_settings.layout_builder.enabled' => TRUE,
+    ]);
+    $layout_bundles = [];
+    foreach ($displays as $display) {
+      $layout_bundles[$display->getTargetEntityTypeId()][] = $display->getTargetBundle();
+    }
+    $layout_bundles = array_map('array_unique', $layout_bundles);
+    return $layout_bundles;
+  }
+
 }
diff --git a/core/modules/layout_builder/src/Plugin/Derivative/FieldBlockDeriver.php b/core/modules/layout_builder/src/Plugin/Derivative/FieldBlockDeriver.php
index a2a94a024d..a93dfe0b0c 100644
--- a/core/modules/layout_builder/src/Plugin/Derivative/FieldBlockDeriver.php
+++ b/core/modules/layout_builder/src/Plugin/Derivative/FieldBlockDeriver.php
@@ -4,6 +4,8 @@
 
 use Drupal\Component\Plugin\Derivative\DeriverBase;
 use Drupal\Component\Plugin\PluginBase;
+use Drupal\Core\Config\ConfigFactoryInterface;
+use Drupal\Core\Config\Entity\ConfigEntityStorageInterface;
 use Drupal\Core\Entity\EntityFieldManagerInterface;
 use Drupal\Core\Entity\EntityTypeRepositoryInterface;
 use Drupal\Core\Field\FieldConfigInterface;
@@ -53,6 +55,20 @@ class FieldBlockDeriver extends DeriverBase implements ContainerDeriverInterface
    */
   protected $formatterManager;
 
+  /**
+   * The entity view display storage.
+   *
+   * @var \Drupal\Core\Config\Entity\ConfigEntityStorageInterface
+   */
+  protected $entityViewDisplayStorage;
+
+  /**
+   * The config factory.
+   *
+   * @var \Drupal\Core\Config\ConfigFactoryInterface
+   */
+  protected $configFactory;
+
   /**
    * Constructs new FieldBlockDeriver.
    *
@@ -64,12 +80,18 @@ class FieldBlockDeriver extends DeriverBase implements ContainerDeriverInterface
    *   The field type manager.
    * @param \Drupal\Core\Field\FormatterPluginManager $formatter_manager
    *   The formatter manager.
+   * @param \Drupal\Core\Config\Entity\ConfigEntityStorageInterface $entity_view_display_storage
+   *   The entity view display storage.
+   * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory
+   *   The config factory.
    */
-  public function __construct(EntityTypeRepositoryInterface $entity_type_repository, EntityFieldManagerInterface $entity_field_manager, FieldTypePluginManagerInterface $field_type_manager, FormatterPluginManager $formatter_manager) {
+  public function __construct(EntityTypeRepositoryInterface $entity_type_repository, EntityFieldManagerInterface $entity_field_manager, FieldTypePluginManagerInterface $field_type_manager, FormatterPluginManager $formatter_manager, ConfigEntityStorageInterface $entity_view_display_storage, ConfigFactoryInterface $config_factory) {
     $this->entityTypeRepository = $entity_type_repository;
     $this->entityFieldManager = $entity_field_manager;
     $this->fieldTypeManager = $field_type_manager;
     $this->formatterManager = $formatter_manager;
+    $this->entityViewDisplayStorage = $entity_view_display_storage;
+    $this->configFactory = $config_factory;
   }
 
   /**
@@ -80,7 +102,9 @@ public static function create(ContainerInterface $container, $base_plugin_id) {
       $container->get('entity_type.repository'),
       $container->get('entity_field.manager'),
       $container->get('plugin.manager.field.field_type'),
-      $container->get('plugin.manager.field.formatter')
+      $container->get('plugin.manager.field.formatter'),
+      $container->get('entity_type.manager')->getStorage('entity_view_display'),
+      $container->get('config.factory')
     );
   }
 
@@ -89,7 +113,7 @@ public static function create(ContainerInterface $container, $base_plugin_id) {
    */
   public function getDerivativeDefinitions($base_plugin_definition) {
     $entity_type_labels = $this->entityTypeRepository->getEntityTypeLabels();
-    foreach ($this->entityFieldManager->getFieldMap() as $entity_type_id => $entity_field_map) {
+    foreach ($this->getFieldMap() as $entity_type_id => $entity_field_map) {
       foreach ($entity_field_map as $field_name => $field_info) {
         // Skip fields without any formatters.
         $options = $this->formatterManager->getOptions($field_info['type']);
@@ -135,4 +159,50 @@ public function getDerivativeDefinitions($base_plugin_definition) {
     return $this->derivatives;
   }
 
+  /**
+   * Returns the entity field map for deriving block definitions.
+   *
+   * @return array
+   *   The entity field map.
+   *
+   * @see \Drupal\Core\Entity\EntityFieldManagerInterface::getFieldMap()
+   */
+  protected function getFieldMap() {
+    $field_map = $this->entityFieldManager->getFieldMap();
+
+    // If all fields are exposed as field blocks, just return the field map
+    // without any further processing.
+    if ($this->configFactory->get('layout_builder.settings')->get('expose_all_field_blocks')) {
+      return $field_map;
+    }
+
+    // Load all entity view displays which are using Layout Builder.
+    /** @var \Drupal\layout_builder\Entity\LayoutEntityDisplayInterface[] $displays */
+    $displays = $this->entityViewDisplayStorage->loadByProperties([
+      'third_party_settings.layout_builder.enabled' => TRUE,
+    ]);
+    $layout_bundles = [];
+    foreach ($displays as $display) {
+      $layout_bundles[$display->getEntityTypeId()][] = $display->getTargetBundle();
+    }
+    $layout_bundles = array_map('array_unique', $layout_bundles);
+
+    // Process $field_map, removing any bundles which are not using Layout
+    // Builder.
+    $field_map = array_intersect_key($field_map, $layout_bundles);
+
+    foreach ($field_map as $entity_type_id => $fields) {
+      foreach ($fields as $field_name => $field_info) {
+        $field_info['bundles'] = array_intersect($field_info['bundles'], $layout_bundles[$entity_type_id]);
+
+        // If no bundles are using Layout Builder, remove this field from the
+        // field map.
+        if (empty($field_info['bundles'])) {
+          unset($field_map[$entity_type_id][$field_name]);
+        }
+      }
+    }
+    return $field_map;
+  }
+
 }
