diff --git a/diff.services.yml b/diff.services.yml
index e04e1dd..1064d80 100755
--- a/diff.services.yml
+++ b/diff.services.yml
@@ -11,7 +11,7 @@ services:
   plugin.manager.diff.builder:
     class: Drupal\diff\DiffBuilderManager
     parent: default_plugin_manager
-    arguments: ['@entity_type.manager', '@config.factory']
+    arguments: ['@entity_type.manager', '@diff.build_settings']
 
   diff.entity_parser:
     class: Drupal\diff\DiffEntityParser
@@ -25,4 +25,8 @@ services:
 
   diff.entity_comparison:
     class: Drupal\diff\DiffEntityComparison
-    arguments: ['@config.factory', '@diff.diff.formatter','@plugin.manager.field.field_type', '@diff.entity_parser']
+    arguments: ['@diff.diff.formatter','@plugin.manager.field.field_type', '@diff.entity_parser', '@diff.build_settings']
+
+  diff.build_settings:
+    class: Drupal\diff\DiffBuildSettings
+    arguments: ['@entity_type.manager', '@config.factory']
diff --git a/src/DiffBuildSettings.php b/src/DiffBuildSettings.php
new file mode 100644
index 0000000..5a22d3c
--- /dev/null
+++ b/src/DiffBuildSettings.php
@@ -0,0 +1,97 @@
+<?php
+
+namespace Drupal\diff;
+
+use Drupal\Core\Config\ConfigFactory;
+use Drupal\Core\Entity\EntityTypeManagerInterface;
+use Drupal\Core\Field\FieldDefinitionInterface;
+
+/**
+ * Diff build settings service that guides the building of diffs.
+ */
+class DiffBuildSettings implements DiffBuildSettingsInterface {
+
+  /**
+   * The entity type manager.
+   *
+   * @var \Drupal\Core\Entity\EntityTypeManagerInterface
+   */
+  protected $entityTypeManager;
+
+  /**
+   * Contains the configuration object factory.
+   *
+   * @var \Drupal\Core\Config\ConfigFactoryInterface
+   */
+  protected $configFactory;
+
+  /**
+   * Wrapper object for reading simple configuration from diff.plugins.yml.
+   */
+  protected $pluginsConfig;
+
+  /**
+   * Constructs a DiffBuildSettings object.
+   *
+   * @param EntityTypeManagerInterface $entity_type_manager
+   *   Entity Manager service.
+   * @param ConfigFactory $config_factory
+   *   A config factory.
+   */
+  public function __construct(EntityTypeManagerInterface $entity_type_manager, ConfigFactory $config_factory) {
+    $this->entityTypeManager = $entity_type_manager;
+    $this->config = $config_factory->get('diff.settings');
+    $this->pluginsConfig = $config_factory->get('diff.plugins');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function fieldSettings($field_key) {
+    return $this->pluginsConfig->get('fields.' . $field_key);
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function displayField(FieldDefinitionInterface $field_definition) {
+    $view_mode = NULL;
+    if ($target_bundle = $field_definition->getTargetBundle()) {
+      $view_mode = $this->config->get('content_type_settings.' . $target_bundle . '.view_mode');
+    }
+    $visible = $this->displayFieldConsideringViewMode($field_definition, $view_mode);
+    return $visible;
+  }
+
+  /**
+   * Determine field visibility based on visibility in a view mode.
+   *
+   * @param FieldDefinitionInterface $field_definition
+   *   The field to consider.
+   * @param string $view_mode
+   *   The view mode to consider.
+   *
+   * @return bool
+   *   Whether or not to display the field.
+   */
+  protected function displayFieldConsideringViewMode(FieldDefinitionInterface $field_definition, $view_mode = NULL) {
+    $visible = TRUE;
+    $storage = $this->entityTypeManager->getStorage('entity_view_display');
+    if ($target_bundle = $field_definition->getTargetBundle()) {
+      // If no view mode is selected use the default view mode.
+      if ($view_mode == NULL) {
+        $view_mode = 'default';
+      }
+      if ($display = $storage->load($field_definition->getTargetEntityTypeId() . '.' . $target_bundle . '.' . $view_mode)) {
+        $visible = (bool) $display->getComponent($field_definition->getName());
+      }
+    }
+    elseif ($field_definition instanceof BaseFieldDefinition) {
+      if (!$display = $field_definition->getDisplayOptions('view')) {
+        $visible = FALSE;
+      }
+    }
+    return $visible;
+  }
+
+}
diff --git a/src/DiffBuildSettingsInterface.php b/src/DiffBuildSettingsInterface.php
new file mode 100644
index 0000000..cc91176
--- /dev/null
+++ b/src/DiffBuildSettingsInterface.php
@@ -0,0 +1,32 @@
+<?php
+
+namespace Drupal\diff;
+
+use Drupal\Core\Field\FieldDefinitionInterface;
+
+interface DiffBuildSettingsInterface {
+
+  /**
+   * Provide diff builder plugin settings for a given field.
+   *
+   * @param string $field_key
+   *   A key to identify the field in diff.plugins.yml,
+   *   with the format entitytype.fieldname.
+   *
+   * @return array
+   *   An array with the structure used in diff.plugins.yml.
+   */
+  public function fieldSettings($field_key);
+
+  /**
+   * Determine whether a field should be displayed as part of a diff.
+   *
+   * @param FieldDefinitionInterface $field_definition
+   *   The field to consider.
+   *
+   * @return bool
+   *   Whether or not to display the field.
+   */
+  public function displayField(FieldDefinitionInterface $field_definition);
+
+}
diff --git a/src/DiffBuilderManager.php b/src/DiffBuilderManager.php
index 93aafa3..c50c7c2 100644
--- a/src/DiffBuilderManager.php
+++ b/src/DiffBuilderManager.php
@@ -3,13 +3,13 @@
 namespace Drupal\diff;
 
 use Drupal\Core\Cache\CacheBackendInterface;
-use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Extension\ModuleHandlerInterface;
 use Drupal\Core\Field\BaseFieldDefinition;
 use Drupal\Core\Field\FieldDefinitionInterface;
 use Drupal\Core\Field\FieldStorageDefinitionInterface;
 use Drupal\Core\Plugin\DefaultPluginManager;
+use Drupal\diff\DiffBuildSettings;
 
 /**
  * Plugin type manager for field diff builders.
@@ -26,14 +26,9 @@ class DiffBuilderManager extends DefaultPluginManager {
   protected $entityTypeManager;
 
   /**
-   * Wrapper object for writing/reading simple configuration from diff.settings.yml
+   * The Diff build settings service.
    */
-  protected $config;
-
-  /**
-   * Wrapper object for writing/reading simple configuration from diff.plugins.yml
-   */
-  protected $pluginsConfig;
+  protected $buildSettings;
 
   /**
    * Constructs a FieldDiffBuilderManager object.
@@ -47,17 +42,16 @@ class DiffBuilderManager extends DefaultPluginManager {
    *   The module handler.
    * @param EntityTypeManagerInterface $entity_type_manager
    *   Entity Manager service.
-   * @param ConfigFactoryInterface $configFactory
-   *   The configuration factory.
+   * @param DiffBuildSettingsInterface $build_settings
+   *   DiffBuildSettings service.
    */
-  public function __construct(\Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler, EntityTypeManagerInterface $entity_type_manager, ConfigFactoryInterface $configFactory) {
+  public function __construct(\Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler, EntityTypeManagerInterface $entity_type_manager, DiffBuildSettingsInterface $build_settings) {
     parent::__construct('Plugin/Diff', $namespaces, $module_handler, '\Drupal\diff\FieldDiffBuilderInterface', 'Drupal\diff\Annotation\FieldDiffBuilder');
 
     $this->setCacheBackend($cache_backend, 'field_diff_builder_plugins');
     $this->alterInfo('field_diff_builder_info');
     $this->entityTypeManager = $entity_type_manager;
-    $this->config = $configFactory->get('diff.settings');
-    $this->pluginsConfig =  $configFactory->get('diff.plugins');
+    $this->buildSettings = $build_settings;
 
   }
 
@@ -114,10 +108,10 @@ class DiffBuilderManager extends DefaultPluginManager {
   }
 
   /**
-   * Selects a default plugin for a field definition.
+   * Selects a plugin for a field.
    *
-   * Checks the display configuration of the field to define if it should be
-   * displayed in the diff comparison.
+   * Checks the build settings for a field to define if it should be
+   * displayed in the diff comparison. If it is, then get its plugin.
    *
    * @param \Drupal\Core\Field\FieldDefinitionInterface $field_definition
    *   The field definition.
@@ -127,23 +121,7 @@ class DiffBuilderManager extends DefaultPluginManager {
    */
   public function getSelectedPluginForFieldDefinition(FieldDefinitionInterface $field_definition) {
     $selected_plugin = NULL;
-    $visible = TRUE;
-    $storage = $this->entityTypeManager->getStorage('entity_view_display');
-    if ($target_bundle = $field_definition->getTargetBundle()) {
-      $view_mode = $this->config->get('content_type_settings.' . $target_bundle . '.view_mode');
-      // If no view mode is selected use the default view mode.
-      if ($view_mode == NULL) {
-        $view_mode = 'default';
-      }
-      if ($display = $storage->load($field_definition->getTargetEntityTypeId() . '.' . $target_bundle . '.' . $view_mode)) {
-        $visible = (bool) $display->getComponent($field_definition->getName());
-      }
-    }
-    elseif ($field_definition instanceof BaseFieldDefinition) {
-      if (!$display = $field_definition->getDisplayOptions('view')) {
-        $visible = FALSE;
-      }
-    }
+    $visible = $this->buildSettings->displayField($field_definition);
     if ($visible) {
       $selected_plugin = $this->getSelectedPluginForFieldStorageDefinition($field_definition->getFieldStorageDefinition());
     }
@@ -163,28 +141,11 @@ class DiffBuilderManager extends DefaultPluginManager {
    *   The selected plugin for the field.
    */
   public function getSelectedPluginForFieldStorageDefinition(FieldStorageDefinitionInterface $field_definition) {
+    $plugin_options = $this->getApplicablePluginOptions($field_definition);
     $field_key = $field_definition->getTargetEntityTypeId() . '.' . $field_definition->getName();
-    $selected_plugin = $this->pluginsConfig->get('fields.' . $field_key);
-    if (!$selected_plugin) {
-      // Get the definition of all FieldDiffBuilder plugins.
-      $plugins = [];
-      foreach ($this->getDefinitions() as $plugin_definition) {
-        if (isset($plugin_definition['field_types'])) {
-          // Iterate through all the field types this plugin supports
-          // and for every such field type add the id of the plugin.
-          foreach ($plugin_definition['field_types'] as $id) {
-            $plugins[$id][] = $plugin_definition['id'];
-          }
-        }
-      }
-
-      // Build a list of all diff plugins supporting the field type of the field.
-      $plugin_options = [];
-      if (isset($plugins[$field_definition->getType()])) {
-        foreach ($plugins[$field_definition->getType()] as $id) {
-          $plugin_options[$id] = $this->getDefinitions()[$id]['label'];
-        }
-      }
+    $selected_plugin = $this->buildSettings->fieldSettings($field_key);
+    // Check if the plugin stored to the fields is still applicable.
+    if (!$selected_plugin || !in_array($selected_plugin['type'], array_keys($plugin_options))) {
       if (!empty($plugin_options)) {
         $selected_plugin['type'] = array_keys($plugin_options)[0];
       }
@@ -194,4 +155,49 @@ class DiffBuilderManager extends DefaultPluginManager {
     }
     return $selected_plugin;
   }
+
+  /**
+   * Gets the applicable plugin options for a given field.
+   *
+   * Loop over the plugins that can be applied to the field and builds an array
+   * of possible plugins based on each plugin weight.
+   *
+   * @param FieldStorageDefinitionInterface $field_definition
+   *   The field storage definition.
+   *
+   * @return array
+   *   The plugin option for the given field based on plugin weight.
+   */
+  public function getApplicablePluginOptions(FieldStorageDefinitionInterface $field_definition) {
+    // Get the definition of all the FieldDiffBuilder plugins.
+    $plugins = [];
+    foreach ($this->getDefinitions() as $plugin_definition) {
+      if (isset($plugin_definition['field_types'])) {
+        // Iterate through all the field types this plugin supports
+        // and for every such field type add the id of the plugin.
+        if(!isset($plugin_definition['weight'])) {
+          $plugin_definition['weight'] = 0;
+        };
+        $plugin['id'] = $plugin_definition['id'];
+        foreach ($plugin_definition['field_types'] as $id) {
+          $plugins[$id][$plugin_definition['id']]['weight'] = $plugin_definition['weight'];
+        }
+      }
+    }
+
+    // Build a list of all diff plugins supporting the field type of the field.
+    $plugin_options = [];
+    if (isset($plugins[$field_definition->getType()])) {
+      // Sort the plugins based on their weight.
+      asort($plugins[$field_definition->getType()]);
+      foreach ($plugins[$field_definition->getType()] as $id => $weight) {
+        $definition = $this->getDefinition($id, FALSE);
+        // Check if the plugin is applicable.
+        if (isset($definition['class']) && in_array($field_definition->getType(), $definition['field_types']) && $definition['class']::isApplicable($field_definition)) {
+          $plugin_options[$id] = $this->getDefinitions()[$id]['label'];
+        }
+      }
+    }
+    return $plugin_options;
+  }
 }
diff --git a/src/DiffEntityComparison.php b/src/DiffEntityComparison.php
index 8184326..374ccbb 100644
--- a/src/DiffEntityComparison.php
+++ b/src/DiffEntityComparison.php
@@ -2,8 +2,8 @@
 
 namespace Drupal\diff;
 
-use Drupal\Core\Config\ConfigFactory;
 use Drupal\Component\Plugin\PluginManagerInterface;
+use Drupal\diff\DiffBuildSettings;
 use Drupal\Core\Entity\ContentEntityInterface;
 use Drupal\Component\Diff\Diff;
 use Drupal\Core\Render\Element;
@@ -16,16 +16,9 @@ use Drupal\Component\Utility\Xss;
 class DiffEntityComparison {
 
   /**
-   * Contains the configuration object factory.
-   *
-   * @var \Drupal\Core\Config\ConfigFactoryInterface
+   * The Diff build settings service.
    */
-  protected $configFactory;
-
-  /**
-   * Wrapper object for writing/reading simple configuration from diff.plugins.yml
-   */
-  protected $pluginsConfig;
+  protected $buildSettings;
 
   /**
    * The diff formatter.
@@ -49,8 +42,6 @@ class DiffEntityComparison {
   /**
    * Constructs a DiffEntityComparison object.
    *
-   * @param ConfigFactory $config_factory
-   *   Diff formatter service.
    * @param DiffFormatter $diff_formatter
    *   Diff formatter service.
    * @param PluginManagerInterface $plugin_manager
@@ -58,9 +49,8 @@ class DiffEntityComparison {
    * @param DiffEntityParser $entity_parser
    *   The diff field builder plugin manager.
    */
-  public function __construct(ConfigFactory $config_factory, DiffFormatter $diff_formatter, PluginManagerInterface $plugin_manager, DiffEntityParser $entity_parser) {
-    $this->configFactory = $config_factory;
-    $this->pluginsConfig = $this->configFactory->get('diff.plugins');
+  public function __construct(DiffFormatter $diff_formatter, PluginManagerInterface $plugin_manager, DiffEntityParser $entity_parser, $build_settings) {
+    $this->buildSettings = $build_settings;
     $this->diffFormatter = $diff_formatter;
     $this->fieldTypeDefinitions = $plugin_manager->getDefinitions();
     $this->entityParser = $entity_parser;
@@ -86,7 +76,7 @@ class DiffEntityComparison {
     foreach ($left_values as $left_key => $values) {
       list (, $field_key) = explode(':', $left_key);
       // Get the compare settings for this field type.
-      $compare_settings = $this->pluginsConfig->get('fields.' . $field_key);
+      $compare_settings = $this->buildSettings->fieldSettings($field_key);
       $result[$left_key] = [
         '#name' => (isset($compare_settings['settings']['show_header']) && $compare_settings['settings']['show_header'] == 0) ? '' : $values['label'],
         '#settings' => $compare_settings,
@@ -108,7 +98,7 @@ class DiffEntityComparison {
     // Fields which exist only on the right entity.
     foreach ($right_values as $right_key => $values) {
       list (, $field_key) = explode(':', $right_key);
-      $compare_settings = $this->pluginsConfig->get('fields.' . $field_key);
+      $compare_settings = $this->buildSettings->fieldSettings($field_key);
       $result[$right_key] = [
         '#name' => (isset($compare_settings['settings']['show_header']) && $compare_settings['settings']['show_header'] == 0) ? '' : $values['label'],
         '#settings' => $compare_settings,
diff --git a/src/DiffEntityParser.php b/src/DiffEntityParser.php
index b347ccf..5d1b7b4 100644
--- a/src/DiffEntityParser.php
+++ b/src/DiffEntityParser.php
@@ -2,7 +2,6 @@
 
 namespace Drupal\diff;
 
-use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Entity\ContentEntityInterface;
 use Drupal\Core\Language\LanguageInterface;
 
@@ -16,26 +15,12 @@ class DiffEntityParser {
   protected $diffBuilderManager;
 
   /**
-   * Wrapper object for writing/reading simple configuration from diff.settings.yml
-   */
-  protected $config;
-
-  /**
-   * Wrapper object for writing/reading simple configuration from diff.plugins.yml
-   */
-  protected $pluginsConfig;
-
-  /**
    * Constructs an EntityComparisonBase object.
    *
    * @param DiffBuilderManager $diffBuilderManager
    *   The diff field builder plugin manager.
-   * @param ConfigFactoryInterface $configFactory
-   *   The configuration factory.
    */
-  public function __construct(DiffBuilderManager $diffBuilderManager, ConfigFactoryInterface $configFactory) {
-    $this->config = $configFactory->get('diff.settings');
-    $this->pluginsConfig =  $configFactory->get('diff.plugins');
+  public function __construct(DiffBuilderManager $diffBuilderManager) {
     $this->diffBuilderManager = $diffBuilderManager;
   }
 
diff --git a/src/FieldDiffBuilderBase.php b/src/FieldDiffBuilderBase.php
index e40fd65..59f9c12 100644
--- a/src/FieldDiffBuilderBase.php
+++ b/src/FieldDiffBuilderBase.php
@@ -3,6 +3,7 @@
 namespace Drupal\diff;
 
 use Drupal\Component\Plugin\PluginBase;
+use Drupal\Core\Field\FieldStorageDefinitionInterface;
 use Drupal\Core\Form\FormStateInterface;
 use Drupal\Core\Config\ConfigFactoryInterface;
 use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
@@ -161,4 +162,10 @@ abstract class FieldDiffBuilderBase extends PluginBase implements FieldDiffBuild
     return array();
   }
 
+  /**
+   * {@inheritdoc}
+   */
+  public static function isApplicable(FieldStorageDefinitionInterface $field_definition) {
+    return TRUE;
+  }
 }
diff --git a/src/FieldDiffBuilderInterface.php b/src/FieldDiffBuilderInterface.php
index fdb07c7..ac76b5a 100644
--- a/src/FieldDiffBuilderInterface.php
+++ b/src/FieldDiffBuilderInterface.php
@@ -3,6 +3,7 @@
 namespace Drupal\diff;
 
 use Drupal\Core\Field\FieldItemListInterface;
+use Drupal\Core\Field\FieldStorageDefinitionInterface;
 use Drupal\Core\Plugin\PluginFormInterface;
 use Drupal\Component\Plugin\ConfigurablePluginInterface;
 
@@ -36,4 +37,14 @@ interface FieldDiffBuilderInterface extends PluginFormInterface, ConfigurablePlu
    */
   public function build(FieldItemListInterface $field_items);
 
+  /**
+   * Returns if the plugin can be used for the provided field.
+   *
+   * @param \Drupal\Core\Field\FieldStorageDefinitionInterface $field_definition
+   *   The field definition that should be checked.
+   *
+   * @return bool
+   *   TRUE if the plugin can be used, FALSE otherwise.
+   */
+  public static function isApplicable(FieldStorageDefinitionInterface $field_definition);
 }
diff --git a/src/Form/FieldsSettingsForm.php b/src/Form/FieldsSettingsForm.php
index 9e7d078..c7d75d9 100644
--- a/src/Form/FieldsSettingsForm.php
+++ b/src/Form/FieldsSettingsForm.php
@@ -114,19 +114,6 @@ class FieldsSettingsForm extends ConfigFormBase {
       ),
     );
 
-    // Get the definition of all @FieldDiffBuilder plugins.
-    $diff_plugin_definitions = $this->diffBuilderManager->getDefinitions();
-    $plugins = array();
-    foreach ($diff_plugin_definitions as $plugin_definition) {
-      if (isset($plugin_definition['field_types'])) {
-        // Iterate through all the field types this plugin supports
-        // and for every such field type add the id of the plugin.
-        foreach ($plugin_definition['field_types'] as $id) {
-          $plugins[$id][] = $plugin_definition['id'];
-        }
-      }
-    }
-
     // Build a row in the table for each field of each entity type. Get all the
     // field plugins.
     foreach ($this->entityTypeManager->getDefinitions() as $entity_type_name => $entity_type) {
@@ -144,7 +131,7 @@ class FieldsSettingsForm extends ConfigFormBase {
 
         $key = $entity_type_name . '.' . $field_name;
         // Build a row in the table for this field.
-        $form['fields'][$key] = $this->buildFieldRow($entity_type, $field_definition, $plugins, $diff_plugin_definitions, $form_state);
+        $form['fields'][$key] = $this->buildFieldRow($entity_type, $field_definition, $form_state);
       }
     }
 
@@ -168,30 +155,20 @@ class FieldsSettingsForm extends ConfigFormBase {
    *   The entity type.
    * @param \Drupal\Core\Field\FieldStorageDefinitionInterface $field_definition
    *   Definition the field type.
-   * @param array $plugins
-   *   An array of field types and the associated field diff builder plugins ids.
-   * @param array $diff_plugin_definitions
-   *   Definitions of all field diff builder plugins.
    * @param FormStateInterface $form_state
    *   THe form state object.
    *
    * @return array
    *   A table row for the field type listing table.
    */
-  protected function buildFieldRow($entity_type, $field_definition, $plugins, $diff_plugin_definitions, FormStateInterface $form_state) {
+  protected function buildFieldRow($entity_type, $field_definition, FormStateInterface $form_state) {
     $entity_type_label = $entity_type->getLabel();
     $field_name = $field_definition->getName();
     $field_type = $field_definition->getType();
     $field_key = $entity_type->id() . '.' . $field_name;
 
     $display_options = $this->diffBuilderManager->getSelectedPluginForFieldStorageDefinition($field_definition);
-    // Build a list of all diff plugins supporting the field type of the field.
-    $plugin_options = [];
-    if (isset($plugins[$field_type])) {
-      foreach ($plugins[$field_type] as $id) {
-        $plugin_options[$id] = $diff_plugin_definitions[$id]['label'];
-      }
-    }
+    $plugin_options = $this->diffBuilderManager->getApplicablePluginOptions($field_definition);
 
     // Base button element for the various plugin settings actions.
     $base_button = [