diff --git a/core/modules/field/src/Tests/Views/FieldUITest.php b/core/modules/field/src/Tests/Views/FieldUITest.php
index bbb8026..b433e0e 100644
--- a/core/modules/field/src/Tests/Views/FieldUITest.php
+++ b/core/modules/field/src/Tests/Views/FieldUITest.php
@@ -86,6 +86,16 @@ public function testHandlerUI() {
     $this->assertEqual($view->field['field_name_0']->options['type'], 'text_trimmed');
     $this->assertEqual($view->field['field_name_0']->options['settings']['trim_length'], $random_number);
 
+    // Now change the formatter back to 'default' which doesn't have any
+    // settings. We want to ensure that the settings are empty then.
+    $edit['options[type]'] = 'text_default';
+    $this->drupalPostForm('admin/structure/views/nojs/handler/test_view_fieldapi/default/field/field_name_0', $edit, t('Apply'));
+    $this->drupalPostForm('admin/structure/views/view/test_view_fieldapi', [], t('Save'));
+    $view = Views::getView('test_view_fieldapi');
+    $view->initHandlers();
+    $this->assertEqual($view->field['field_name_0']->options['type'], 'text_default');
+    $this->assertEqual($view->field['field_name_0']->options['settings'], []);
+
     // Ensure that the view depends on the field storage.
     $dependencies = \Drupal::service('config.manager')->findConfigEntityDependents('config', [$this->fieldStorages[0]->getConfigDependencyName()]);
     $this->assertTrue(isset($dependencies['views.view.test_view_fieldapi']), 'The view is dependent on the field storage.');
diff --git a/core/modules/views/src/Plugin/views/HandlerBase.php b/core/modules/views/src/Plugin/views/HandlerBase.php
index af1db69..c2a271b 100644
--- a/core/modules/views/src/Plugin/views/HandlerBase.php
+++ b/core/modules/views/src/Plugin/views/HandlerBase.php
@@ -820,4 +820,19 @@ public function submitTemporaryForm($form, FormStateInterface $form_state) {
     // Write to cache
     $view->cacheSet();
   }
+
+  /**
+   * Calculates options stored on the handler
+   *
+   * @param array $options
+   *   The options stored in the handler
+   * @param array $form_state_options
+   *   The newly submitted form state options.
+   *
+   * @return array
+   *   The new options
+   */
+  public function submitFormCalculateOptions(array $options, array $form_state_options) {
+    return $form_state_options + $options;
+  }
 }
diff --git a/core/modules/views/src/Plugin/views/field/Field.php b/core/modules/views/src/Plugin/views/field/Field.php
index 52288a6..5826939 100644
--- a/core/modules/views/src/Plugin/views/field/Field.php
+++ b/core/modules/views/src/Plugin/views/field/Field.php
@@ -477,7 +477,8 @@ public function buildOptionsForm(&$form, FormStateInterface $form_state) {
 
     // Get the settings form.
     $settings_form = array('#value' => array());
-    if ($formatter = $this->getFormatterInstance()) {
+    $format = isset($form_state->getUserInput()['options']['type']) ? $form_state->getUserInput()['options']['type'] : $this->options['type'];
+    if ($formatter = $this->getFormatterInstance($format)) {
       $settings_form = $formatter->settingsForm($form, $form_state);
       // Convert field UI selector states to work in the Views field form.
       FormHelper::rewriteStatesSelector($settings_form, "fields[{$field->getName()}][settings_edit_form]", 'options');
@@ -486,6 +487,21 @@ public function buildOptionsForm(&$form, FormStateInterface $form_state) {
   }
 
   /**
+   * {@inheritdoc}
+   */
+  public function submitFormCalculateOptions(array $options, array $form_state_options) {
+    // When we change the formatter type we don't want to keep any of the
+    // previous configured formatter settings, as there might be schema
+    // conflict.
+    unset($options['settings']);
+    $options = $form_state_options + $options;
+    if (!isset($options['settings'])) {
+      $options['settings'] = [];
+    }
+    return $options;
+  }
+
+  /**
    * Provide options for multiple value fields.
    */
   function multiple_options_form(&$form, FormStateInterface $form_state) {
@@ -942,13 +958,16 @@ protected function addSelfTokens(&$tokens, $item) {
    * @return \Drupal\Core\Field\FormatterInterface|null
    *   The field formatter instance.
    */
-  protected function getFormatterInstance() {
-    $settings = $this->options['settings'] + $this->formatterPluginManager->getDefaultSettings($this->options['type']);
+  protected function getFormatterInstance($format = NULL) {
+    if(!isset($format)) {
+      $format = $this->options['type'];
+    }
+    $settings = $this->options['settings'] + $this->formatterPluginManager->getDefaultSettings($format);
 
     $options = [
       'field_definition' => $this->getFieldDefinition(),
       'configuration' => [
-        'type' => $this->options['type'],
+        'type' => $format,
         'settings' => $settings,
         'label' => '',
         'weight' => 0,
diff --git a/core/modules/views_ui/src/Form/Ajax/ConfigHandler.php b/core/modules/views_ui/src/Form/Ajax/ConfigHandler.php
index d3fdd6a..84b147d 100644
--- a/core/modules/views_ui/src/Form/Ajax/ConfigHandler.php
+++ b/core/modules/views_ui/src/Form/Ajax/ConfigHandler.php
@@ -242,7 +242,7 @@ public function submitForm(array &$form, FormStateInterface $form_state) {
 
     // Add the incoming options to existing options because items using
     // the extra form may not have everything in the form here.
-    $options = $form_state->getValue('options') + $handler->options;
+    $options = $handler->submitFormCalculateOptions($handler->options, $form_state->getValue('options', []));
 
     // This unpacks only options that are in the definition, ensuring random
     // extra stuff on the form is not sent through.