diff --git a/js/panels_ipe/panels_ipe.js b/js/panels_ipe/panels_ipe.js
index 5101a64..215a754 100644
--- a/js/panels_ipe/panels_ipe.js
+++ b/js/panels_ipe/panels_ipe.js
@@ -32,10 +32,7 @@
         revert_tab.set({loading: true});
         $.ajax({
           url: drupalSettings.path.baseUrl + 'admin/panelizer/panels_ipe/' + entity.entity_type_id + '/' + entity.entity_id + '/' + entity.view_mode + '/revert_to_default',
-          data: {
-            // @todo: Use the default this entity is on.
-            default: 'default'
-          },
+          data: {},
           type: 'POST'
         }).done(function (data) {
           location.reload();
diff --git a/panelizer.module b/panelizer.module
index 404cf88..aa7f922 100644
--- a/panelizer.module
+++ b/panelizer.module
@@ -216,13 +216,15 @@ function panelizer_form_entity_view_display_edit_form_alter(&$form, FormStateInt
     ];
     // If this display mode is panelized, then show a link to its settings.
     if (!empty($settings['enable'])) {
+      $form['#cache']['tags'][] = "{$display->getTargetEntityTypeId()}:{$display->getTargetBundle()}:{$display->getMode()}";
       $form['panelizer']['displays'] = [
         '#type' => 'table',
         '#caption' => t('Panelized Displays'),
         '#header' => [t('Label'), t('Default'), t('Operations')],
       ];
-      $operations = [];
       foreach ($display->getThirdPartySetting('panelizer', 'displays', []) as $machine_name => $panels_display) {
+        // Reset operations when in the foreach loop.
+        $operations = [];
         $display_name = $machine_name;
         $machine_name ="{$display->getTargetEntityTypeId()}__{$display->getTargetBundle()}__{$display->getMode()}__$machine_name";
         $operations['edit'] = [
@@ -234,11 +236,11 @@ function panelizer_form_entity_view_display_edit_form_alter(&$form, FormStateInt
             'title' => t('Set default'),
             'url' => Url::fromRoute('panelizer.default.select', ['machine_name' => $machine_name]),
           ];
+          $operations['delete'] = [
+            'title' => t('Delete'),
+            'url' => Url::fromRoute('panelizer.default.delete', ['machine_name' => $machine_name]),
+          ];
         }
-        $operations['delete'] = [
-          'title' => t('Delete'),
-          'url' => Url::fromRoute('panelizer.default.delete', ['machine_name' => $machine_name]),
-        ];
         $form['panelizer']['displays'][$machine_name] = [
           'label' => ['#markup' => $panels_display['label']],
           'default' => ['#markup' => $settings['default'] == $display_name ? 'TRUE' : 'FALSE'],
diff --git a/panelizer.routing.yml b/panelizer.routing.yml
index 37f91a0..04189a6 100644
--- a/panelizer.routing.yml
+++ b/panelizer.routing.yml
@@ -49,7 +49,8 @@ panelizer.default.delete:
     _form: '\Drupal\panelizer\Form\PanelizerDefaultDelete'
     _title: 'Delete panelizer default'
   requirements:
-    _permission: 'administer panelizer'
+    _panelizer_field_ui_view_mode_access: 'TRUE'
+    _custom_access: '\Drupal\panelizer\Access\PanelizerDefaultsDisplayAccess::isNotDefaultDisplay'
 
 panelizer.default.select:
   path: '/admin/structure/panelizer/set_default/{machine_name}'
@@ -57,7 +58,8 @@ panelizer.default.select:
     _form: '\Drupal\panelizer\Form\PanelizerDefaultSelect'
     _title: 'Set as default'
   requirements:
-    _permission: 'administer panelizer'
+    _panelizer_field_ui_view_mode_access: 'TRUE'
+    _custom_access: '\Drupal\panelizer\Access\PanelizerDefaultsDisplayAccess::isNotDefaultDisplay'
 
 # Contexts
 panelizer.wizard.step.context.add:
diff --git a/panelizer.services.yml b/panelizer.services.yml
index b6153cf..d36b86c 100644
--- a/panelizer.services.yml
+++ b/panelizer.services.yml
@@ -1,4 +1,9 @@
 services:
+  access_check.panelizer.view_mode:
+    class: Drupal\panelizer\Access\ViewModeAccessCheck
+    arguments: ['@access_check.field_ui.view_mode']
+    tags:
+      - { name: access_check, applies_to: _panelizer_field_ui_view_mode_access }
   plugin.manager.panelizer_entity:
     class: Drupal\panelizer\Plugin\PanelizerEntityManager
     parent: default_plugin_manager
diff --git a/src/Access/PanelizerDefaultsDisplayAccess.php b/src/Access/PanelizerDefaultsDisplayAccess.php
new file mode 100644
index 0000000..ccca0a5
--- /dev/null
+++ b/src/Access/PanelizerDefaultsDisplayAccess.php
@@ -0,0 +1,38 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\panelizer\Controller\PanelizerDefaultsDisplayController.
+ */
+
+namespace Drupal\panelizer\Access;
+
+use Drupal\Core\Access\AccessResult;
+
+/**
+ * Provides a custom access checking mechanism for default displays.
+ */
+class PanelizerDefaultsDisplayAccess {
+
+  /**
+   * Custom access checker for determining the default display of the bundle.
+   *
+   * @param string $machine_name
+   *   The machine name of the default display.
+   *
+   * @return \Drupal\Core\Access\AccessResult
+   */
+  public function isNotDefaultDisplay($machine_name) {
+    list($entity_type, $bundle, $view_mode, $default) = explode('__', $machine_name);
+    /** @var \Drupal\panelizer\Panelizer $panelizer */
+    $panelizer = \Drupal::service('panelizer');
+    $settings = $panelizer->getPanelizerSettings($entity_type, $bundle, $view_mode);
+    if ($settings['default'] != $default) {
+      $access = AccessResult::allowed();
+    }
+    else {
+      $access = AccessResult::forbidden();
+    }
+    return $access->addCacheTags(["$entity_type:$bundle:$view_mode", "$entity_type:$bundle:$view_mode:$default"]);
+  }
+}
\ No newline at end of file
diff --git a/src/Access/ViewModeAccessCheck.php b/src/Access/ViewModeAccessCheck.php
new file mode 100644
index 0000000..db0ff21
--- /dev/null
+++ b/src/Access/ViewModeAccessCheck.php
@@ -0,0 +1,66 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\panelizer\Access\ViewModeAccessCheck.
+ */
+
+namespace Drupal\panelizer\Access;
+
+use Drupal\Core\Routing\Access\AccessInterface;
+use Drupal\Core\Routing\RouteMatchInterface;
+use Drupal\Core\Session\AccountInterface;
+use Drupal\field_ui\Access\ViewModeAccessCheck as FieldUIViewModeAccessCheck;
+use Symfony\Component\Routing\Route;
+
+/**
+ * An access check as an adapter around field_ui's custom access check.
+ */
+class ViewModeAccessCheck implements AccessInterface {
+
+  /**
+   * The decorated ViewModeAccessCheck from field_ui.
+   *
+   * @var \Drupal\field_ui\Access\ViewModeAccessCheck
+   */
+  protected $accessCheck;
+
+  /**
+   * ViewModeAccessCheck constructor.
+   *
+   * @param \Drupal\field_ui\Access\ViewModeAccessCheck $access_check
+   */
+  public function __construct(FieldUIViewModeAccessCheck $access_check) {
+    $this->accessCheck = $access_check;
+  }
+
+  /**
+   * Adapt the panelizer defaults access check to correspond to field ui.
+   *
+   * @param \Symfony\Component\Routing\Route $route
+   *   The original route definition.
+   * @param \Drupal\Core\Routing\RouteMatchInterface $route_match
+   *   The route matched.
+   * @param \Drupal\Core\Session\AccountInterface $account
+   *   The current user's account.
+   * @param string $machine_name
+   *   The machine name of the panelizer default.
+   *
+   * @return \Drupal\Core\Access\AccessResultInterface
+   * @throws \Exception
+   */
+  public function access(Route $route, RouteMatchInterface $route_match, AccountInterface $account, $machine_name) {
+    $parts = explode('__', $machine_name);
+    if (count($parts) != 4) {
+      throw new \Exception('The provided machine_name is not well formed.');
+    }
+    list($entity_type_id, $bundle, $view_mode) = $parts;
+    $defaults = [
+      'entity_type_id' => $entity_type_id,
+    ] + $route->getDefaults();
+    $route->setDefaults($defaults);
+    $route->setRequirement('_field_ui_view_mode_access', 'administer ' . $entity_type_id . ' display');
+    return $this->accessCheck->access($route, $route_match, $account, $view_mode, $bundle);
+  }
+
+}
diff --git a/src/Controller/PanelizerPanelsIPEController.php b/src/Controller/PanelizerPanelsIPEController.php
index d33ed64..7cebed9 100644
--- a/src/Controller/PanelizerPanelsIPEController.php
+++ b/src/Controller/PanelizerPanelsIPEController.php
@@ -57,16 +57,26 @@ class PanelizerPanelsIPEController extends ControllerBase {
    *   The entity.
    * @param string $view_mode
    *   The view mode.
-   * @param \Symfony\Component\HttpFoundation\Request $request
-   *   The request.
    *
    * @return \Symfony\Component\HttpFoundation\Response
    *   An empty response.
    *
    * @throws \Symfony\Component\HttpKernel\Exception\BadRequestHttpException
    */
-  public function revertToDefault(FieldableEntityInterface $entity, $view_mode, Request $request) {
-    $default = $request->get('default');
+  public function revertToDefault(FieldableEntityInterface $entity, $view_mode) {
+    // Get the bundle specific default display as a fallback.
+    $settings = $this->panelizer->getPanelizerSettings($entity->getEntityTypeId(), $entity->bundle(), $view_mode);
+    $default = $settings['default'];
+    // Check the entity for a documented default to which we should revert.
+    if ($entity->hasField('panelizer') && $entity->panelizer->first()) {
+      foreach ($entity->panelizer as $item) {
+        if ($item->view_mode == $view_mode && !empty($item->default)) {
+          $default = $item->default;
+          break;
+        }
+      }
+    }
+    // If we somehow ended up not having a default, throw an exception.
     if (empty($default)) {
       throw new BadRequestHttpException("Default name to revert to must be passed!");
     }
diff --git a/src/Form/PanelizerDefaultDelete.php b/src/Form/PanelizerDefaultDelete.php
index fe5503b..af039af 100644
--- a/src/Form/PanelizerDefaultDelete.php
+++ b/src/Form/PanelizerDefaultDelete.php
@@ -7,6 +7,7 @@
 
 namespace Drupal\panelizer\Form;
 
+use Drupal\Core\Cache\CacheTagsInvalidatorInterface;
 use Drupal\Core\Entity\EntityTypeManagerInterface;
 use Drupal\Core\Form\ConfirmFormBase;
 use Drupal\Core\Form\FormStateInterface;
@@ -67,6 +68,13 @@ class PanelizerDefaultDelete extends ConfirmFormBase {
   protected $panelsDisplayManager;
 
   /**
+   * The cache tag invalidator.
+   *
+   * @var \Drupal\Core\Cache\CacheTagsInvalidatorInterface
+   */
+  protected $invalidator;
+
+  /**
    * PanelizerDefaultDelete constructor.
    *
    * @param \Drupal\Core\Entity\EntityTypeManagerInterface $entity_type_manager
@@ -76,10 +84,11 @@ class PanelizerDefaultDelete extends ConfirmFormBase {
    * @param \Drupal\panels\PanelsDisplayManagerInterface $panels_display_manager
    *   The Panels display manager.
    */
-  public function __construct(EntityTypeManagerInterface $entity_type_manager, PanelizerInterface $panelizer, PanelsDisplayManagerInterface $panels_display_manager) {
+  public function __construct(EntityTypeManagerInterface $entity_type_manager, PanelizerInterface $panelizer, PanelsDisplayManagerInterface $panels_display_manager, CacheTagsInvalidatorInterface $invalidator) {
     $this->entityTypeManager = $entity_type_manager;
     $this->panelizer = $panelizer;
     $this->panelsDisplayManager = $panels_display_manager;
+    $this->invalidator = $invalidator;
   }
 
   /**
@@ -89,7 +98,8 @@ class PanelizerDefaultDelete extends ConfirmFormBase {
     return new static(
       $container->get('entity_type.manager'),
       $container->get('panelizer'),
-      $container->get('panels.display_manager')
+      $container->get('panels.display_manager'),
+      $container->get('cache_tags.invalidator')
     );
   }
 
@@ -155,6 +165,8 @@ class PanelizerDefaultDelete extends ConfirmFormBase {
     $display->setThirdPartySetting('panelizer', 'displays', $displays);
     $display->save();
     $form_state->setRedirectUrl($this->getCancelUrl());
+    $tag = "{$this->entityTypeId}:{$this->bundle}:{$this->viewMode}";
+    $this->invalidator->invalidateTags([$tag]);
   }
 
 }
diff --git a/src/Form/PanelizerDefaultSelect.php b/src/Form/PanelizerDefaultSelect.php
index e811996..01f15b1 100644
--- a/src/Form/PanelizerDefaultSelect.php
+++ b/src/Form/PanelizerDefaultSelect.php
@@ -7,6 +7,7 @@
 
 namespace Drupal\panelizer\Form;
 
+use Drupal\Core\Cache\CacheTagsInvalidatorInterface;
 use Drupal\Core\Form\ConfirmFormBase;
 use Drupal\Core\Form\FormStateInterface;
 use Drupal\Core\Url;
@@ -51,13 +52,21 @@ class PanelizerDefaultSelect extends ConfirmFormBase {
   protected $panelizer;
 
   /**
+   * The cache tag invalidator.
+   *
+   * @var \Drupal\Core\Cache\CacheTagsInvalidatorInterface
+   */
+  protected $invalidator;
+
+  /**
    * PanelizerDefaultSelect constructor.
    *
    * @param \Drupal\panelizer\PanelizerInterface $panelizer
    *   The Panelizer service.
    */
-  public function __construct(PanelizerInterface $panelizer) {
+  public function __construct(PanelizerInterface $panelizer, CacheTagsInvalidatorInterface $invalidator) {
     $this->panelizer = $panelizer;
+    $this->invalidator = $invalidator;
   }
 
   /**
@@ -65,7 +74,8 @@ class PanelizerDefaultSelect extends ConfirmFormBase {
    */
   public static function create(ContainerInterface $container) {
     return new static(
-      $container->get('panelizer')
+      $container->get('panelizer'),
+      $container->get('cache_tags.invalidator')
     );
   }
 
@@ -127,6 +137,8 @@ class PanelizerDefaultSelect extends ConfirmFormBase {
     $settings['default'] = $this->displayId;
     $this->panelizer->setPanelizerSettings($this->entityTypeId, $this->bundle, $this->viewMode, $settings, $display);
     $form_state->setRedirectUrl($this->getCancelUrl());
+    $tag = "{$this->entityTypeId}:{$this->bundle}:{$this->viewMode}";
+    $this->invalidator->invalidateTags([$tag]);
   }
 
 }
diff --git a/src/Panelizer.php b/src/Panelizer.php
index 163d965..5db8330 100644
--- a/src/Panelizer.php
+++ b/src/Panelizer.php
@@ -219,15 +219,25 @@ class Panelizer implements PanelizerInterface {
       }
       if (isset($values[$view_mode])) {
         $panelizer_item = $values[$view_mode];
+        // Check for a customized display first and use that if present.
+        if (!empty($panelizer_item->panels_display)) {
+          // @todo: validate schema after https://www.drupal.org/node/2392057 is fixed.
+          return $this->panelsManager->importDisplay($panelizer_item->panels_display, FALSE);
+        }
+        // If not customized, use the specified default.
         if (!empty($panelizer_item->default)) {
-          return $this->getDefaultPanelsDisplay($panelizer_item->default, $entity->getEntityTypeId(), $entity->bundle(), $view_mode, $display);
+          // If we're using this magic key use the settings default.
+          if ($panelizer_item->default == '__bundle_default__') {
+            $default = $settings['default'];
+          }
+          else {
+            $default = $panelizer_item->default;
+          }
+          return $this->getDefaultPanelsDisplay($default, $entity->getEntityTypeId(), $entity->bundle(), $view_mode, $display);
         }
-
-        // @todo: validate schema after https://www.drupal.org/node/2392057 is fixed.
-        return $this->panelsManager->importDisplay($panelizer_item->panels_display, FALSE);
       }
     }
-
+    // If the field has no input to give us, use the settings default.
     return $this->getDefaultPanelsDisplay($settings['default'], $entity->getEntityTypeId(), $entity->bundle(), $view_mode, $display);
   }
 
diff --git a/src/Plugin/Field/FieldWidget/PanelizerWidget.php b/src/Plugin/Field/FieldWidget/PanelizerWidget.php
index 0aa9326..5d6defd 100644
--- a/src/Plugin/Field/FieldWidget/PanelizerWidget.php
+++ b/src/Plugin/Field/FieldWidget/PanelizerWidget.php
@@ -87,15 +87,20 @@ class PanelizerWidget extends WidgetBase {
     foreach ($entity_view_modes as $view_mode => $view_mode_info) {
       $display = EntityViewDisplay::collectRenderDisplay($entity, $view_mode);
       $displays[$view_mode] = $display->getThirdPartySetting('panelizer', 'displays', []);
-      if (!isset($values[$view_mode])) {
+      // If we don't have a value, or the default is __bundle_default__ and our
+      // panels_display is empty, set the default to __bundle_default__.
+      if (!isset($values[$view_mode]) || ($values[$view_mode]['default'] == '__bundle_default__' && empty($values[$view_mode]['panels_display']))) {
         if ($display->getThirdPartySetting('panelizer', 'enable', FALSE)) {
           $values[$view_mode] = [
-            'default' => 'default',
+            'default' => '__bundle_default__',
             'panels_display' => [],
           ];
         }
       }
-      elseif (!empty($values[$view_mode]['default']) && !empty($values[$view_mode]['panels_display'])) {
+      // If our default isn't empty and is not set to __bundle_default__ and
+      // our panels_display is not empty, empty the panels_display so that the
+      // documented default will properly display.
+      elseif ((!empty($values[$view_mode]['default']) && $values[$view_mode]['default'] != '__bundle_default__') && !empty($values[$view_mode]['panels_display'])) {
         $values[$view_mode]['panels_display'] = [];
       }
     }
@@ -110,7 +115,11 @@ class PanelizerWidget extends WidgetBase {
 
       $settings = $this->getPanelizer()->getPanelizerSettings($entity_type_id, $entity->bundle(), $view_mode);
       if (!empty($settings['allow'])) {
-        $options = [];
+        // We default to this option when the user hasn't previous interacted
+        // with the field.
+        $options = [
+          '__bundle_default__' => $this->t('Current default display'),
+        ];
         foreach ($displays[$view_mode] as $machine_name => $panels_display) {
           $options[$machine_name] = $panels_display['label'];
         }
@@ -120,6 +129,8 @@ class PanelizerWidget extends WidgetBase {
           '#options' => $options,
           '#default_value' => $value['default'],
         ];
+        // If we have a value in panels_display, prevent the user from
+        // interacting with the widget for the view modes that are overridden.
         if (!empty($value['panels_display'])) {
           $element[$delta]['default']['#disabled'] = TRUE;
           $element[$delta]['default']['#options'][$value['default']] = $this->t('Custom Override');
diff --git a/src/Plugin/PanelsStorage/PanelizerFieldPanelsStorage.php b/src/Plugin/PanelsStorage/PanelizerFieldPanelsStorage.php
index 7ea92c8..5259f30 100644
--- a/src/Plugin/PanelsStorage/PanelizerFieldPanelsStorage.php
+++ b/src/Plugin/PanelsStorage/PanelizerFieldPanelsStorage.php
@@ -134,9 +134,21 @@ class PanelizerFieldPanelsStorage extends PanelsStorageBase implements Container
     $id = $panels_display->getStorageId();
     if ($entity = $this->loadEntity($id)) {
       list (,, $view_mode) = explode(':', $id);
+      // If we're dealing with an entity that has a documented default, we
+      // don't want to lose that information when we save our customizations.
+      // This enables us to revert to the correct default at a later date.
       if ($entity instanceof FieldableEntityInterface) {
+        $default = NULL;
+        if ($entity->hasField('panelizer') && $entity->panelizer->first()) {
+          foreach ($entity->panelizer as $item) {
+            if ($item->view_mode == $view_mode) {
+              $default = $item->default;
+              break;
+            }
+          }
+        }
         try {
-          $this->panelizer->setPanelsDisplay($entity, $view_mode, NULL, $panels_display);
+          $this->panelizer->setPanelsDisplay($entity, $view_mode, $default, $panels_display);
         }
         catch (PanelizerException $e) {
           // Translate to expected exception type.