diff --git a/panels_ipe/panels_ipe.services.yml b/panels_ipe/panels_ipe.services.yml
new file mode 100644
index 0000000..a10a616
--- /dev/null
+++ b/panels_ipe/panels_ipe.services.yml
@@ -0,0 +1,4 @@
+services:
+  plugin.manager.ipe_access:
+    class: Drupal\panels_ipe\Plugin\IPEAccessManager
+    parent: default_plugin_manager
diff --git a/panels_ipe/src/Annotation/IPEAccess.php b/panels_ipe/src/Annotation/IPEAccess.php
new file mode 100644
index 0000000..0239d8e
--- /dev/null
+++ b/panels_ipe/src/Annotation/IPEAccess.php
@@ -0,0 +1,38 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\panels_ipe\Annotation\IPEAccess.
+ */
+
+namespace Drupal\panels_ipe\Annotation;
+
+use Drupal\Component\Annotation\Plugin;
+
+/**
+ * Defines a IPE Access item annotation object.
+ *
+ * @see \Drupal\panels_ipe\Plugin\IPEAccessManager
+ * @see plugin_api
+ *
+ * @Annotation
+ */
+class IPEAccess extends Plugin {
+
+  /**
+   * The plugin ID.
+   *
+   * @var string
+   */
+  public $id;
+
+  /**
+   * The label of the plugin.
+   *
+   * @var \Drupal\Core\Annotation\Translation
+   *
+   * @ingroup plugin_translatable
+   */
+  public $label;
+
+}
diff --git a/panels_ipe/src/Plugin/DisplayBuilder/InPlaceEditorDisplayBuilder.php b/panels_ipe/src/Plugin/DisplayBuilder/InPlaceEditorDisplayBuilder.php
index 64b6dd7..d6dbba9 100644
--- a/panels_ipe/src/Plugin/DisplayBuilder/InPlaceEditorDisplayBuilder.php
+++ b/panels_ipe/src/Plugin/DisplayBuilder/InPlaceEditorDisplayBuilder.php
@@ -154,6 +154,9 @@ class InPlaceEditorDisplayBuilder extends StandardDisplayBuilder {
   public function build(PanelsDisplayVariant $panels_display) {
     // Check to see if the current user has permissions to use the IPE.
     $has_permission = $this->account->hasPermission('access panels in-place editing') && $this->panelsStorage->access($panels_display->getStorageType(), $panels_display->getStorageId(), 'update', $this->account)->isAllowed();
+    if ($has_permission) {
+      $has_permission = \Drupal::service('plugin.manager.ipe_access')->access($panels_display);
+    }
 
     // Attach the Panels In-place editor library based on permissions.
     if ($has_permission) {
diff --git a/panels_ipe/src/Plugin/IPEAccessBase.php b/panels_ipe/src/Plugin/IPEAccessBase.php
new file mode 100644
index 0000000..609e2a0
--- /dev/null
+++ b/panels_ipe/src/Plugin/IPEAccessBase.php
@@ -0,0 +1,12 @@
+<?php
+/**
+ * @file
+ * Contains \Drupal\panels_ipe\Plugin\IPEAccessBase.php
+ */
+
+namespace Drupal\panels_ipe\Plugin;
+
+
+use Drupal\Core\Plugin\PluginBase;
+
+abstract class IPEAccessBase extends PluginBase implements IPEAccessInterface {}
diff --git a/panels_ipe/src/Plugin/IPEAccessInterface.php b/panels_ipe/src/Plugin/IPEAccessInterface.php
new file mode 100644
index 0000000..996d61e
--- /dev/null
+++ b/panels_ipe/src/Plugin/IPEAccessInterface.php
@@ -0,0 +1,34 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\panels_ipe\Plugin\IPEAccessInterface.php
+ */
+
+namespace Drupal\panels_ipe\Plugin;
+
+use Drupal\Component\Plugin\PluginInspectionInterface;
+use Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant;
+
+/**
+ * Defines an interface for IPE Access plugins.
+ */
+interface IPEAccessInterface extends PluginInspectionInterface {
+
+  /**
+   * Provides logic to determine if a given plugin applies to a display.
+   *
+   * @param \Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant $display
+   *
+   * @return boolean
+   */
+  public function applies(PanelsDisplayVariant $display);
+
+  /**
+   * @param \Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant $display
+   *
+   * @return mixed
+   */
+  public function access(PanelsDisplayVariant $display);
+
+}
diff --git a/panels_ipe/src/Plugin/IPEAccessManager.php b/panels_ipe/src/Plugin/IPEAccessManager.php
new file mode 100644
index 0000000..f2d3479
--- /dev/null
+++ b/panels_ipe/src/Plugin/IPEAccessManager.php
@@ -0,0 +1,65 @@
+<?php
+
+/**
+ * @file
+ * Contains \Drupal\panels_ipe\Plugin\IPEAccessManager.
+ */
+
+namespace Drupal\panels_ipe\Plugin;
+
+use Drupal\Core\Plugin\DefaultPluginManager;
+use Drupal\Core\Cache\CacheBackendInterface;
+use Drupal\Core\Extension\ModuleHandlerInterface;
+use Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant;
+
+/**
+ * Provides the IPE Access plugin manager.
+ */
+class IPEAccessManager extends DefaultPluginManager implements IPEAccessManagerInterface {
+
+  /**
+   * Constructor for IPEAccessManager objects.
+   *
+   * @param \Traversable $namespaces
+   *   An object that implements \Traversable which contains the root paths
+   *   keyed by the corresponding namespace to look for plugin implementations.
+   * @param \Drupal\Core\Cache\CacheBackendInterface $cache_backend
+   *   Cache backend instance to use.
+   * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler
+   *   The module handler to invoke the alter hook with.
+   */
+  public function __construct(\Traversable $namespaces, CacheBackendInterface $cache_backend, ModuleHandlerInterface $module_handler) {
+    parent::__construct('Plugin/IPEAccess', $namespaces, $module_handler, 'Drupal\panels_ipe\Plugin\IPEAccessInterface', 'Drupal\panels_ipe\Annotation\IPEAccess');
+
+    $this->alterInfo('panels_ipe_ipe_access_info');
+    $this->setCacheBackend($cache_backend, 'panels_ipe_ipe_access_plugins');
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function applies(PanelsDisplayVariant $display) {
+    $applies = [];
+    foreach ($this->getDefinitions() as $plugin_id => $definition) {
+      /** @var \Drupal\panels_ipe\Plugin\IPEAccessInterface $plugin */
+      $plugin = $this->createInstance($plugin_id);
+      if ($plugin->applies($display)) {
+        $applies[$plugin_id] = $plugin;
+      }
+    }
+    return $applies;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function access(PanelsDisplayVariant $display) {
+    foreach ($this->applies($display) as $plugin_id => $plugin) {
+      if (!$plugin->access($display)) {
+        return FALSE;
+      }
+    }
+    return TRUE;
+  }
+
+}
diff --git a/panels_ipe/src/Plugin/IPEAccessManagerInterface.php b/panels_ipe/src/Plugin/IPEAccessManagerInterface.php
new file mode 100644
index 0000000..0024c25
--- /dev/null
+++ b/panels_ipe/src/Plugin/IPEAccessManagerInterface.php
@@ -0,0 +1,27 @@
+<?php
+/**
+ * @file
+ * Contains \Drupal\panels_ipe\Plugin\IPEAccessManagerInterface.php
+ */
+namespace Drupal\panels_ipe\Plugin;
+use Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant;
+
+
+/**
+ * Provides the IPE Access plugin manager.
+ */
+interface IPEAccessManagerInterface {
+  /**
+   * @param \Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant $display
+   *
+   * @return \Drupal\panels_ipe\Plugin\IPEAccessInterface[]
+   */
+  public function applies(PanelsDisplayVariant $display);
+
+  /**
+   * @param \Drupal\panels\Plugin\DisplayVariant\PanelsDisplayVariant $display
+   *
+   * @return bool
+   */
+  public function access(PanelsDisplayVariant $display);
+}