diff --git a/permissions_by_term.module b/permissions_by_term.module
index 5886d98..df2ab53 100644
--- a/permissions_by_term.module
+++ b/permissions_by_term.module
@@ -85,13 +85,43 @@ function permissions_by_term_submit($form, FormState $formState) {
   $termId = $formState->getFormObject()->getEntity()->id();
   /* @var \Drupal\permissions_by_term\AccessStorage $access_storage */
   $access_storage = \Drupal::service('permissions_by_term.access_storage');
-  $access_storage->saveTermPermissions($formState, $termId);
-  /**
-   * @var \Drupal\permissions_by_term\NodeAccess $nodeAccess
-   */
-  $nodeAccess = \Drupal::service('permissions_by_term.node_access');
-  $nodeAccess->rebuildByTid($termId, $formState);
-  Cache::invalidateTags(['search_index:node_search']);
+  $access_update = $access_storage->saveTermPermissions($formState, $termId);
+
+  // Check if we need to rebuild node_access by term id
+  $update_node_access = false;
+
+  // Has anything has changed?
+  foreach($access_update as $values) {
+    if(!empty($values)) {
+      $update_node_access = true;
+      break;
+    }
+  }
+  // Are there existing role term permissions?
+  if($update_node_access == false) {
+    $role_permissions = $access_storage->getExistingRoleTermPermissionsByTid($termId);
+    if(!empty($role_permissions)) {
+      $update_node_access = true;
+    }
+  }
+  // Are there existing user term permissions?
+  if($update_node_access == false) {
+    $role_permissions = $access_storage->getExistingUserTermPermissionsByTid($termId);
+    if(!empty($role_permissions)) {
+      $update_node_access = true;
+    }
+  }
+
+  // Do we rebuild node access?
+  if($update_node_access == true) {
+    die('w00');
+    /**
+     * @var \Drupal\permissions_by_term\NodeAccess $nodeAccess
+     */
+    $nodeAccess = \Drupal::service('permissions_by_term.node_access');
+    $nodeAccess->rebuildByTid($termId, $formState);
+    Cache::invalidateTags(['search_index:node_search']);
+  }
 }
 
 /**