diff -u b/core/modules/toolbar/src/Tests/ToolbarCacheContextsTest.php b/core/modules/toolbar/src/Tests/ToolbarCacheContextsTest.php
--- b/core/modules/toolbar/src/Tests/ToolbarCacheContextsTest.php
+++ b/core/modules/toolbar/src/Tests/ToolbarCacheContextsTest.php
@@ -27,125 +27,98 @@
    */
   public static $modules = ['toolbar', 'test_page_test'];
 
+  /**
+   * An authenticated user to use for testing.
+   *
+   * @var \Drupal\user\UserInterface
+   */
   protected $adminUser;
 
+  /**
+   * An authenticated user to use for testing.
+   *
+   * @var \Drupal\user\UserInterface
+   */
+  protected $adminUser2;
+
+  /**
+   * A list of default permissions for test users.
+   *
+   * @var array
+   */
+  protected $perms = [
+    'access toolbar',
+    'access administration pages',
+    'administer site configuration',
+  ];
+
+  /**
+   * {@inheritdoc}
+   */
   protected function setUp() {
     parent::setUp();
 
-    $perms = [
-      'access toolbar',
-      'access administration pages',
-      'administer site configuration',
-    ];
-    $this->adminUser = $this->drupalCreateUser($perms);
+    $this->adminUser = $this->drupalCreateUser($this->perms);
+    $this->adminUser2 = $this->drupalCreateUser($this->perms);
   }
 
   /**
    * Tests toolbar cache contexts.
    */
   public function testToolbarCacheContextsCaller() {
-    // Test with the default combination.
-    $anonymous_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-    ];
-    $authenticated_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-      'user',
-    ];
-    $this->testToolbarCacheContexts($anonymous_cache_contexts, $authenticated_cache_contexts);
+    // Test with default combination and permission to see toolbar.
+    $this->testToolbarCacheContexts(['user']);
 
-    // Test without user menu. User module is required so we have to
-    // manually remove the user toolbar.
+    // Test without user toolbar tab. User module is a required module so we have to
+    // manually remove the user toolbar tab.
     \Drupal::service('module_installer')->install(['toolbar_disable_user_toolbar']);
-    $anonymous_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-    ];
-    $authenticated_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-      'user.permissions',
-    ];
-    $this->testToolbarCacheContexts($anonymous_cache_contexts, $authenticated_cache_contexts);
+    $this->testToolbarCacheContexts(['user.permissions']);
 
     // Test with the toolbar and contextual enabled.
     \Drupal::service('module_installer')->install(['contextual']);
-
-    $anonymous_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-    ];
-    $authenticated_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-      'user.permissions',
-    ];
-    $this->testToolbarCacheContexts($anonymous_cache_contexts, $authenticated_cache_contexts);
+    $this->adminUser2 = $this->drupalCreateUser(array_merge($this->perms, ['access contextual links']));
+    $this->testToolbarCacheContexts(['user.permissions']);
     \Drupal::service('module_installer')->uninstall(['contextual']);
 
     // Test with the tour module enabled.
     \Drupal::service('module_installer')->install(['tour']);
-    $this->rebuildContainer();
-
-    $anonymous_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-    ];
-    $authenticated_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-      'user.permissions',
-    ];
-    $this->testToolbarCacheContexts($anonymous_cache_contexts, $authenticated_cache_contexts);
+    $this->adminUser2 = $this->drupalCreateUser(array_merge($this->perms, ['access tour']));
+    $this->testToolbarCacheContexts(['user.permissions']);
     \Drupal::service('module_installer')->uninstall(['tour']);
 
     // Test with shortcut module enabled.
     \Drupal::service('module_installer')->install(['shortcut']);
-
-    $anonymous_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-    ];
-    $authenticated_cache_contexts = [
-      'languages:language_interface',
-      'theme',
-      'user',
-    ];
-    $this->testToolbarCacheContexts($anonymous_cache_contexts, $authenticated_cache_contexts);
+    $this->adminUser2 = $this->drupalCreateUser(array_merge($this->perms, ['access shortcuts', 'administer shortcuts']));
+    $this->testToolbarCacheContexts(['user']);
   }
 
   /**
-   * Tests toolbar cache contexts with contextual toolbar.
+   * Tests that cache contexts are applied for both users.
    *
-   * @param array $anonymous_cache_contexts
-   *   Expected cache contexts for anonymous user.
-   * @param array $authenticated_cache_contexts
-   *   Expected cache contexts for authenticated user.
+   * @param array $cache_contexts
+   *   Expected cache contexts for both users.
    */
-  private function testToolbarCacheContexts(array $anonymous_cache_contexts, array $authenticated_cache_contexts) {
-    // All the caches has to be cleared since modules have changed.
+  private function testToolbarCacheContexts(array $cache_contexts) {
+    // Caches has to be rebuilt since modules might have added some routes.
     drupal_flush_all_caches();
-    // Make sure that user without permissions to see toolbar doesn't have
-    // user cache context.
-    $this->drupalGet('test-page');
-
-    // Assert that the toolbar is not present when user doesn't have
-    // permissions.
-    $this->assertNoRaw('id="toolbar-administration"');
 
-    $this->assertCacheContexts($anonymous_cache_contexts);
+    // Default cache contexts that should exist on all test cases.
+    $default_cache_contexts = [
+      'languages:language_interface',
+      'theme',
+    ];
+    $cache_contexts = array_merge($default_cache_contexts, $cache_contexts);
 
-    // User with a permission to see toolbar should have user context.
+    // Assert contexts for user1 which has only default permissions.
     $this->drupalLogin($this->adminUser);
     $this->drupalGet('test-page');
-
-    // Assert that the toolbar is present in the HTML.
-    $this->assertRaw('id="toolbar-administration"');
-
-    $this->assertCacheContexts($authenticated_cache_contexts);
+    $this->assertCacheContexts($cache_contexts);
     $this->drupalLogout();
+
+    // Assert contexts for user2 which has some additional permissions.
+    $this->drupalLogin($this->adminUser2);
+    $this->drupalGet('test-page');
+    $this->assertCacheContexts($cache_contexts);
   }
 
 }