Use two cache entries for ChainedFast using atomic-exchange for the invalidation timestamp

Problem/Motivation

APCu writes need a lock() internally, which is slow and can bring a site, which is mistakenly writing too much to APCu down. (More unlikely after the cache tag clearing invalidating chainedfast has landed).

However APCu has a reader-lock atomic-exchange operation, which is WAY faster, because it does not need to lock the whole list, but only the individual item.

But in ChainedFast we 99% write the same item. Only the expiration timestamp has changed.

Steps to reproduce

- Make backend "dirty" all the time
- Loadtest umami
- See that apcu is becoming a bottleneck, because writes are piling up

Proposed resolution

- Instead of writing items again and again (which are 99% the same), we just update the timestamp
- And instead of writing to the same cache key again and again, we use a cheap CAS-lock to ensure it's only written once

I have written some rough code years ago, which I am hereby sharing with the community.

Unfortuantely the backend needs to be aware that it's used from a chained fast backend.

Here is the code for that backend (https://gist.github.com/LionsAd/25a60c064199a1c87e000cb7a2a4a73c):

<?php

namespace Drupal\Core\Cache;

use Drupal\Component\Assertion\Inspector;

/**
 * Stores cache items in the Alternative PHP Cache User Cache (APCu).
 */
class CFAwareApcuBackend extends ApcuBackend {

  /**
   * The base time to use.
   *
   * @var int
   */
  protected $baseTime;

  /**
   * {@inheritdoc}
   */
  public function __construct($bin, $site_prefix, CacheTagsChecksumInterface $checksum_provider) {
    parent::__construct($bin, $site_prefix, $checksum_provider);
    $this->baseTime = strtotime('midnight first day of this month') + 2;
  }

  /**
   * {@inheritdoc}
   */
  public function get($cid, $allow_invalid = FALSE) {
    $cids = [$cid];
    $data = $this->getMultiple($cids, $allow_invalid);
    return !empty($data[$cid]) ? $data[$cid] : FALSE;
  }

  /**
   * {@inheritdoc}
   */
  public function getMultiple(&$cids, $allow_invalid = FALSE) {
    $start = microtime(TRUE);
    $cids_copy = $cids;

    // Translate the requested cache item IDs to APCu keys.
    $map = [];
    $extra_map = [];

    foreach ($cids as $cid) {
      $key = $this->getApcuKey($cid);
      $map[$key] = $cid;

      $created_key = $key . ':apcu_created_time';
      $lock_key = $key . ':apcu_update_lock';

      $extra_map[$created_key] = $key;
      $extra_map[$lock_key] = $key;
    }

    $result = apcu_fetch(array_keys($map+$extra_map));
    $cache = [];
    if ($result) {
      foreach ($result as $key => $item) {
        // Ignore extra keys.
        if (isset($extra_map[$key])) {
          continue;
        }

        // Prepare item.
        $item = $this->prepareItem($item, $allow_invalid);
        if (!$item) {
          continue;
        }

        // Store APCu data in the item.
        $item->apcu_data = [];

        // Overwrite created value from APCu.
        $created_key = $key . ':apcu_created_time';

        $item->created = 0;
        if (isset($result[$created_key])) {
          $item->apcu_data[$created_key] = $result[$created_key];
          $item->created = $result[$created_key] / 1000 + (float) $this->baseTime;
        }

        // Store lock data values from APCu in the item.
        $lock_key = $key . ':apcu_update_lock';

        if (isset($result[$lock_key])) {
          $item->apcu_data[$lock_key] = $result[$lock_key];
        }

        $cache[$map[$key]] = $item;
      }
    }
    unset($result);

    $cids = array_diff($cids, array_keys($cache));
    return $cache;
  }

  /**
   * {@inheritdoc}
   */
  public function set($cid, $data, $expire = CacheBackendInterface::CACHE_PERMANENT, array $tags = [], $items = []) {
    assert(Inspector::assertAllStrings($tags), 'Cache tags must be strings.');

    $created_microtime = round(microtime(TRUE), 3);
    $created = (int) (($created_microtime - $this->baseTime) * 1000);

    // Fetch data, write lock and created values.
    $key = $this->getApcuKey($cid);
    $created_key = $key . ':apcu_created_time';
    $lock_key = $key . ':apcu_update_lock';

    // Do we have a valid old item?
    $old_item = FALSE;
    if (isset($items[$cid]) && $items[$cid]->valid) {
      $old_item = $items[$cid];
      $old_result = $old_item->apcu_data;
    }
    else {
      $old_result = apcu_fetch([$key, $created_key, $lock_key]);
      if (isset($old_result[$key])) {
        $old_item = $this->prepareItem($old_result[$key], FALSE);
      }
    }

    // Ensure created key exists.
    $old_created = isset($old_result[$created_key]) ? $old_result[$created_key] : 0;
    if (!isset($old_result[$created_key])) {
      apcu_add($created_key, 0);
    }

    // Is the old item data still valid?
    if ($old_item && is_string($old_item->data) && $old_item->data === $data) {
      // Just update the created timestamp.
      apcu_cas($created_key, $old_created, $created);
      return;
    }

    // Ensure lock key exists.
    $old_lock_value = isset($old_result[$lock_key]) ? $old_result[$lock_key] : 0;
    if (!isset($old_result[$lock_key])) {
      apcu_add($lock_key, 0);
    }

    // See if we win the race to update the item.
    $lock_value = mt_rand(0, 2147483647);
    if (apcu_cas($lock_key, $old_lock_value, $lock_value)) {
      parent::set($cid, $data, $expire, $tags);

      // If this fails, someone else has updated the item already.
      apcu_cas($created_key, $old_created, $created);
    }
  }

  /**
   * {@inheritdoc}
   */
  public function setMultiple(array $items = []) {
    foreach ($items as $cid => $item) {
      $this->set($cid, $item['data'], isset($item['expire']) ? $item['expire'] : CacheBackendInterface::CACHE_PERMANENT, isset($item['tags']) ? $item['tags'] : []);
    }
  }

  /**
   * {@inheritdoc}
   */
  protected function getIterator($search = NULL, $format = APC_ITER_ALL, $chunk_size = 100, $list = APC_LIST_ACTIVE) {
    throw new \Exception("This should not be called as it can deadlock the APCu cache.");
  }

}

And here is the hacked patch for the ChainedFast backend (https://gist.github.com/LionsAd/60b2fa2fcc602405010c1b075b1bb41a):

diff --git a/web/core/lib/Drupal/Core/Cache/ApcuBackendFactory.php b/web/core/lib/Drupal/Core/Cache/ApcuBackendFactory.php
--- a/web/core/lib/Drupal/Core/Cache/ApcuBackendFactory.php
+++ b/web/core/lib/Drupal/Core/Cache/ApcuBackendFactory.php
@@ -46,6 +46,9 @@ public function __construct($root, $site_path, CacheTagsChecksumInterface $check
     else {
       $this->backendClass = 'Drupal\Core\Cache\Apcu4Backend';
     }
+
+    $this->backendClass = 'Drupal\Core\Cache\CFAwareApcuBackend';
   }
 
   /**
diff --git a/web/core/lib/Drupal/Core/Cache/ChainedFastBackend.php b/web/core/lib/Drupal/Core/Cache/ChainedFastBackend.php
--- a/web/core/lib/Drupal/Core/Cache/ChainedFastBackend.php
+++ b/web/core/lib/Drupal/Core/Cache/ChainedFastBackend.php
@@ -166,7 +166,10 @@ public function getMultiple(&$cids, $allow_invalid = FALSE) {
         $cache[$item->cid] = $item;
         // Don't write the cache tags to the fast backend as any cache tag
         // invalidation results in an invalidation of the whole fast backend.
-        $this->fastBackend->set($item->cid, $item->data, $item->expire);
+        if (!$allow_invalid || $item->valid) {
+          // @todo Add interface check.
+          $this->fastBackend->set($item->cid, $item->data, $item->expire, [], $items);
+        }
       }
     }

The main difference is that the fast backend is getting the context of all the $items that are stored, so that it can lookup the item existing in $items. So set() is getting an additional argument in this interface.

The reason that it needs the items or original $item to look at the apcu_data, which in the case that the data came from the ChainedFastBackend in the first place can avoid the read lookup, because only the new timestamp needs to be compared.

Remaining tasks

User interface changes

API changes

Data model changes

Release notes snippet