diff --git c/core/modules/update/lib/Drupal/update/Controller/UpdateController.php w/core/modules/update/lib/Drupal/update/Controller/UpdateController.php index 6816466..64e782c 100644 --- c/core/modules/update/lib/Drupal/update/Controller/UpdateController.php +++ w/core/modules/update/lib/Drupal/update/Controller/UpdateController.php @@ -8,7 +8,7 @@ namespace Drupal\update\Controller; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; -use Drupal\Core\Extension\ModuleHandlerInterface; +use Drupal\update\UpdateCompareManager; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -17,29 +17,27 @@ class UpdateController implements ContainerInjectionInterface { /** - * Module handler service. + * Update comparison service * - * @var \Drupal\Core\Extension\ModuleHandlerInterface + * @var \Drupal\update\UpdateCompareManager */ - protected $moduleHandler; + protected $updateCompareManager; /** * Constructs update status data. * - * @param \Drupal\Core\Extension\ModuleHandlerInterface $module_handler - * Module Handler Service. + * @param \Drupal\update\UpdateCompareManager $updateCompareManager + * Update Comparison Service. */ - public function __construct(ModuleHandlerInterface $module_handler) { - $this->moduleHandler = $module_handler; + public function __construct(UpdateCompareManager $updateCompareManager) { + $this->updateCompareManager = $updateCompareManager; } /** * {@inheritdoc} */ public static function create(ContainerInterface $container) { - return new static( - $container->get('module_handler') - ); + return new static($container->get('update.compare')); } /** @@ -53,8 +51,7 @@ public function updateStatus() { '#theme' => 'update_report' ); if ($available = update_get_available(TRUE)) { - $this->moduleHandler->loadInclude('update', 'compare.inc'); - $build['#data'] = update_calculate_project_data($available); + $build['#data'] = $this->updateCompareManager->calculateProjectData($available); } else { $build['#data'] = _update_no_data(); diff --git c/core/modules/update/lib/Drupal/update/Controller/UpdateFetchController.php w/core/modules/update/lib/Drupal/update/Controller/UpdateFetchController.php new file mode 100644 index 0000000..3e4b47e --- /dev/null +++ w/core/modules/update/lib/Drupal/update/Controller/UpdateFetchController.php @@ -0,0 +1,114 @@ +get('update.fetch')); + } + + /** + * Constructs a UpdateFetchController object. + * + * @param \Drupal\update\UpdateFetchManager $updateFetchManager + * Update Fetch Service. + */ + public function __construct(UpdateFetchManager $updateFetchManager) { + $this->updateFetchManager = $updateFetchManager; + } + + /** + * Checks for updates and displays the update status report. + * + * Manually checks the update status without the use of cron. + * + * @see update_menu() + */ + public function updateManualStatus() { + $this->updateFetchManager->refresh(); + $batch = array( + 'operations' => array( + array('\Drupal\update\Controller\UpdateFetchController::fetchDataBatch', array()), + ), + 'finished' => 'update_fetch_data_finished', + 'title' => t('Checking available update data'), + 'progress_message' => t('Trying to check available update data ...'), + 'error_message' => t('Error checking available update data.'), + ); + batch_set($batch); + return batch_process('admin/reports/updates'); + } + + /** + * Batch callback: Processes a step in batch for fetching available update data. + * + * @param array $context + * Reference to an array used for Batch API storage. + */ + public static function fetchDataBatch(array &$context) { + $queue = \Drupal::queue('update_fetch_tasks'); + if (empty($context['sandbox']['max'])) { + $context['finished'] = 0; + $context['sandbox']['max'] = $queue->numberOfItems(); + $context['sandbox']['progress'] = 0; + $context['message'] = t('Checking available update data ...'); + $context['results']['updated'] = 0; + $context['results']['failures'] = 0; + $context['results']['processed'] = 0; + } + + // Grab another item from the fetch queue. + for ($i = 0; $i < 5; $i++) { + if ($item = $queue->claimItem()) { + if (\Drupal::service('update.fetch')->processFetchTask($item->data)) { + $context['results']['updated']++; + $context['message'] = t('Checked available update data for %title.', array('%title' => $item->data['info']['name'])); + } + else { + $context['message'] = t('Failed to check available update data for %title.', array('%title' => $item->data['info']['name'])); + $context['results']['failures']++; + } + $context['sandbox']['progress']++; + $context['results']['processed']++; + $context['finished'] = $context['sandbox']['progress'] / $context['sandbox']['max']; + $queue->deleteItem($item); + } + else { + // If the queue is currently empty, we're done. It's possible that + // another thread might have added new fetch tasks while we were + // processing this batch. In that case, the usual 'finished' math could + // get confused, since we'd end up processing more tasks that we thought + // we had when we started and initialized 'max' with numberOfItems(). By + // forcing 'finished' to be exactly 1 here, we ensure that batch + // processing is terminated. + $context['finished'] = 1; + return; + } + } + } + +} diff --git c/core/modules/update/lib/Drupal/update/Tests/UpdateContribTest.php w/core/modules/update/lib/Drupal/update/Tests/UpdateContribTest.php index a184092..8825d38 100644 --- c/core/modules/update/lib/Drupal/update/Tests/UpdateContribTest.php +++ w/core/modules/update/lib/Drupal/update/Tests/UpdateContribTest.php @@ -7,6 +7,7 @@ namespace Drupal\update\Tests; +use Drupal\update\UpdateCompareManager; use Drupal\Core\Utility\ProjectInfo; /** @@ -268,7 +269,7 @@ function testUpdateShowDisabledThemes() { * Tests updates with a hidden base theme. */ function testUpdateHiddenBaseTheme() { - module_load_include('compare.inc', 'update'); + $compareManager = new UpdateCompareManager($this->container->get('module_handler'), $this->container->get('keyvalue.expirable')); // Enable the subtheme. theme_enable(array('update_test_subtheme')); diff --git c/core/modules/update/lib/Drupal/update/Tests/UpdateCoreTest.php w/core/modules/update/lib/Drupal/update/Tests/UpdateCoreTest.php index f0c55be..f6514c4 100644 --- c/core/modules/update/lib/Drupal/update/Tests/UpdateCoreTest.php +++ w/core/modules/update/lib/Drupal/update/Tests/UpdateCoreTest.php @@ -7,6 +7,7 @@ namespace Drupal\update\Tests; + /** * Tests behavior related to discovering and listing updates to Drupal core. */ @@ -195,26 +196,27 @@ function testServiceUnavailable() { * Tests that exactly one fetch task per project is created and not more. */ function testFetchTasks() { + $service = $this->container->get('update.fetch'); $projecta = array( 'name' => 'aaa_update_test', ); $projectb = array( 'name' => 'bbb_update_test', ); - $queue = \Drupal::queue('update_fetch_tasks'); + $queue = $this->container->get('queue')->get('update_fetch_tasks'); $this->assertEqual($queue->numberOfItems(), 0, 'Queue is empty'); - update_create_fetch_task($projecta); + $service->createFetchTask($projecta); $this->assertEqual($queue->numberOfItems(), 1, 'Queue contains one item'); - update_create_fetch_task($projectb); + $service->createFetchTask($projectb); $this->assertEqual($queue->numberOfItems(), 2, 'Queue contains two items'); // Try to add project a again. - update_create_fetch_task($projecta); + $service->createFetchTask($projecta); $this->assertEqual($queue->numberOfItems(), 2, 'Queue still contains two items'); // Clear storage and try again. update_storage_clear(); - drupal_static_reset('_update_create_fetch_task'); - update_create_fetch_task($projecta); + $service->clearFetchTasks(); + $service->createFetchTask($projecta); $this->assertEqual($queue->numberOfItems(), 2, 'Queue contains two items'); } diff --git c/core/modules/update/lib/Drupal/update/Tests/UpdateCoreUnitTest.php w/core/modules/update/lib/Drupal/update/Tests/UpdateCoreUnitTest.php index 89c41c2..14839a7 100644 --- c/core/modules/update/lib/Drupal/update/Tests/UpdateCoreUnitTest.php +++ w/core/modules/update/lib/Drupal/update/Tests/UpdateCoreUnitTest.php @@ -7,12 +7,15 @@ namespace Drupal\update\Tests; -use Drupal\simpletest\UnitTestBase; +use Drupal\simpletest\DrupalUnitTestBase; +use Drupal\update\UpdateCompareManager; +use Drupal\update\UpdateFetchManager; +use Symfony\Component\DependencyInjection\ContainerInterface; /** * Tests update functionality unrelated to the database. */ -class UpdateCoreUnitTest extends UnitTestBase { +class UpdateCoreUnitTest extends DrupalUnitTestBase { /** * Modules to enable. @@ -29,15 +32,19 @@ public static function getInfo() { ); } - function setUp() { - parent::setUp(); - module_load_include('inc', 'update', 'update.fetch'); - } - /** * Tests that _update_build_fetch_url() builds the URL correctly. */ function testUpdateBuildFetchUrl() { + $service = new UpdateFetchManager( + $this->container->get('config.factory'), + $this->container->get('queue'), + $this->container->get('keyvalue'), + $this->container->get('keyvalue.expirable'), + $this->container->get('state'), + $this->container->get('http_default_client'), + new UpdateCompareManager($this->container->get('module_handler'), $this->container->get('keyvalue.expirable')) + ); //first test that we didn't break the trivial case $project['name'] = 'update_test'; $project['project_type'] = ''; @@ -61,7 +68,7 @@ function testUpdateBuildFetchUrl() { $expected = 'http://www.example.com/' . $project['name'] . '/' . \Drupal::CORE_COMPATIBILITY; $expected .= '?site_key=site_key'; $expected .= '&list=' . rawurlencode('module1,module2'); - $url = _update_build_fetch_url($project, $site_key); + $url = $service->buildFetchUrl($project, $site_key); $this->assertEqual($url, $expected, "When site_key provided, '$url' should be '$expected'."); // http://drupal.org/node/1481156 test incorrect logic when URL contains @@ -70,8 +77,7 @@ function testUpdateBuildFetchUrl() { $expected = 'http://www.example.com/?project=/' . $project['name'] . '/' . \Drupal::CORE_COMPATIBILITY; $expected .= '&site_key=site_key'; $expected .= '&list=' . rawurlencode('module1,module2'); - $url = _update_build_fetch_url($project, $site_key); + $url = $service->buildFetchUrl($project, $site_key); $this->assertEqual($url, $expected, "When ? is present, '$url' should be '$expected'."); - } } diff --git c/core/modules/update/lib/Drupal/update/UpdateCompareManager.php w/core/modules/update/lib/Drupal/update/UpdateCompareManager.php new file mode 100644 index 0000000..46954e6 --- /dev/null +++ w/core/modules/update/lib/Drupal/update/UpdateCompareManager.php @@ -0,0 +1,664 @@ +moduleHandler = $moduleHandler; + $this->updateStore = $keyValueExpirable->get('update'); + } + + /** + * Fetches an array of installed and enabled projects. + * + * This is only responsible for generating an array of projects (taking into + * account projects that include more than one module or theme). Other + * information like the specific version and install type (official release, + * dev snapshot, etc) is handled later in update_process_project_info() since + * that logic is only required when preparing the status report, not for + * fetching the available release data. + * + * This array is fairly expensive to construct, since it involves a lot of disk + * I/O, so we store the results. However, since this is not the data about + * available updates fetched from the network, it is acceptable to invalidate it + * somewhat quickly. If we keep this data for very long, site administrators are + * more likely to see incorrect results if they upgrade to a newer version of a + * module or theme but do not visit certain pages that automatically clear this + * data. + * + * @return array + * An associative array of currently enabled projects keyed by the + * machine-readable project short name. Each project contains: + * - name: The machine-readable project short name. + * - info: An array with values from the main .info.yml file for this project. + * - name: The human-readable name of the project. + * - package: The package that the project is grouped under. + * - version: The version of the project. + * - project: The Drupal.org project name. + * - datestamp: The date stamp of the project's main .info.yml file. + * - _info_file_ctime: The maximum file change time for all of the .info.yml + * files included in this project. + * - datestamp: The date stamp when the project was released, if known. + * - includes: An associative array containing all projects included with this + * project, keyed by the machine-readable short name with the human-readable + * name as value. + * - project_type: The type of project. Allowed values are 'module' and + * 'theme'. + * - project_status: This indicates if the project is enabled and will always + * be TRUE, as the function only returns enabled projects. + * - sub_themes: If the project is a theme it contains an associative array of + * all sub-themes. + * - base_themes: If the project is a theme it contains an associative array + * of all base-themes. + * + * @see UpdateCompareManager::processProjectInfo() + * @see UpdateCompareManager::calculateProjectData() + * @see UpdateCompareManager::projectStorage() + */ + public function getProjects() { + if (empty($this->projects)) { + // Retrieve the projects from storage, if present. + $this->projects = $this->projectStorage('update_project_projects'); + if (empty($this->projects)) { + // Still empty, so we have to rebuild. + $this->projects = array(); + $module_data = system_rebuild_module_data(); + $theme_data = system_rebuild_theme_data(); + $project_info = new ProjectInfo(); + $project_info->processInfoList($this->projects, $module_data, 'module', TRUE); + $project_info->processInfoList($this->projects, $theme_data, 'theme', TRUE); + if (config('update.settings')->get('check.disabled_extensions')) { + $project_info->processInfoList($this->projects, $module_data, 'module', FALSE); + $project_info->processInfoList($this->projects, $theme_data, 'theme', FALSE); + } + // Allow other modules to alter projects before fetching and comparing. + $this->moduleHandler->alter('update_projects', $this->projects); + // Store the site's project data for at most 1 hour. + $this->updateStore->setWithExpire('update_project_projects', $this->projects, 3600); + } + } + return $this->projects; + } + + /** + * Determines version and type information for currently installed projects. + * + * Processes the list of projects on the system to figure out the currently + * installed versions, and other information that is required before we can + * compare against the available releases to produce the status report. + * + * @param array $projects + * Array of project information from getProjects(). + */ + public function processProjectInfo(array &$projects) { + foreach ($projects as $key => $project) { + // Assume an official release until we see otherwise. + $install_type = 'official'; + + $info = $project['info']; + + if (isset($info['version'])) { + // Check for development snapshots + if (preg_match('@(dev|HEAD)@', $info['version'])) { + $install_type = 'dev'; + } + + // Figure out what the currently installed major version is. We need + // to handle both contribution (e.g. "5.x-1.3", major = 1) and core + // (e.g. "5.1", major = 5) version strings. + $matches = array(); + if (preg_match('/^(\d+\.x-)?(\d+)\..*$/', $info['version'], $matches)) { + $info['major'] = $matches[2]; + } + elseif (!isset($info['major'])) { + // This would only happen for version strings that don't follow the + // drupal.org convention. We let contribs define "major" in their + // .info.yml in this case, and only if that's missing would we hit this. + $info['major'] = -1; + } + } + else { + // No version info available at all. + $install_type = 'unknown'; + $info['version'] = t('Unknown'); + $info['major'] = -1; + } + + // Finally, save the results we care about into the $projects array. + $projects[$key]['existing_version'] = $info['version']; + $projects[$key]['existing_major'] = $info['major']; + $projects[$key]['install_type'] = $install_type; + } + } + + /** + * Calculates the current update status of all projects on the site. + * + * The results of this function are expensive to compute, especially on sites + * with lots of modules or themes, since it involves a lot of comparisons and + * other operations. Therefore, we store the results. However, since this is not + * the data about available updates fetched from the network, it is ok to + * invalidate it somewhat quickly. If we keep this data for very long, site + * administrators are more likely to see incorrect results if they upgrade to a + * newer version of a module or theme but do not visit certain pages that + * automatically clear this. + * + * @param array $available + * Data about available project releases. + * + * @return array + * An array of installed projects with current update status information. + * + * @see update_get_available() + * @see UpdateCompareManager::getProjects() + * @see UpdateCompareManager::processProjectInfo() + * @see UpdateCompareManager::projectStorage() + */ + public function calculateProjectData(array $available) { + // Retrieve the projects from storage, if present. + $projects = $this->projectStorage('update_project_data'); + // If $projects is empty, then the data must be rebuilt. + // Otherwise, return the data and skip the rest of the function. + if (!empty($projects)) { + return $projects; + } + $projects = $this->getProjects(); + $this->processProjectInfo($projects); + foreach ($projects as $project => $project_info) { + if (isset($available[$project])) { + $this->calculateProjectUpdateStatus($projects[$project], $available[$project]); + } + else { + $projects[$project]['status'] = UPDATE_UNKNOWN; + $projects[$project]['reason'] = t('No available releases found'); + } + } + // Give other modules a chance to alter the status (for example, to allow a + // contrib module to provide fine-grained settings to ignore specific + // projects or releases). + $this->moduleHandler->alter('update_status', $projects); + + // Store the site's update status for at most 1 hour. + $this->updateStore->setWithExpire('update_project_data', $projects, 3600); + return $projects; + } + + /** + * Calculates the current update status of a specific project. + * + * This function is the heart of the update status feature. For each project it + * is invoked with, it first checks if the project has been flagged with a + * special status like "unsupported" or "insecure", or if the project node + * itself has been unpublished. In any of those cases, the project is marked + * with an error and the next project is considered. + * + * If the project itself is valid, the function decides what major release + * series to consider. The project defines what the currently supported major + * versions are for each version of core, so the first step is to make sure the + * current version is still supported. If so, that's the target version. If the + * current version is unsupported, the project maintainer's recommended major + * version is used. There's also a check to make sure that this function never + * recommends an earlier release than the currently installed major version. + * + * Given a target major version, the available releases are scanned looking for + * the specific release to recommend (avoiding beta releases and development + * snapshots if possible). For the target major version, the highest patch level + * is found. If there is a release at that patch level with no extra ("beta", + * etc.), then the release at that patch level with the most recent release date + * is recommended. If every release at that patch level has extra (only betas), + * then the latest release from the previous patch level is recommended. For + * example: + * + * - 1.6-bugfix <-- recommended version because 1.6 already exists. + * - 1.6 + * + * or + * + * - 1.6-beta + * - 1.5 <-- recommended version because no 1.6 exists. + * - 1.4 + * + * Also, the latest release from the same major version is looked for, even beta + * releases, to display to the user as the "Latest version" option. + * Additionally, the latest official release from any higher major versions that + * have been released is searched for to provide a set of "Also available" + * options. + * + * Finally, and most importantly, the release history continues to be scanned + * until the currently installed release is reached, searching for anything + * marked as a security update. If any security updates have been found between + * the recommended release and the installed version, all of the releases that + * included a security fix are recorded so that the site administrator can be + * warned their site is insecure, and links pointing to the release notes for + * each security update can be included (which, in turn, will link to the + * official security announcements for each vulnerability). + * + * This function relies on the fact that the .xml release history data comes + * sorted based on major version and patch level, then finally by release date + * if there are multiple releases such as betas from the same major.patch + * version (e.g., 5.x-1.5-beta1, 5.x-1.5-beta2, and 5.x-1.5). Development + * snapshots for a given major version are always listed last. + * + * @param array $project_data + * An array containing information about a specific project. + * @param array $available + * Data about available project releases of a specific project. + */ + public function calculateProjectUpdateStatus(array &$project_data, array $available) { + foreach (array('title', 'link') as $attribute) { + if (!isset($project_data[$attribute]) && isset($available[$attribute])) { + $project_data[$attribute] = $available[$attribute]; + } + } + + // If the project status is marked as something bad, there's nothing else + // to consider. + if (isset($available['project_status'])) { + switch ($available['project_status']) { + case 'insecure': + $project_data['status'] = UPDATE_NOT_SECURE; + if (empty($project_data['extra'])) { + $project_data['extra'] = array(); + } + $project_data['extra'][] = array( + 'class' => array('project-not-secure'), + 'label' => t('Project not secure'), + 'data' => t('This project has been labeled insecure by the Drupal security team, and is no longer available for download. Immediately disabling everything included by this project is strongly recommended!'), + ); + break; + case 'unpublished': + case 'revoked': + $project_data['status'] = UPDATE_REVOKED; + if (empty($project_data['extra'])) { + $project_data['extra'] = array(); + } + $project_data['extra'][] = array( + 'class' => array('project-revoked'), + 'label' => t('Project revoked'), + 'data' => t('This project has been revoked, and is no longer available for download. Disabling everything included by this project is strongly recommended!'), + ); + break; + case 'unsupported': + $project_data['status'] = UPDATE_NOT_SUPPORTED; + if (empty($project_data['extra'])) { + $project_data['extra'] = array(); + } + $project_data['extra'][] = array( + 'class' => array('project-not-supported'), + 'label' => t('Project not supported'), + 'data' => t('This project is no longer supported, and is no longer available for download. Disabling everything included by this project is strongly recommended!'), + ); + break; + case 'not-fetched': + $project_data['status'] = UPDATE_NOT_FETCHED; + $project_data['reason'] = t('Failed to get available update data.'); + break; + + default: + // Assume anything else (e.g. 'published') is valid and we should + // perform the rest of the logic in this function. + break; + } + } + + if (!empty($project_data['status'])) { + // We already know the status for this project, so there's nothing else to + // compute. Record the project status into $project_data and we're done. + $project_data['project_status'] = $available['project_status']; + return; + } + + // Figure out the target major version. + $existing_major = $project_data['existing_major']; + $supported_majors = array(); + if (isset($available['supported_majors'])) { + $supported_majors = explode(',', $available['supported_majors']); + } + elseif (isset($available['default_major'])) { + // Older release history XML file without supported or recommended. + $supported_majors[] = $available['default_major']; + } + + if (in_array($existing_major, $supported_majors)) { + // Still supported, stay at the current major version. + $target_major = $existing_major; + } + elseif (isset($available['recommended_major'])) { + // Since 'recommended_major' is defined, we know this is the new XML + // format. Therefore, we know the current release is unsupported since + // its major version was not in the 'supported_majors' list. We should + // find the best release from the recommended major version. + $target_major = $available['recommended_major']; + $project_data['status'] = UPDATE_NOT_SUPPORTED; + } + elseif (isset($available['default_major'])) { + // Older release history XML file without recommended, so recommend + // the currently defined "default_major" version. + $target_major = $available['default_major']; + } + else { + // Malformed XML file? Stick with the current version. + $target_major = $existing_major; + } + + // Make sure we never tell the admin to downgrade. If we recommended an + // earlier version than the one they're running, they'd face an + // impossible data migration problem, since Drupal never supports a DB + // downgrade path. In the unfortunate case that what they're running is + // unsupported, and there's nothing newer for them to upgrade to, we + // can't print out a "Recommended version", but just have to tell them + // what they have is unsupported and let them figure it out. + $target_major = max($existing_major, $target_major); + + $release_patch_changed = ''; + $patch = ''; + + // If the project is marked as UPDATE_FETCH_PENDING, it means that the + // data we currently have (if any) is stale, and we've got a task queued + // up to (re)fetch the data. In that case, we mark it as such, merge in + // whatever data we have (e.g. project title and link), and move on. + if (!empty($available['fetch_status']) && $available['fetch_status'] == UPDATE_FETCH_PENDING) { + $project_data['status'] = UPDATE_FETCH_PENDING; + $project_data['reason'] = t('No available update data'); + $project_data['fetch_status'] = $available['fetch_status']; + return; + } + + // Defend ourselves from XML history files that contain no releases. + if (empty($available['releases'])) { + $project_data['status'] = UPDATE_UNKNOWN; + $project_data['reason'] = t('No available releases found'); + return; + } + foreach ($available['releases'] as $version => $release) { + // First, if this is the existing release, check a few conditions. + if ($project_data['existing_version'] === $version) { + if (isset($release['terms']['Release type']) && + in_array('Insecure', $release['terms']['Release type'])) { + $project_data['status'] = UPDATE_NOT_SECURE; + } + elseif ($release['status'] == 'unpublished') { + $project_data['status'] = UPDATE_REVOKED; + if (empty($project_data['extra'])) { + $project_data['extra'] = array(); + } + $project_data['extra'][] = array( + 'class' => array('release-revoked'), + 'label' => t('Release revoked'), + 'data' => t('Your currently installed release has been revoked, and is no longer available for download. Disabling everything included in this release or upgrading is strongly recommended!'), + ); + } + elseif (isset($release['terms']['Release type']) && + in_array('Unsupported', $release['terms']['Release type'])) { + $project_data['status'] = UPDATE_NOT_SUPPORTED; + if (empty($project_data['extra'])) { + $project_data['extra'] = array(); + } + $project_data['extra'][] = array( + 'class' => array('release-not-supported'), + 'label' => t('Release not supported'), + 'data' => t('Your currently installed release is now unsupported, and is no longer available for download. Disabling everything included in this release or upgrading is strongly recommended!'), + ); + } + } + + // Otherwise, ignore unpublished, insecure, or unsupported releases. + if ($release['status'] == 'unpublished' || + (isset($release['terms']['Release type']) && + (in_array('Insecure', $release['terms']['Release type']) || + in_array('Unsupported', $release['terms']['Release type'])))) { + continue; + } + + // See if this is a higher major version than our target and yet still + // supported. If so, record it as an "Also available" release. + // Note: Some projects have a HEAD release from CVS days, which could + // be one of those being compared. They would not have version_major + // set, so we must call isset first. + if (isset($release['version_major']) && $release['version_major'] > $target_major) { + if (in_array($release['version_major'], $supported_majors)) { + if (!isset($project_data['also'])) { + $project_data['also'] = array(); + } + if (!isset($project_data['also'][$release['version_major']])) { + $project_data['also'][$release['version_major']] = $version; + $project_data['releases'][$version] = $release; + } + } + // Otherwise, this release can't matter to us, since it's neither + // from the release series we're currently using nor the recommended + // release. We don't even care about security updates for this + // branch, since if a project maintainer puts out a security release + // at a higher major version and not at the lower major version, + // they must remove the lower version from the supported major + // versions at the same time, in which case we won't hit this code. + continue; + } + + // Look for the 'latest version' if we haven't found it yet. Latest is + // defined as the most recent version for the target major version. + if (!isset($project_data['latest_version']) + && $release['version_major'] == $target_major) { + $project_data['latest_version'] = $version; + $project_data['releases'][$version] = $release; + } + + // Look for the development snapshot release for this branch. + if (!isset($project_data['dev_version']) + && $release['version_major'] == $target_major + && isset($release['version_extra']) + && $release['version_extra'] == 'dev') { + $project_data['dev_version'] = $version; + $project_data['releases'][$version] = $release; + } + + // Look for the 'recommended' version if we haven't found it yet (see + // phpdoc at the top of this function for the definition). + if (!isset($project_data['recommended']) + && $release['version_major'] == $target_major + && isset($release['version_patch'])) { + if ($patch != $release['version_patch']) { + $patch = $release['version_patch']; + $release_patch_changed = $release; + } + if (empty($release['version_extra']) && $patch == $release['version_patch']) { + $project_data['recommended'] = $release_patch_changed['version']; + $project_data['releases'][$release_patch_changed['version']] = $release_patch_changed; + } + } + + // Stop searching once we hit the currently installed version. + if ($project_data['existing_version'] === $version) { + break; + } + + // If we're running a dev snapshot and have a timestamp, stop + // searching for security updates once we hit an official release + // older than what we've got. Allow 100 seconds of leeway to handle + // differences between the datestamp in the .info.yml file and the + // timestamp of the tarball itself (which are usually off by 1 or 2 + // seconds) so that we don't flag that as a new release. + if ($project_data['install_type'] == 'dev') { + if (empty($project_data['datestamp'])) { + // We don't have current timestamp info, so we can't know. + continue; + } + elseif (isset($release['date']) && ($project_data['datestamp'] + 100 > $release['date'])) { + // We're newer than this, so we can skip it. + continue; + } + } + + // See if this release is a security update. + if (isset($release['terms']['Release type']) + && in_array('Security update', $release['terms']['Release type'])) { + $project_data['security updates'][] = $release; + } + } + + // If we were unable to find a recommended version, then make the latest + // version the recommended version if possible. + if (!isset($project_data['recommended']) && isset($project_data['latest_version'])) { + $project_data['recommended'] = $project_data['latest_version']; + } + + // + // Check to see if we need an update or not. + // + + if (!empty($project_data['security updates'])) { + // If we found security updates, that always trumps any other status. + $project_data['status'] = UPDATE_NOT_SECURE; + } + + if (isset($project_data['status'])) { + // If we already know the status, we're done. + return; + } + + // If we don't know what to recommend, there's nothing we can report. + // Bail out early. + if (!isset($project_data['recommended'])) { + $project_data['status'] = UPDATE_UNKNOWN; + $project_data['reason'] = t('No available releases found'); + return; + } + + // If we're running a dev snapshot, compare the date of the dev snapshot + // with the latest official version, and record the absolute latest in + // 'latest_dev' so we can correctly decide if there's a newer release + // than our current snapshot. + if ($project_data['install_type'] == 'dev') { + if (isset($project_data['dev_version']) && $available['releases'][$project_data['dev_version']]['date'] > $available['releases'][$project_data['latest_version']]['date']) { + $project_data['latest_dev'] = $project_data['dev_version']; + } + else { + $project_data['latest_dev'] = $project_data['latest_version']; + } + } + + // Figure out the status, based on what we've seen and the install type. + switch ($project_data['install_type']) { + case 'official': + if ($project_data['existing_version'] === $project_data['recommended'] || $project_data['existing_version'] === $project_data['latest_version']) { + $project_data['status'] = UPDATE_CURRENT; + } + else { + $project_data['status'] = UPDATE_NOT_CURRENT; + } + break; + + case 'dev': + $latest = $available['releases'][$project_data['latest_dev']]; + if (empty($project_data['datestamp'])) { + $project_data['status'] = UPDATE_NOT_CHECKED; + $project_data['reason'] = t('Unknown release date'); + } + elseif (($project_data['datestamp'] + 100 > $latest['date'])) { + $project_data['status'] = UPDATE_CURRENT; + } + else { + $project_data['status'] = UPDATE_NOT_CURRENT; + } + break; + + default: + $project_data['status'] = UPDATE_UNKNOWN; + $project_data['reason'] = t('Invalid info'); + } + } + + /** + * Retrieves update storage data or empties it. + * + * Two very expensive arrays computed by this module are the list of all + * installed modules and themes (and .info.yml data, project associations, etc), and + * the current status of the site relative to the currently available releases. + * These two arrays are stored and used whenever possible. The data is cleared + * whenever the administrator visits the status report, available updates + * report, or the module or theme administration pages, since we should always + * recompute the most current values on any of those pages. + * + * Note: while both of these arrays are expensive to compute (in terms of disk + * I/O and some fairly heavy CPU processing), neither of these is the actual + * data about available updates that we have to fetch over the network from + * updates.drupal.org. That information is stored in the + * 'update_available_releases' collection -- it needs to persist longer than 1 + * hour and never get invalidated just by visiting a page on the site. + * + * @param string $key + * The key of data to return. Valid options are 'update_project_data' and + * 'update_project_projects'. + * + * @return array + * The stored value of the $projects array generated by + * calculateProjectData() or getProjects(), or an empty array + * when the storage is cleared. + */ + public function projectStorage($key) { + $projects = array(); + + // On certain paths, we should clear the data and recompute the projects for + // update status of the site to avoid presenting stale information. + $paths = array( + 'admin/modules', + 'admin/modules/update', + 'admin/appearance', + 'admin/appearance/update', + 'admin/reports', + 'admin/reports/updates', + 'admin/reports/updates/update', + 'admin/reports/status', + 'admin/reports/updates/check', + ); + if (in_array(current_path(), $paths)) { + $this->updateStore->delete($key); + } + else { + $projects = $this->updateStore->get($key); + } + return $projects; + } +} \ No newline at end of file diff --git c/core/modules/update/lib/Drupal/update/UpdateFetchManager.php w/core/modules/update/lib/Drupal/update/UpdateFetchManager.php new file mode 100644 index 0000000..58be2a0 --- /dev/null +++ w/core/modules/update/lib/Drupal/update/UpdateFetchManager.php @@ -0,0 +1,427 @@ +updateConfig = $config->get('update.settings'); + $this->queue = $queue->get('update_fetch_tasks'); + $this->fetchTaskStore = $keyValue->get('update_fetch_task'); + $this->availableReleaseStore = $keyValueExpirable->get('update_available_releases'); + $this->updateStore = $keyValueExpirable->get('update'); + $this->state = $state; + $this->updateCompareManger = $updateCompareManager; + $this->httpClient = $httpClient; + } + + /** + * Creates a new fetch task after loading the necessary include file. + * + * @param array $project + * Associative array of information about a project. See updateGetProjects() + * for the keys used. + * + * @see UpdateFetchManager::processFetchTask() + */ + public function createFetchTask(array $project) { + if (empty($this->fetchTasks)) { + $this->fetchTasks = $this->fetchTaskStore->getAll(); + } + if (empty($this->fetchTasks[$project['name']])) { + $this->queue->createItem($project); + $this->fetchTaskStore->set($project['name'], $project); + $this->fetchTasks[$project['name']] = REQUEST_TIME; + } + } + + /** + * Empties the fetchTasks property. + */ + public function clearFetchTasks() { + $this->fetchTasks = NULL; + } + + /** + * Attempts to drain the queue of tasks for release history data to fetch. + */ + public function fetchData() { + $end = time() + $this->updateConfig->get('fetch.timeout'); + while (time() < $end && ($item = $this->queue->claimItem())) { + $this->processFetchTask($item->data); + $this->queue->deleteItem($item); + } + } + + /** + * Processes a task to fetch available update data for a single project. + * + * Once the release history XML data is downloaded, it is parsed and saved in an + * entry just for that project. + * + * @param array $project + * Associative array of information about the project to fetch data for. + * + * @return bool + * TRUE if we fetched parsable XML, otherwise FALSE. + */ + public function processFetchTask(array $project) { + global $base_url; + $fail = &drupal_static(__FUNCTION__, array()); + // This can be in the middle of a long-running batch, so REQUEST_TIME won't + // necessarily be valid. + $request_time_difference = time() - REQUEST_TIME; + if (empty($fail)) { + // If we have valid data about release history XML servers that we have + // failed to fetch from on previous attempts, load that. + $fail = $this->updateStore->get('fetch_failures'); + } + + $max_fetch_attempts = $this->updateConfig->get('fetch.max_attempts'); + + $success = FALSE; + $available = array(); + $site_key = Crypt::hmacBase64($base_url, drupal_get_private_key()); + $url = $this->buildFetchUrl($project, $site_key); + $fetch_url_base = $this->getFetchUrlBase($project); + $project_name = $project['name']; + + if (empty($fail[$fetch_url_base]) || $fail[$fetch_url_base] < $max_fetch_attempts) { + try { + $data = $this->httpClient + ->get($url, array('Accept' => 'text/xml')) + ->send() + ->getBody(TRUE); + } + catch (RequestException $exception) { + watchdog_exception('update', $exception); + } + } + + if (!empty($data)) { + $available = $this->parseXml($data); + // @todo: Purge release data we don't need (http://drupal.org/node/238950). + if (!empty($available)) { + // Only if we fetched and parsed something sane do we return success. + $success = TRUE; + } + } + else { + $available['project_status'] = 'not-fetched'; + if (empty($fail[$fetch_url_base])) { + $fail[$fetch_url_base] = 1; + } + else { + $fail[$fetch_url_base]++; + } + } + + $frequency = $this->updateConfig->get('check.interval_days'); + $available['last_fetch'] = REQUEST_TIME + $request_time_difference; + $this->availableReleaseStore->setWithExpire($project_name, $available, $request_time_difference + (60 * 60 * 24 * $frequency)); + + // Stash the $fail data back in the DB for the next 5 minutes. + $this->updateStore->setWithExpire('fetch_failures', $fail, $request_time_difference + (60 * 5)); + + // Whether this worked or not, we did just (try to) check for updates. + $this->state->set('update.last_check', REQUEST_TIME + $request_time_difference); + + // Now that we processed the fetch task for this project, clear out the + // record for this task so we're willing to fetch again. + $this->fetchTaskStore->delete($project_name); + + return $success; + } + + /** + * Generates the URL to fetch information about project updates. + * + * This figures out the right URL to use, based on the project's .info.yml file + * and the global defaults. Appends optional query arguments when the site is + * configured to report usage stats. + * + * @param array $project + * The array of project information from updateGetProjects(). + * @param string $site_key + * (optional) The anonymous site key hash. Defaults to an empty string. + * + * @return string + * The URL for fetching information about updates to the specified project. + * + * @see UpdateFetchManager::fetchData() + * @see UpdateFetchManager::processFetchTask() + * @see UpdateCompareManager::getProjects() + */ + public function buildFetchUrl(array $project, $site_key = '') { + $name = $project['name']; + $url = $this->getFetchUrlBase($project); + $url .= '/' . $name . '/' . DRUPAL_CORE_COMPATIBILITY; + + // Only append usage information if we have a site key and the project is + // enabled. We do not want to record usage statistics for disabled projects. + if (!empty($site_key) && (strpos($project['project_type'], 'disabled') === FALSE)) { + // Append the site key. + $url .= (strpos($url, '?') !== FALSE) ? '&' : '?'; + $url .= 'site_key='; + $url .= rawurlencode($site_key); + + // Append the version. + if (!empty($project['info']['version'])) { + $url .= '&version='; + $url .= rawurlencode($project['info']['version']); + } + + // Append the list of modules or themes enabled. + $list = array_keys($project['includes']); + $url .= '&list='; + $url .= rawurlencode(implode(',', $list)); + } + return $url; + } + + /** + * Returns the base of the URL to fetch available update data for a project. + * + * @param array $project + * The array of project information from updateGetProjects(). + * + * @return string + * The base of the URL used for fetching available update data. This does + * not include the path elements to specify a particular project, version, + * site_key, etc. + * + * @see UpdateFetchManager::buildFetchUrl() + */ + public function getFetchUrlBase(array $project) { + if (isset($project['info']['project status url'])) { + $url = $project['info']['project status url']; + } + else { + $url = $this->updateConfig->get('fetch.url'); + if (empty($url)) { + $url = UPDATE_DEFAULT_URL; + } + } + return $url; + } + + /** + * Parses the XML of the Drupal release history info files. + * + * @param string $raw_xml + * A raw XML string of available release data for a given project. + * + * @return array|NULL + * Array of parsed data about releases for a given project, or NULL if there + * was an error parsing the string. + */ + public function parseXml($raw_xml) { + try { + $xml = new SimpleXMLElement($raw_xml); + } + catch (Exception $e) { + // SimpleXMLElement::__construct produces an E_WARNING error message for + // each error found in the XML data and throws an exception if errors + // were detected. Catch any exception and return failure (NULL). + return NULL; + } + // If there is no valid project data, the XML is invalid, so return failure. + if (!isset($xml->short_name)) { + return NULL; + } + $short_name = (string) $xml->short_name; + $data = array(); + foreach ($xml as $k => $v) { + $data[$k] = (string) $v; + } + $data['releases'] = array(); + if (isset($xml->releases)) { + foreach ($xml->releases->children() as $release) { + $version = (string) $release->version; + $data['releases'][$version] = array(); + foreach ($release->children() as $k => $v) { + $data['releases'][$version][$k] = (string) $v; + } + $data['releases'][$version]['terms'] = array(); + if ($release->terms) { + foreach ($release->terms->children() as $term) { + if (!isset($data['releases'][$version]['terms'][(string) $term->name])) { + $data['releases'][$version]['terms'][(string) $term->name] = array(); + } + $data['releases'][$version]['terms'][(string) $term->name][] = (string) $term->value; + } + } + } + } + return $data; + } + + /** + * Performs any notifications that should be done once cron fetches new data. + * + * This method checks the status of the site using the new data and, depending + * on the configuration of the site, notifies administrators via e-mail if there + * are new releases or missing security updates. + * + * @see update_requirements() + */ + public function cronNotify() { + module_load_install('update'); + $status = update_requirements('runtime'); + $params = array(); + $notify_all = ($this->updateConfig->get('notification.threshold') == 'all'); + foreach (array('core', 'contrib') as $report_type) { + $type = 'update_' . $report_type; + if (isset($status[$type]['severity']) + && ($status[$type]['severity'] == REQUIREMENT_ERROR || ($notify_all && $status[$type]['reason'] == UPDATE_NOT_CURRENT))) { + $params[$report_type] = $status[$type]['reason']; + } + } + if (!empty($params)) { + $notify_list = $this->updateConfig->get('notification.emails'); + if (!empty($notify_list)) { + $default_langcode = language_default()->id; + foreach ($notify_list as $target) { + if ($target_user = user_load_by_mail($target)) { + $target_langcode = user_preferred_langcode($target_user); + } + else { + $target_langcode = $default_langcode; + } + $message = drupal_mail('update', 'status_notify', $target, $target_langcode, $params); + // Track when the last mail was successfully sent to avoid sending + // too many e-mails. + if ($message['result']) { + $this->state->set('update.last_email_notification', REQUEST_TIME); + } + } + } + } + } + + /** + * Clears out all the available update data and initiates re-fetching. + */ + public function refresh() { + + // Since we're fetching new available update data, we want to clear + // of both the projects we care about, and the current update status of the + // site. We do *not* want to clear the cache of available releases just yet, + // since that data (even if it's stale) can be useful during + // getProjects() for example, to modules that implement + // hook_system_info_alter() such as cvs_deploy. + $this->updateStore->delete('update_project_projects'); + $this->updateStore->delete('update_project_data'); + + $projects = $this->updateCompareManger->getProjects(); + + // Now that we have the list of projects, we should also clear the available + // release data, since even if we fail to fetch new data, we need to clear + // out the stale data at this point. + $this->availableReleaseStore->deleteAll(); + + foreach ($projects as $key => $project) { + $this->createFetchTask($project); + } + } + +} diff --git c/core/modules/update/update.install w/core/modules/update/update.install index 505b1ca..18da2a1 100644 --- c/core/modules/update/update.install +++ w/core/modules/update/update.install @@ -29,8 +29,7 @@ function update_requirements($phase) { $requirements = array(); if ($phase == 'runtime') { if ($available = update_get_available(FALSE)) { - module_load_include('inc', 'update', 'update.compare'); - $data = update_calculate_project_data($available); + $data = \Drupal::service('update.compare')->calculateProjectData($available); // First, populate the requirements for core: $requirements['update_core'] = _update_requirement_check($data['drupal'], 'core'); // We don't want to check drupal a second time. diff --git c/core/modules/update/update.manager.inc w/core/modules/update/update.manager.inc index 6063dfc..c2db049 100644 --- c/core/modules/update/update.manager.inc +++ w/core/modules/update/update.manager.inc @@ -95,8 +95,7 @@ function update_manager_update_form($form, $form_state = array(), $context) { // project in the form, regardless of if it's enabled or disabled. $form['project_downloads'] = array('#tree' => TRUE); - module_load_include('inc', 'update', 'update.compare'); - $project_data = update_calculate_project_data($available); + $project_data = \Drupal::service('update.compare')->calculateProjectData($available); foreach ($project_data as $name => $project) { // Filter out projects which are up to date already. if ($project['status'] == UPDATE_CURRENT) { diff --git c/core/modules/update/update.module w/core/modules/update/update.module index 4258644..0b0ecee 100644 --- c/core/modules/update/update.module +++ w/core/modules/update/update.module @@ -262,8 +262,8 @@ function update_cron() { // If the configured update interval has elapsed, we want to invalidate // the data for all projects, attempt to re-fetch, and trigger any // configured notifications about the new status. - update_refresh(); - update_fetch_data(); + \Drupal::service('update.fetch')->refresh(); + \Drupal::service('update.fetch')->fetchData(); } else { // Otherwise, see if any individual projects are now stale or still @@ -274,8 +274,7 @@ function update_cron() { if ((REQUEST_TIME - $last_email_notice) > $interval) { // If configured time between notifications elapsed, send email about // updates possibly available. - module_load_include('inc', 'update', 'update.fetch'); - _update_cron_notify(); + \Drupal::service('update.fetch')->cronNotify(); } // Clear garbage from disk. @@ -363,11 +362,11 @@ function update_get_available($refresh = FALSE) { // Grab whatever data we currently have. $available = \Drupal::keyValueExpirable('update_available_releases')->getAll(); - $projects = update_get_projects(); + $projects = \Drupal::service('update.compare')->getProjects(); foreach ($projects as $key => $project) { // If there's no data at all, we clearly need to fetch some. if (empty($available[$key])) { - update_create_fetch_task($project); + \Drupal::service('update.fetch')->createFetchTask($project); $needs_refresh = TRUE; continue; } @@ -390,14 +389,14 @@ function update_get_available($refresh = FALSE) { // If we think this project needs to fetch, actually create the task now // and remember that we think we're missing some data. if (!empty($available[$key]['fetch_status']) && $available[$key]['fetch_status'] == UPDATE_FETCH_PENDING) { - update_create_fetch_task($project); + \Drupal::service('update.fetch')->createFetchTask($project); $needs_refresh = TRUE; } } if ($needs_refresh && $refresh) { // Attempt to drain the queue of fetch tasks. - update_fetch_data(); + \Drupal::service('update.fetch')->fetchData(); // After processing the queue, we've (hopefully) got better data, so pull // the latest data again and use that directly. $available = \Drupal::keyValueExpirable('update_available_releases')->getAll(); @@ -407,37 +406,12 @@ function update_get_available($refresh = FALSE) { } /** - * Creates a new fetch task after loading the necessary include file. - * - * @param $project - * Associative array of information about a project. See update_get_projects() - * for the keys used. - * - * @see _update_create_fetch_task() - */ -function update_create_fetch_task($project) { - module_load_include('inc', 'update', 'update.fetch'); - _update_create_fetch_task($project); -} - -/** * Refreshes the release data after loading the necessary include file. * * @see _update_refresh() */ function update_refresh() { - module_load_include('inc', 'update', 'update.fetch'); - _update_refresh(); -} - -/** - * Attempts to fetch update data after loading the necessary include file. - * - * @see _update_fetch_data() - */ -function update_fetch_data() { - module_load_include('inc', 'update', 'update.fetch'); - _update_fetch_data(); + return \Drupal::service('update.fetch')->refresh(); } /** @@ -783,3 +757,29 @@ function update_delete_file_if_stale($path) { } } } + +/** + * Batch callback: Performs actions when all fetch tasks have been completed. + * + * @param $success + * TRUE if the batch operation was successful; FALSE if there were errors. + * @param $results + * An associative array of results from the batch operation, including the key + * 'updated' which holds the total number of projects we fetched available + * update data for. + */ +function update_fetch_data_finished($success, $results) { + if ($success) { + if (!empty($results)) { + if (!empty($results['updated'])) { + drupal_set_message(format_plural($results['updated'], 'Checked available update data for one project.', 'Checked available update data for @count projects.')); + } + if (!empty($results['failures'])) { + drupal_set_message(format_plural($results['failures'], 'Failed to get available update data for one project.', 'Failed to get available update data for @count projects.'), 'error'); + } + } + } + else { + drupal_set_message(t('An error occurred trying to get available update data.'), 'error'); + } +} diff --git c/core/modules/update/update.services.yml w/core/modules/update/update.services.yml index 0ffafc2..e518258 100644 --- c/core/modules/update/update.services.yml +++ w/core/modules/update/update.services.yml @@ -4,3 +4,11 @@ services: arguments: ['@settings'] tags: - { name: access_check } + + update.fetch: + class: Drupal\update\UpdateFetchManager + arguments: ['@config.factory', '@queue', '@keyvalue', '@keyvalue.expirable', '@state', '@http_default_client', '@update.compare'] + + update.compare: + class: Drupal\update\UpdateCompareManager + arguments: ['@module_handler', '@keyvalue.expirable']