diff --git a/automatic_updates.services.yml b/automatic_updates.services.yml index 61b9e56..8f486c9 100644 --- a/automatic_updates.services.yml +++ b/automatic_updates.services.yml @@ -108,3 +108,12 @@ services: class: Drupal\automatic_updates\EventSubscriber\CronOverride tags: - { name: config.factory.override } + automatic_updates.update: + class: Drupal\automatic_updates\Services\InPlaceUpdate + arguments: + - '@logger.channel.automatic_updates' + - '@plugin.manager.archiver' + - '@config.factory' + - '@file_system' + - '@http_client' + - '@automatic_updates.drupal_finder' diff --git a/composer.json b/composer.json index 58ff78d..e2a3c13 100644 --- a/composer.json +++ b/composer.json @@ -14,7 +14,7 @@ "ext-json": "*", "composer/semver": "^1.0", "ocramius/package-versions": "^1.4", - "webflo/drupal-finder": "^1.1" + "webflo/drupal-finder": "^1.2" }, "require-dev": { "drupal/ctools": "3.2.0" diff --git a/config/install/automatic_updates.settings.yml b/config/install/automatic_updates.settings.yml index 0e900e9..a5dce8b 100644 --- a/config/install/automatic_updates.settings.yml +++ b/config/install/automatic_updates.settings.yml @@ -5,3 +5,4 @@ check_frequency: 43200 enable_readiness_checks: true hashes_uri: 'https://updates.drupal.org/release-hashes' ignored_paths: "modules/custom/*\nthemes/custom/*\nprofiles/custom/*" +download_uri: 'https://www.drupal.org/in-place-updates' diff --git a/config/schema/automatic_updates.schema.yml b/config/schema/automatic_updates.schema.yml index 0a45a08..88a2ed4 100644 --- a/config/schema/automatic_updates.schema.yml +++ b/config/schema/automatic_updates.schema.yml @@ -23,3 +23,6 @@ automatic_updates.settings: ignored_paths: type: string label: 'List of files paths to ignore when running readiness checks' + download_uri: + type: string + label: 'URI for downloading in-place update assets' diff --git a/drupalci.yml b/drupalci.yml index 8fa0ea2..8939752 100644 --- a/drupalci.yml +++ b/drupalci.yml @@ -4,19 +4,22 @@ build: assessment: validate_codebase: phplint: - container_composer: phpcs: # phpcs will use core's specified version of Coder. sniff-all-files: true halt-on-fail: true testing: container_command: - commands: "cd ${SOURCE_DIR} && sudo -u www-data composer require drupal/ctools:3.2.0 --prefer-source --optimize-autoloader" + commands: + - cd ${SOURCE_DIR} && sudo -u www-data composer require drupal/ctools:3.2.0 --prefer-source --optimize-autoloader + - cd ${SOURCE_DIR} && sudo -u www-data curl https://www.drupal.org/files/issues/2019-09-12/3031379-154.patch | git apply - + # run_tests task is executed several times in order of performance speeds. # halt-on-fail can be set on the run_tests tasks in order to fail fast. # suppress-deprecations is false in order to be alerted to usages of # deprecated code. run_tests.standard: - types: 'Simpletest,PHPUnit-Unit,PHPUnit-Kernel,PHPUnit-Functional' + types: 'PHPUnit-Unit,PHPUnit-Kernel,PHPUnit-Functional,PHPUnit-Build' testgroups: '--all' suppress-deprecations: false + halt-on-fail: false diff --git a/src/Services/InPlaceUpdate.php b/src/Services/InPlaceUpdate.php new file mode 100644 index 0000000..9aff8bb --- /dev/null +++ b/src/Services/InPlaceUpdate.php @@ -0,0 +1,418 @@ +logger = $logger; + $this->archiveManager = $archive_manager; + $this->configFactory = $config_factory; + $this->fileSystem = $file_system; + $this->httpClient = $http_client; + $drupal_finder->locateRoot(getcwd()); + $this->rootPath = $drupal_finder->getDrupalRoot(); + $this->vendorPath = rtrim($drupal_finder->getVendorDir(), '/\\') . DIRECTORY_SEPARATOR; + } + + /** + * {@inheritdoc} + */ + public function update($project_name, $project_type, $from_version, $to_version) { + $success = FALSE; + if ($project_name === 'drupal') { + $project_root = $this->rootPath; + } + else { + $project_root = drupal_get_path($project_type, $project_name); + } + if ($archive = $this->getArchive($project_name, $from_version, $to_version)) { + if ($this->backup($archive, $project_root)) { + $success = $this->processUpdate($archive, $project_root); + } + if (!$success) { + $this->rollback($project_root); + } + } + return $success; + } + + /** + * Get an archive with the quasi-patch contents. + * + * @param string $project_name + * The project name. + * @param string $from_version + * The current project version. + * @param string $to_version + * The desired next project version. + * + * @return \Drupal\Core\Archiver\ArchiverInterface|null + * The archive or NULL if download fails. + */ + protected function getArchive($project_name, $from_version, $to_version) { + $url = $this->buildUrl($project_name, $this->getQuasiPatchFileName($project_name, $from_version, $to_version)); + $destination = $this->fileSystem->realpath($this->fileSystem->getDestinationFilename("temporary://$project_name.zip", FileSystemInterface::EXISTS_RENAME)); + try { + $this->httpClient->get($url, ['sink' => $destination]); + /** @var \Drupal\Core\Archiver\ArchiverInterface $archive */ + return $this->archiveManager->getInstance(['filepath' => $destination]); + + } + catch (RequestException $exception) { + $this->logger->error('Update for @project to version @version failed for reason @message', [ + '@project' => $project_name, + '@version' => $version, + '@message' => $exception->getMessage(), + ]); + } + } + + /** + * Process update. + * + * @param \Drupal\Core\Archiver\ArchiverInterface $archive + * The archive. + * @param string $project_root + * The project root directory. + * + * @return bool + * Return TRUE if update succeeds, FALSE otherwise. + */ + protected function processUpdate(ArchiverInterface $archive, $project_root) { + $archive->extract($this->getTempDirectory()); + foreach ($this->getFilesList($this->getTempDirectory()) as $file) { + $file_path = substr($file->getRealPath(), strlen($this->getTempDirectory() . self::ARCHIVE_DIRECTORY)); + $real_path = $this->getRealPath($file_path, $project_root); + try { + $directory = dirname($real_path); + $result = $this->fileSystem->prepareDirectory($directory, FileSystemInterface::CREATE_DIRECTORY); + $this->fileSystem->copy($file->getRealPath(), $real_path, FileSystemInterface::EXISTS_REPLACE); + } + catch (FileException $exception) { + return FALSE; + } + } + foreach ($this->getDeletions() as $deletion) { + try { + $this->fileSystem->delete($real_path = $this->getRealPath($deletion, $project_root)); + } + catch (FileException $exception) { + return FALSE; + } + } + return TRUE; + } + + /** + * Backup before an update. + * + * @param \Drupal\Core\Archiver\ArchiverInterface $archive + * The archive. + * @param string $project_root + * The project root directory. + * + * @return bool + * Return TRUE if backup succeeds, FALSE otherwise. + */ + protected function backup(ArchiverInterface $archive, $project_root) { + $backup = $this->fileSystem->createFilename('automatic_updates-backup', 'temporary://'); + $this->fileSystem->prepareDirectory($backup); + $this->backup = $this->fileSystem->realpath($backup) . DIRECTORY_SEPARATOR; + if (!$this->backup) { + return FALSE; + } + foreach ($archive->listContents() as $file) { + $this->stripPrefix($file); + $success = $this->doBackup($file, $project_root); + if (!$success) { + return FALSE; + } + } + $archive->extract($this->getTempDirectory(), [self::DELETION_MANIFEST]); + foreach ($this->getDeletions() as $deletion) { + $success = $this->doBackup($deletion, $project_root); + if (!$success) { + return FALSE; + } + } + return TRUE; + } + + /** + * Remove the files folder prefix from files from the archive. + * + * @param string $file + * The file path. + */ + protected function stripPrefix(&$file) { + if (substr($file, 0, 6) === self::ARCHIVE_DIRECTORY) { + $file = substr($file, 6); + } + } + + /** + * Execute file backup. + * + * @param string $file + * The file to backup. + * @param string $project_root + * The project root directory. + * + * @return bool + * Return TRUE if backup succeeds, FALSE otherwise. + */ + protected function doBackup($file, $project_root) { + $directory = $this->backup . dirname($file); + if (!file_exists($directory) && !$this->fileSystem->mkdir($directory, NULL, TRUE)) { + return FALSE; + } + $real_path = $this->getRealPath($file, $project_root); + if (file_exists($real_path) && !is_dir($real_path)) { + try { + $this->fileSystem->copy($real_path, $this->backup . $file, FileSystemInterface::EXISTS_REPLACE); + } + catch (FileException $exception) { + return FALSE; + } + } + return TRUE; + } + + /** + * Rollback after a failed update. + * + * @param string $project_root + * The project root directory. + */ + protected function rollback($project_root) { + if (!$this->backup) { + return; + } + foreach ($this->getFilesList($this->backup) as $file) { + $file_path = substr($file->getRealPath(), strlen($this->backup)); + try { + $this->fileSystem->copy($file->getRealPath(), $this->getRealPath($file_path, $project_root), FileSystemInterface::EXISTS_REPLACE); + } + catch (FileException $exception) { + $this->logger->error('@file was not rolled back successfully.', ['@file' => $file->getRealPath()]); + } + } + } + + /** + * Provide a recursive list of files, excluding directories. + * + * @param string $directory + * The directory to recurse for files. + * + * @return \RecursiveIteratorIterator|\SplFileInfo[] + * The iterator of SplFileInfos. + */ + protected function getFilesList($directory) { + $filter = function ($file, $file_name, $iterator) { + /** @var \SplFileInfo $file */ + /** @var string $file_name */ + /** @var \RecursiveDirectoryIterator $iterator */ + if ($iterator->hasChildren() && !in_array($file->getFilename(), ['.git'], TRUE)) { + return TRUE; + } + return $file->isFile() && !in_array($file->getFilename(), [self::DELETION_MANIFEST], TRUE); + }; + + $innerIterator = new \RecursiveDirectoryIterator($directory, \RecursiveDirectoryIterator::SKIP_DOTS); + return new \RecursiveIteratorIterator(new \RecursiveCallbackFilterIterator($innerIterator, $filter)); + } + + /** + * Build a project quasi-patch download URL. + * + * @param string $project_name + * The project name. + * @param string $file_name + * The file name. + * + * @return string + * The URL endpoint with for an extension. + */ + protected function buildUrl($project_name, $file_name) { + $uri = $this->configFactory->get('automatic_updates.settings')->get('download_uri'); + return Url::fromUri($uri . "/$project_name/" . $file_name)->toString(); + } + + /** + * Get the quasi-patch file name. + * + * @param string $project_name + * The project name. + * @param string $from_version + * The current project version. + * @param string $to_version + * The desired next project version. + * + * @return string + * The quasi-patch file name. + */ + protected function getQuasiPatchFileName($project_name, $from_version, $to_version) { + return "$project_name-$from_version-to-$to_version.zip"; + } + + /** + * Get the real path of a file. + * + * @param string $file_path + * The file path. + * @param string $project_root + * The project root directory. + * + * @return string + * The real path of a file. + */ + protected function getRealPath($file_path, $project_root) { + if (substr($file_path, 0, 6) === 'vendor/') { + return $this->vendorPath . substr($file_path, 7); + } + return rtrim($project_root, '/\\') . DIRECTORY_SEPARATOR . $file_path; + } + + /** + * Provides the temporary extraction directory. + * + * @return string + * The temporary directory. + */ + protected function getTempDirectory() { + if (!$this->tempDirectory) { + $this->tempDirectory = $this->fileSystem->createFilename('automatic_updates-update', 'temporary://'); + $this->fileSystem->prepareDirectory($this->tempDirectory); + $this->tempDirectory = $this->fileSystem->realpath($this->tempDirectory) . DIRECTORY_SEPARATOR; + } + return $this->tempDirectory; + } + + /** + * Get an iterator of files to delete. + * + * @return \ArrayIterator + * Iterator of files to delete. + */ + protected function getDeletions() { + $deletions = []; + if (!file_exists($this->getTempDirectory() . self::DELETION_MANIFEST)) { + return new \ArrayIterator(); + } + $handle = fopen($this->getTempDirectory() . self::DELETION_MANIFEST, "r"); + if ($handle) { + while (($deletion = fgets($handle)) !== FALSE) { + if ($result = trim($deletion)) { + $deletions[] = $result; + } + } + fclose($handle); + } + return new \ArrayIterator($deletions); + } +} diff --git a/src/Services/UpdateInterface.php b/src/Services/UpdateInterface.php new file mode 100644 index 0000000..ee685a4 --- /dev/null +++ b/src/Services/UpdateInterface.php @@ -0,0 +1,27 @@ +updater = $updater; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('automatic_updates.update') + ); + } + + /** + * Builds the response. + */ + public function update($project, $type, $from, $to) { + $updated = $this->updater->update($project, $type, $from, $to); + return [ + '#markup' => $updated ? $this->t('Update successful') : $this->t('Update Failed'), + ]; + } + +} diff --git a/tests/modules/test_automatic_updates/src/EventSubscriber/DisableThemeCsrfRouteSubscriber.php b/tests/modules/test_automatic_updates/src/EventSubscriber/DisableThemeCsrfRouteSubscriber.php new file mode 100644 index 0000000..d8222b8 --- /dev/null +++ b/tests/modules/test_automatic_updates/src/EventSubscriber/DisableThemeCsrfRouteSubscriber.php @@ -0,0 +1,23 @@ +get('system.theme_set_default')) { + $route->setRequirements(['_permission' => 'administer themes']); + } + } + +} diff --git a/tests/modules/test_automatic_updates/test_automatic_updates.info.yml b/tests/modules/test_automatic_updates/test_automatic_updates.info.yml index 506856c..1373e2c 100644 --- a/tests/modules/test_automatic_updates/test_automatic_updates.info.yml +++ b/tests/modules/test_automatic_updates/test_automatic_updates.info.yml @@ -3,3 +3,5 @@ type: module description: 'Tests for Automatic Updates' package: Testing core: 8.x +dependencies: + - automatic_updates:automatic_updates diff --git a/tests/modules/test_automatic_updates/test_automatic_updates.routing.yml b/tests/modules/test_automatic_updates/test_automatic_updates.routing.yml index 7c9ce34..4d113f3 100644 --- a/tests/modules/test_automatic_updates/test_automatic_updates.routing.yml +++ b/tests/modules/test_automatic_updates/test_automatic_updates.routing.yml @@ -19,3 +19,12 @@ test_automatic_updates.hashes_endpoint: _title: 'SHA512SUMS' requirements: _access: 'TRUE' +test_automatic_updates.inplace-update: + path: '/automatic_updates/in-place-update/{project}/{type}/{from}/{to}' + defaults: + _title: 'Update' + _controller: '\Drupal\test_automatic_updates\Controller\InPlaceUpdateController::update' + requirements: + _access: 'TRUE' + options: + no_cache: 'TRUE' diff --git a/tests/modules/test_automatic_updates/test_automatic_updates.services.yml b/tests/modules/test_automatic_updates/test_automatic_updates.services.yml new file mode 100644 index 0000000..da4b277 --- /dev/null +++ b/tests/modules/test_automatic_updates/test_automatic_updates.services.yml @@ -0,0 +1,5 @@ +services: + test_automatic_updates.route_subscriber: + class: Drupal\test_automatic_updates\EventSubscriber\DisableThemeCsrfRouteSubscriber + tags: + - { name: event_subscriber } diff --git a/tests/src/Build/InPlaceUpdateTest.php b/tests/src/Build/InPlaceUpdateTest.php new file mode 100644 index 0000000..ba956ae --- /dev/null +++ b/tests/src/Build/InPlaceUpdateTest.php @@ -0,0 +1,219 @@ +assetsServer)) { + $this->assetsServer->stop(); + } + } + + /** + * @covers ::update + * @dataProvider coreVersionsProvider + */ + public function testCoreUpdate($from_version, $to_version) { + $this->copyCodebase(); + // We have to fetch the tags for this shallow repo. It might not be a + // shallow clone, therefore we use executeCommand instead of assertCommand. + $this->executeCommand('git fetch --unshallow --tags'); + $this->assertCommand("git checkout $from_version -f"); + $fs = new Filesystem(); + $fs->chmod($this->getWorkspaceDirectory() . '/sites/default', 0700, 0000); + $this->assertCommandErrorOutputContains('Generating autoload files', 'COMPOSER_DISCARD_CHANGES=true composer install --no-dev --no-interaction'); + $this->assertCommandErrorOutputContains('Generating autoload files', 'COMPOSER_DISCARD_CHANGES=true composer require ocramius/package-versions:^1.4 webflo/drupal-finder:^1.1 composer/semver:^1.0 --no-interaction'); + $this->installQuickStart('minimal'); + + // Assert that the site is functional before updating. + $this->assertVisit(); + $assert = $this->mink->assertSession(); + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + + // Currently, this test has to use extension_discovery_scan_tests so we can + // enable test modules. + $fs->chmod($this->getWorkspaceDirectory() . '/sites/default/settings.php', 0640, 0000); + file_put_contents($this->getWorkspaceDirectory() . '/sites/default/settings.php', '$settings[\'extension_discovery_scan_tests\'] = TRUE;' . PHP_EOL, FILE_APPEND); + // We also need to update the download URI to use the dynamically generated + // hostname and port number of a new asset server. + $assets_port = $this->findAvailablePort(); + $this->assetsServer = $this->instantiateServer(static::$hostName, $assets_port); + $uri = 'http://localhost:' . $assets_port . '/modules/contrib/automatic_updates/tests/assets/fixtures'; + file_put_contents($this->getWorkspaceDirectory() . '/sites/default/settings.php', "\$config['automatic_updates.settings']['download_uri'] = '$uri';" . PHP_EOL, FILE_APPEND); + + // Log in so that we can install modules. + $this->formLogin($this->adminUsername, $this->adminPassword); + $this->moduleEnable('automatic_updates'); + $this->moduleEnable('test_automatic_updates'); + + // Confirm we are running correct Drupal version. + $finder = new Finder(); + $finder->files()->in($this->getWorkspaceDirectory())->path('core/lib/Drupal.php'); + $finder->contains("/const VERSION = '$from_version'/"); + $this->assertTrue($finder->hasResults()); + $this->assertFileExists($this->getWorkspaceDirectory() . '/vendor/brumann/polyfill-unserialize/tests/UnserializeTest.php'); + + // Update the site. + $this->assertVisit("automatic_updates/in-place-update/drupal/core/$from_version/$to_version"); + + // Assert that the update worked. + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + $assert->pageTextContains('Update successful'); + $finder = new Finder(); + $finder->files()->in($this->getWorkspaceDirectory())->path('core/lib/Drupal.php'); + $finder->contains("/const VERSION = '$to_version'/"); + $this->assertTrue($finder->hasResults()); + $this->assertVisit('admin/reports/status'); + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + $assert->pageTextContains("Drupal Version $to_version"); + $this->assertFileNotExists($this->getWorkspaceDirectory() . '/vendor/brumann/polyfill-unserialize/tests/UnserializeTest.php'); + } + + /** + * @covers ::update + * @dataProvider contribProjectsProvider + */ + public function testContribUpdate($project, $project_type, $from_version, $to_version) { + $this->destroyBuild = FALSE; + $this->copyCodebase(); + $fs = new Filesystem(); + $fs->chmod($this->getWorkspaceDirectory() . '/sites/default', 0700, 0000); + $this->assertCommandErrorOutputContains('Generating autoload files', 'COMPOSER_DISCARD_CHANGES=true composer install --no-dev --no-interaction'); + $this->installQuickStart('standard'); + + // Download and install the module. + $fs->mkdir($this->getWorkspaceDirectory() . "/{$project_type}s/contrib/$project"); + $this->assertCommand("curl -fsSL https://ftp.drupal.org/files/projects/$project-$from_version.tar.gz | tar xvz -C {$project_type}s/contrib/$project --strip 1"); + $finder = new Finder(); + $finder->files()->in($this->getWorkspaceDirectory())->path("{$project_type}s/contrib/$project/$project.info.yml"); + $finder->contains("/version: '$from_version'/"); + $this->assertTrue($finder->hasResults()); + + // Assert that the site is functional before updating. + $this->assertVisit(); + $assert = $this->mink->assertSession(); + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + + // Currently, this test has to use extension_discovery_scan_tests so we can + // enable test modules. + $fs->chmod($this->getWorkspaceDirectory() . '/sites/default/settings.php', 0640, 0000); + file_put_contents($this->getWorkspaceDirectory() . '/sites/default/settings.php', '$settings[\'extension_discovery_scan_tests\'] = TRUE;' . PHP_EOL, FILE_APPEND); + + // We need to update the download URI to use the dynamically generated + // hostname and port number of a new asset server. + $assets_port = $this->findAvailablePort(); + $this->assetsServer = $this->instantiateServer(static::$hostName, $assets_port); + $uri = 'http://localhost:' . $assets_port . '/modules/contrib/automatic_updates/tests/assets/fixtures'; + $fs->chmod($this->getWorkspaceDirectory() . '/sites/default/settings.php', 0640, 0000); + file_put_contents($this->getWorkspaceDirectory() . '/sites/default/settings.php', "\$config['automatic_updates.settings']['download_uri'] = '$uri';" . PHP_EOL, FILE_APPEND); + + // Log in so that we can install projects. + $this->formLogin($this->adminUsername, $this->adminPassword); + $this->moduleEnable('automatic_updates'); + $this->moduleEnable('test_automatic_updates'); + if (is_callable([$this, "{$project_type}Enable"])) { + call_user_func([$this, "{$project_type}Enable"], $project); + } + + // Update the contrib project. + $this->assertVisit("automatic_updates/in-place-update/$project/$project_type/$from_version/$to_version"); + + // Assert that the update worked. + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + $assert->pageTextContains('Update successful'); + $finder = new Finder(); + $finder->files()->in($this->getWorkspaceDirectory())->path("{$project_type}s/contrib/$project/$project.info.yml"); + $finder->contains("/version: '$to_version'/"); + $this->assertTrue($finder->hasResults()); + $assert->responseHeaderEquals('X-Generator', 'Drupal 8 (https://www.drupal.org)'); + $assert->elementExists('css', 'html'); + } + + /** + * Core versions data provider. + */ + public function coreVersionsProvider() { + $datum[] = [ + 'from' => '8.7.0', + 'to' => '8.7.1', + ]; + return $datum; + } + + /** + * Contrib project data provider. + */ + public function contribProjectsProvider() { + $datum[] = [ + 'project' => 'bootstrap', + 'type' => 'theme', + 'from' => '8.x-3.19', + 'to' => '8.x-3.20', + ]; + $datum[] = [ + 'project' => 'token', + 'type' => 'module', + 'from' => '8.x-1.4', + 'to' => '8.x-1.5', + ]; + return $datum; + } + + /** + * Helper method that uses Drupal's module page to enable a module. + */ + protected function moduleEnable($module_name) { + $this->assertVisit('admin/modules', 200); + $assert = $this->mink->assertSession(); + $field = Html::getClass("edit-modules $module_name enable"); + $assert->fieldExists($field)->check(); + $session = $this->mink->getSession(); + $session->getPage()->findButton('Install')->submit(); + $assert->fieldExists($field)->isChecked(); + } + + /** + * Helper method that uses Drupal's theme page to enable a theme. + */ + protected function themeEnable($theme_name) { + $this->moduleEnable('test_automatic_updates'); + $this->assertVisit("/admin/appearance/default?theme=$theme_name", 200); + $assert = $this->mink->assertSession(); + $assert->pageTextContains('is now the default theme'); + $assert->pageTextNotContains('theme was not found'); + } + +}