diff --git a/flysystem.services.yml b/flysystem.services.yml index 5ab8c66..185170c 100644 --- a/flysystem.services.yml +++ b/flysystem.services.yml @@ -18,7 +18,18 @@ services: class: Drupal\flysystem\FlysystemFactory arguments: ['@plugin.manager.flysystem', '@file_system', '@cache.flysystem', '@logger.channel.flysystem'] + flysystem_image_style_copier: + class: Drupal\flysystem\ImageStyleCopier + arguments: ['@lock', '@file_system', '@logger.channel.image'] + tags: + - { name: event_subscriber } + path_processor.flysystem: class: Drupal\flysystem\PathProcessor\FlysystemPathProcessor tags: - { name: path_processor_inbound, priority: 400 } + + path_processor.flysystem_redirect: + class: Drupal\flysystem\PathProcessor\FlysystemImageStyleRedirectProcessor + tags: + - { name: path_processor_inbound, priority: 400 } diff --git a/src/Controller/ImageStyleDownloadController.php b/src/Controller/ImageStyleDownloadController.php new file mode 100644 index 0000000..fc5d37e --- /dev/null +++ b/src/Controller/ImageStyleDownloadController.php @@ -0,0 +1,291 @@ +lock = $lock; + $this->imageFactory = $image_factory; + $this->logger = $this->getLogger('image'); + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('lock'), + $container->get('image.factory') + ); + } + + /** + * Generates a derivative, given a style and image path. + * + * After generating an image, transfer it to the requesting agent. + * + * @param \Symfony\Component\HttpFoundation\Request $request + * The request object. + * @param string $scheme + * The file scheme, defaults to 'private'. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to deliver. + * + * @return \Symfony\Component\HttpFoundation\BinaryFileResponse|\Symfony\Component\HttpFoundation\Response + * The transferred file as response or some error response. + * + * @throws \Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException + * Thrown when the user does not have access to the file. + * @throws \Symfony\Component\HttpKernel\Exception\ServiceUnavailableHttpException + * Thrown when the file is still being generated. + */ + public function deliver(Request $request, $scheme, ImageStyleInterface $image_style) { + $target = $request->query->get('file'); + $image_uri = $scheme . '://' . $target; + + $this->validateRequest($request, $image_style, $scheme, $target); + + $derivative_uri = $image_style->buildUri($image_uri); + $headers = array(); + + // If using the private scheme, let other modules provide headers and + // control access to the file. + if ($scheme == 'private') { + if (file_exists($derivative_uri)) { + return parent::download($request, $scheme); + } + else { + $headers = $this->moduleHandler()->invokeAll('file_download', array($image_uri)); + if (in_array(-1, $headers) || empty($headers)) { + throw new AccessDeniedHttpException(); + } + } + } + + // Don't try to generate file if source is missing. + try { + $image_uri = $this->validateSource($image_uri); + } + catch (FileNotFoundException $e) { + $this->logger->notice('Source image at %source_image_path not found while trying to generate derivative image at %derivative_path.', array('%source_image_path' => $image_uri, '%derivative_path' => $derivative_uri)); + return new Response($this->t('Error generating image, missing source file.'), 404); + } + + $success = $this->generate($image_style, $image_uri, $derivative_uri); + + if ($success) { + return $this->send($scheme, $derivative_uri, $headers); + } + else { + $this->logger->notice('Unable to generate the derived image located at %path.', array('%path' => $derivative_uri)); + return new Response($this->t('Error generating image.'), 500); + } + } + + /** + * Validate that a source image exists, checking for double extensions. + * + * If the image style converted the extension, it has been added to the + * original file, resulting in filenames like image.png.jpeg. So to find + * the actual source image, we remove the extension and check if that + * image exists. + * + * @param string $image_uri + * The URI to the source image. + * + * @return string + * The original $image_uri, or the source with the original extension. + * + * @throws \Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException + * Thrown when no valid source image is found. + */ + protected function validateSource($image_uri) { + if (!file_exists($image_uri)) { + $path_info = pathinfo($image_uri); + $converted_image_uri = $path_info['dirname'] . DIRECTORY_SEPARATOR . $path_info['filename']; + if (!file_exists($converted_image_uri)) { + throw new FileNotFoundException($converted_image_uri); + } + // The converted file does exist, use it as the source. + return $converted_image_uri; + } + + return $image_uri; + } + + /** + * Return a response of the derived image. + * + * @param string $scheme + * The URI scheme of $derivative_uri. + * @param string $derivative_uri + * The URI of the derived image. + * @param array $headers + * (optional) An array of headers to return in the response. + * + * @return \Symfony\Component\HttpFoundation\BinaryFileResponse + * A response with the derived image. + */ + protected function send($scheme, $derivative_uri, $headers = array()) { + $image = $this->imageFactory->get($derivative_uri); + $uri = $image->getSource(); + $headers += array( + 'Content-Type' => $image->getMimeType(), + 'Content-Length' => $image->getFileSize(), + ); + // \Drupal\Core\EventSubscriber\FinishResponseSubscriber::onRespond() + // sets response as not cacheable if the Cache-Control header is not + // already modified. We pass in FALSE for non-private schemes for the + // $public parameter to make sure we don't change the headers. + return new BinaryFileResponse($uri, 200, $headers, $scheme !== 'private'); + } + + /** + * Generate an image derivative. + * + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to use for the derivative. + * @param string $image_uri + * The URI of the original image. + * @param string $derivative_uri + * The URI of the derived image. + * + * @return bool + * TRUE if the image exists or was generated, FALSE otherwise. + */ + protected function generate(ImageStyleInterface $image_style, $image_uri, $derivative_uri) { + // Don't start generating the image if the derivative already exists or if + // generation is in progress in another thread. + $lock_name = 'image_style_deliver:' . $image_style->id() . ':' . Crypt::hashBase64($image_uri); + if (!file_exists($derivative_uri)) { + $lock_acquired = $this->lock->acquire($lock_name); + if (!$lock_acquired) { + // Tell client to retry again in 3 seconds. Currently no browsers are + // known to support Retry-After. + throw new ServiceUnavailableHttpException(3, $this->t('Image generation in progress. Try again shortly.')); + } + } + + // Try to generate the image, unless another thread just did it while we + // were acquiring the lock. + $success = file_exists($derivative_uri) || $image_style->createDerivative($image_uri, $derivative_uri); + + if (!empty($lock_acquired)) { + $this->lock->release($lock_name); + } + + return $success; + } + + /** + * Validate an incoming derivative request. + * + * Check that the style is defined, the scheme is valid, and the image + * derivative token is valid. Sites which require image derivatives to be + * generated without a token can set the + * 'image.settings:allow_insecure_derivatives' configuration to TRUE to + * bypass the latter check, but this will increase the site's vulnerability + * to denial-of-service attacks. To prevent this variable from leaving the + * site vulnerable to the most serious attacks, a token is always required + * when a derivative of a style is requested. + * The $target variable for a derivative of a style has + * styles//... as structure, so we check if the $target variable + * starts with styles/. + * + * @param \Symfony\Component\HttpFoundation\Request $request + * The incoming derivative request. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to use for the derivative. + * @param string $scheme + * The URI scheme of $target. + * @param string $target + * The path for the generated derivative. + * + * @throws \Symfony\Component\HttpKernel\Exception\AccessDeniedHttpException + * Thrown when the image style, the scheme, or the path token is invalid. + */ + protected function validateRequest(Request $request, ImageStyleInterface $image_style, $scheme, $target) { + $valid = !empty($image_style) && file_stream_wrapper_valid_scheme($scheme); + $image_uri = $scheme . '://' . $target; + if (!$this->config('image.settings') + ->get('allow_insecure_derivatives') || strpos(ltrim($target, '\/'), 'styles/') === 0 + ) { + $valid &= $request->query->get(IMAGE_DERIVATIVE_TOKEN) === $image_style->getPathToken($image_uri); + } + if (!$valid) { + throw new AccessDeniedHttpException(); + } + } + + /** + * Returns a channel logger object. + * + * @param string $channel + * The name of the channel. Can be any string, but the general practice is + * to use the name of the subsystem calling this. + * + * @return \Psr\Log\LoggerInterface + * The logger for this channel. + */ + protected function getLogger($channel) { + if (!$this->loggerFactory) { + $this->loggerFactory = \Drupal::getContainer()->get('logger.factory'); + } + return $this->loggerFactory->get($channel); + } + +} diff --git a/src/Controller/ImageStyleRedirectController.php b/src/Controller/ImageStyleRedirectController.php new file mode 100644 index 0000000..e03ddf8 --- /dev/null +++ b/src/Controller/ImageStyleRedirectController.php @@ -0,0 +1,255 @@ +fileStorage = $file_storage; + $this->fileSystem = $file_system; + $this->imageStyleCopier = $image_style_copier; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('lock'), + $container->get('image.factory'), + $container->get('entity.manager')->getStorage('file'), + $container->get('file_system'), + $container->get('flysystem_image_style_copier') + ); + } + + /** + * {@inheritdoc} + */ + public function deliver(Request $request, $scheme, ImageStyleInterface $image_style) { + $target = $request->query->get('file'); + $source_uri = $scheme . '://' . $target; + + $this->validateRequest($request, $image_style, $scheme, $target); + + // Don't try to generate file if source is missing. + try { + $source_uri = $this->validateSource($source_uri); + } + catch (FileNotFoundException $e) { + $derivative_uri = $image_style->buildUri($source_uri); + $this->logger->notice('Source image at %source_image_path not found while trying to generate derivative image at %derivative_path.', ['%source_image_path' => $source_uri, '%derivative_path' => $derivative_uri]); + return new Response($this->t('Error generating image, missing source file.'), 404); + } + + // If the image already exists on the adapter, deliver it instead. + try { + return $this->redirectAdapterImage($source_uri, $image_style); + } + catch (FileNotFoundException $e) { + return $this->deliverTemporary($scheme, $target, $image_style); + } + } + + /** + * Generate a temporary image for an image style. + * + * @param string $scheme + * The file scheme, defaults to 'private'. + * @param string $source_path + * The image file to generate the temporary image for. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to generate. + * + * @throws \RuntimeException + * Thrown when generate() failed to generate an image. + * + * @return \Drupal\file\Entity\File + * The temporary image that was generated. + */ + protected function generateTemporaryImage($scheme, $source_path, ImageStyleInterface $image_style) { + $image_uri = "$scheme://$source_path"; + $destination_temp = $image_style->buildUri("temporary://flysystem/$scheme/$source_path"); + + // Try to generate the temporary image, watching for other threads that may + // also be trying to generate the temporary image. + try { + $success = $this->generate($image_style, $image_uri, $destination_temp); + if (!$success) { + throw new \RuntimeException('The temporary image could not be generated'); + } + } + catch (ServiceUnavailableHttpException $e) { + // This exception is only thrown if the lock could not be acquired. + $tries = 0; + + do { + if (file_exists($destination_temp)) { + break; + } + + // The file still doesn't exist. + usleep(250000); + $tries++; + } while ($tries < 4); + + // We waited for more than 1 second for the temporary image to appear. + // Since local image generation should be fast, fail out here to try to + // limit PHP process demands. + if ($tries >= 4) { + throw $e; + } + } + + return $destination_temp; + } + + /** + * Flushes the output buffer and copies the temporary images to the adapter. + */ + protected function flushCopy() { + // We have to call both of these to actually flush the image. + Response::closeOutputBuffers(0, TRUE); + flush(); + $this->imageStyleCopier->processCopyTasks(); + } + + /** + * Redirects to to an adapter hosted image, if it exists. + * + * @param string $source_uri + * The URI to the source image. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to redirect to. + * + * @throws \Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException + * Thrown if the derivative does not exist on the adapter. + * + * @return \Drupal\Core\Routing\TrustedRedirectResponse + * A redirect to the image if it exists. + */ + protected function redirectAdapterImage($source_uri, ImageStyleInterface $image_style) { + $derivative_uri = $image_style->buildUri($source_uri); + + if (file_exists($derivative_uri)) { + // We can't just return TrustedRedirectResponse because core throws an + // exception about missing cache metadata. + // https://www.drupal.org/node/2638686 + // https://www.drupal.org/node/2630808 + // http://drupal.stackexchange.com/questions/187086/trustedresponseredirect-failing-how-to-prevent-cache-metadata + + // @todo Figure out why caching this response leads to stale images being + // served. + + // $url = Url::fromUri($image_style->buildUrl($source_uri))->toString(TRUE); + // $response = new TrustedRedirectResponse($url->getGeneratedUrl()); + // $response->addCacheableDependency($url); + + $response = new TrustedRedirectResponse($image_style->buildUrl($source_uri)); + $response->addCacheableDependency(0); + + return $response; + } + + throw new FileNotFoundException(sprintf('%derivative_uri does not exist', $derivative_uri)); + } + + /** + * Delivers a generate an image, deliver it, and upload it to the adapter. + * + * @param string $scheme + * The scheme of the source image. + * @param string $source_path + * The path of the source image. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to generate. + * + * @return \Symfony\Component\HttpFoundation\BinaryFileResponse|\Symfony\Component\HttpFoundation\Response + * The image response, or an error response if image generation failed. + */ + protected function deliverTemporary($scheme, $source_path, ImageStyleInterface $image_style) { + $source_uri = $scheme . '://' . $source_path; + $derivative_uri = $image_style->buildUri($source_uri); + try { + $temporary_uri = $this->generateTemporaryImage($scheme, $source_path, $image_style); + } + catch (\RuntimeException $e) { + $this->logger->notice('Unable to generate the derived image located at %path.', ['%path' => $derivative_uri]); + return new Response($this->t('Error generating image.'), 500); + } + + // Register a copy task with the kernel terminate handler. + $this->imageStyleCopier->addCopyTask($temporary_uri, $source_uri, $image_style); + + // Symfony's kernel terminate handler is documented to only executes after + // flushing with fastcgi, and not with mod_php or regular CGI. However, + // it appears to work with mod_php. We assume it doesn't and register a + // shutdown handler unless we know we are under fastcgi. If images have + // been previously flushed and uploaded, this call will do nothing. + // + // https://github.com/symfony/symfony-docs/issues/6520 + if (!function_exists('fastcgi_finish_request')) { + drupal_register_shutdown_function(function () { + $this->flushCopy(); + }); + } + + return $this->send($scheme, $temporary_uri); + } + +} diff --git a/src/ImageStyleCopier.php b/src/ImageStyleCopier.php new file mode 100644 index 0000000..5b913df --- /dev/null +++ b/src/ImageStyleCopier.php @@ -0,0 +1,159 @@ +lock = $lock; + $this->fileSystem = $file_system; + $this->logger = $logger; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + return new static( + $container->get('lock'), + $container->get('file_system'), + $container->get('logger.channel.image') + ); + } + + /** + * {@inheritdoc} + */ + public static function getSubscribedEvents() { + $events = []; + $events[KernelEvents::TERMINATE] = 'processCopyTasks'; + + return $events; + } + + /** + * Adds a task to generate and copy an image derivative. + * + * @param string $temporary_uri + * The URI of the temporary image to copy from. + * @param string $source_uri + * The URI of the source image. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style being copied. + */ + public function addCopyTask($temporary_uri, $source_uri, ImageStyleInterface $image_style) { + $this->copyTasks[] = func_get_args(); + } + + /** + * Processes all image copy tasks. + */ + public function processCopyTasks() { + foreach ($this->copyTasks as $task) { + list($temporary_uri, $source_uri, $image_style) = $task; + $this->copyToAdapter($temporary_uri, $source_uri, $image_style); + } + + $this->copyTasks = []; + } + + /** + * Generates an image with the remote stream wrapper. + * + * @param string $temporary_uri + * The temporary file URI to copy to the adapter. + * @param string $source_uri + * The URI of the source image. + * @param \Drupal\image\ImageStyleInterface $image_style + * The image style to generate. + */ + protected function copyToAdapter($temporary_uri, $source_uri, ImageStyleInterface $image_style) { + $derivative_uri = $image_style->buildUri($source_uri); + + // file_unmanaged_copy() doesn't distinguish between a FALSE return due to + // and error or a FALSE return due to an existing file. If we can't acquire + // this lock, we know another thread is uploading the image and we ignore + // uploading it in this thread. + $lock_name = 'flysystem_copy_to_adapter:' . $image_style->id() . ':' . Crypt::hashBase64($source_uri); + + if (!$this->lock->acquire($lock_name)) { + $this->logger->info('Another copy of %image to %destination is in progress', ['%image' => $temporary_uri, '%destination' => $derivative_uri]); + return; + } + + try { + // Get the folder for the final location of this style. + $directory = $this->fileSystem->dirname($derivative_uri); + + // Build the destination folder tree if it doesn't already exist. + if (!file_prepare_directory($directory, FILE_CREATE_DIRECTORY | FILE_MODIFY_PERMISSIONS)) { + $this->logger->error('Failed to create image style directory: %directory', ['%directory' => $directory]); + return; + } + + if (!file_unmanaged_copy($temporary_uri, $derivative_uri, FILE_EXISTS_REPLACE)) { + $this->logger->error('Unable to copy %image to %destination', ['%image' => $temporary_uri, '%directory' => $directory]); + return; + } + + } + finally { + $this->lock->release($lock_name); + } + } + +} diff --git a/src/PathProcessor/FlysystemImageStyleRedirectProcessor.php b/src/PathProcessor/FlysystemImageStyleRedirectProcessor.php new file mode 100644 index 0000000..735256d --- /dev/null +++ b/src/PathProcessor/FlysystemImageStyleRedirectProcessor.php @@ -0,0 +1,47 @@ +query->set('file', $file); + + return static::STYLES_PATH . '/' . $image_style . '/' . $scheme; + } + +} diff --git a/src/Plugin/ImageStyleGenerationTrait.php b/src/Plugin/ImageStyleGenerationTrait.php index fb38520..66f88b1 100644 --- a/src/Plugin/ImageStyleGenerationTrait.php +++ b/src/Plugin/ImageStyleGenerationTrait.php @@ -4,6 +4,7 @@ namespace Drupal\flysystem\Plugin; use Drupal\Component\Utility\Crypt; use Drupal\image\Entity\ImageStyle; +use Symfony\Component\Routing\Generator\UrlGeneratorInterface; /** * Helper trait for generating URLs from adapter plugins. @@ -18,6 +19,10 @@ trait ImageStyleGenerationTrait { * * @return bool * True on success, false on failure. + * + * @deprecated Adapters should use generateImageUrl() to enable non-blocking + * image uploads. Will be removed before Flysystem 8.x-1.0. + * */ protected function generateImageStyle($target) { if (strpos($target, 'styles/') !== 0 || substr_count($target, '/') < 3) { @@ -65,4 +70,24 @@ trait ImageStyleGenerationTrait { return $success; } + /** + * Return the external URL for a generated image. + * + * @param string $target + * The target URI. + * + * @return string + * The generated URL. + */ + protected function generateImageUrl($target) { + list(, $style, $scheme, $file) = explode('/', $target, 4); + $args = [ + 'image_style' => $style, + 'scheme' => $scheme, + 'filepath' => $file, + ]; + + return \Drupal::urlGenerator()->generate('flysystem.image_stye_redirect.serve', $args, UrlGeneratorInterface::ABSOLUTE_URL); + } + } diff --git a/src/Routing/FlysystemRoutes.php b/src/Routing/FlysystemRoutes.php index cfb3272..686e0e4 100644 --- a/src/Routing/FlysystemRoutes.php +++ b/src/Routing/FlysystemRoutes.php @@ -100,7 +100,7 @@ class FlysystemRoutes implements ContainerInjectionInterface { ); if ($this->moduleHandler->moduleExists('image')) { - // Public image route. + // Public image route for alternate public directories. $routes['flysystem.' . $scheme . '.style_public'] = new Route( '/' . $settings['config']['root'] . '/styles/{image_style}/' . $scheme, [ @@ -115,7 +115,7 @@ class FlysystemRoutes implements ContainerInjectionInterface { } if ($this->moduleHandler->moduleExists('image')) { - // Internal image rotue. + // Public image route that proxies the response through Drupal. $routes['flysystem.image_style'] = new Route( '/_flysystem/styles/{image_style}/{scheme}', [ @@ -126,6 +126,31 @@ class FlysystemRoutes implements ContainerInjectionInterface { 'scheme' => '^[a-zA-Z0-9+.-]+$', ] ); + + // Public image route that serves initially from Drupal, and then + // redirects to a remote URL when it's ready. + $routes['flysystem.image_stye_redirect'] = new Route( + "/_flysystem-style-redirect/{image_style}/{scheme}", + [ + '_controller' => 'Drupal\flysystem\Controller\ImageStyleRedirectController::deliver', + ], + [ + '_access' => 'TRUE', + 'scheme' => '^[a-zA-Z0-9+.-]+$', + ] + ); + + $routes['flysystem.image_stye_redirect.serve'] = new Route( + "/_flysystem-style-redirect/{image_style}/{scheme}/{filepath}", + [ + '_controller' => 'Drupal\flysystem\Controller\ImageStyleRedirectController::deliver', + ], + [ + '_access' => 'TRUE', + 'scheme' => '^[a-zA-Z0-9+.-]+$', + 'filepath' => '.+', + ] + ); } return $routes;