diff --git a/src/Context/ContextUploader.php b/src/Context/ContextUploader.php index 4ced03c..94c1034 100644 --- a/src/Context/ContextUploader.php +++ b/src/Context/ContextUploader.php @@ -8,6 +8,7 @@ use Exception; use Psr\Log\LoggerInterface; use Drupal\tmgmt_smartling\Exceptions\SmartlingBaseException; use Smartling\Context\Params\UploadContextParameters; +use Smartling\Context\Params\UploadResourceParameters; use Smartling\Exceptions\SmartlingApiException; class ContextUploader { @@ -88,15 +89,17 @@ class ContextUploader { throw new EmptyContextParameterException('Context url must be a non-empty field.'); } + $smartling_context_directory = $proj_settings['scheme'] . '://tmgmt_smartling_context'; + $smartling_context_file = $smartling_context_directory . '/' . str_replace('.', '_', $filename) . '.html'; + + // Upload context body. try { $html = $this->getContextualizedPage($url, $proj_settings); // Save context file. - $path = $proj_settings['scheme'] . '://tmgmt_smartling_context/' . str_replace('.', '_', $filename) . '.html'; - $dirname = dirname($path); - - if (file_prepare_directory($dirname, FILE_CREATE_DIRECTORY) && ($file = file_save_data($html, $path, FILE_EXISTS_REPLACE))) { + if (file_prepare_directory($smartling_context_directory, FILE_CREATE_DIRECTORY) && ($file = file_save_data($html, $smartling_context_file, FILE_EXISTS_REPLACE))) { $response = $this->uploadContextBody($file, $proj_settings); + $this->uploadContextMissingResources($smartling_context_directory, $proj_settings); if (!empty($response)) { $this->logger->info('Context upload for file @filename completed successfully.', ['@filename' => $filename]); @@ -104,7 +107,7 @@ class ContextUploader { } else { $this->logger->error("Can't save context file: @path", [ - '@path' => $path, + '@path' => $smartling_context_file, ]); } } catch (SmartlingApiException $e) { @@ -144,8 +147,7 @@ class ContextUploader { $params->setName($file->getFileName()); $api = $this->getApi($proj_settings, 'context'); - $response = $api->uploadContext($params); - $response = $api->matchContext($response['contextUid']); + $response = $api->uploadAndMatchContext($params); } catch (Exception $e) { $response = []; watchdog_exception('tmgmt_smartling', $e); @@ -155,6 +157,115 @@ class ContextUploader { } /** + * @param $smartling_context_directory + * @param $proj_settings + */ + protected function uploadContextMissingResources($smartling_context_directory, $proj_settings) { + // Cache for resources which we can't upload. Do not try to re-upload them + // for 1 hour. After 1 hour cache will be reset and we will try again. + $cache_name = 'smartling_context_resources_cache'; + $time_to_live = 60 * 60; + $cache = \Drupal::cache()->get($cache_name); + $cached_data = empty($cache) ? [] : $cache->data; + $update_cache = FALSE; + $smartling_context_resources_directory = $smartling_context_directory . '/resources'; + + // Do nothing if directory for resources isn't accessible. + if (!file_prepare_directory($smartling_context_resources_directory, FILE_CREATE_DIRECTORY)) { + $this->logger->error("Context resources directory @dir doesn't exist or is not writable. Missing resources were not uploaded. Context might look incomplete.", [ + '@dir' => $smartling_context_directory, + ]); + + return; + } + + try { + global $base_url; + + $api = $this->getApi($proj_settings, 'context'); + $all_missing_resources = $api->getAllMissingResources(); + + // Method getAllMissingResources can return not all missing resources + // in case it took to much time. Log this information. + if (!$all_missing_resources['all']) { + $this->logger->warning('Not all missing context resources are received. Context might look incomplete.'); + } + + // Walk through missing resources and try to upload them. + foreach ($all_missing_resources['items'] as $item) { + // Handle situation when we have a resource with relative path. + if (strpos($item['url'], 'http://') === FALSE && + strpos($item['url'], 'https://') === FALSE + ) { + $item['url'] = $base_url . '/' . ltrim($item['url'], '/'); + } + + // If current resource isn't in the cache and it's accessible then + // it means we can try to upload it. + if (!in_array($item['resourceId'], $cached_data) && $this->assetInliner->remote_file_exists($item['url'], $proj_settings)) { + $smartling_context_resource_file = $smartling_context_resources_directory . '/' . $item['resourceId']; + $smartling_context_resource_file_content = $this->assetInliner->getUrlContents($item['url'], + 0, + 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10', + $proj_settings + ); + + // Ensure that resources directory is accessible, resource + // downloaded properly and only then upload it. ContextAPI will not + // be able to fopen() resource which is behind basic auth. So + // download it first (with a help of curl), save it to smartling's + // directory and then upload. + if ($file = file_save_data($smartling_context_resource_file_content, $smartling_context_resource_file, FILE_EXISTS_REPLACE)) { + $stream_wrapper_manager = \Drupal::service('stream_wrapper_manager')->getViaUri($file->getFileUri()); + $params = new UploadResourceParameters(); + $params->setFile($stream_wrapper_manager->realpath()); + $is_resource_uploaded = $api->uploadResource($item['resourceId'], $params); + + // Resource isn't uploaded for some reason. Log this info and set + // resource id into the cache. We will not try to upload this + // resource for the next hour. + if (!$is_resource_uploaded) { + $update_cache = TRUE; + $cached_data[] = $item['resourceId']; + + $this->logger->warning("Can't upload context resource file with id = @id and url = @url. Context might look incomplete.", [ + '@id' => $item['resourceId'], + '@url' => $item['url'], + ]); + } + } + // We can't save context resource file. Log this info. + else { + $this->logger->error("Can't save context resource file: @path", [ + '@path' => $smartling_context_resource_file, + ]); + } + } + else { + // Current resource isn't accessible (or already in the cache). + // If first case then add inaccessible resource into the cache. + if (!in_array($item['resourceId'], $cached_data)) { + $update_cache = TRUE; + $cached_data[] = $item['resourceId']; + + $this->logger->warning("File @file can not be downloaded.", [ + '@file' => $item['url'], + ]); + } + } + } + + // Set failed resources into the cache for the next hour. + if ($update_cache) { + \Drupal::cache()->set($cache_name, $cached_data, time() + $time_to_live); + } + } + catch (Exception $e) { + watchdog_exception('tmgmt_smartling', $e); + } + } + + /** * @param $filename * @return bool */ diff --git a/src/Context/HtmlAssetInliner.php b/src/Context/HtmlAssetInliner.php index 1e5d662..2c049d7 100644 --- a/src/Context/HtmlAssetInliner.php +++ b/src/Context/HtmlAssetInliner.php @@ -170,7 +170,7 @@ class HtmlAssetInliner { * * @return bool */ - private function remote_file_exists($url) { + public function remote_file_exists($url, $proj_settings) { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); # don't download content @@ -178,6 +178,8 @@ class HtmlAssetInliner { curl_setopt($ch, CURLOPT_FAILONERROR, 1); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + $this->applySettingsToCurl($proj_settings, $ch); + if (curl_exec($ch) !== FALSE) { return TRUE; } @@ -318,7 +320,7 @@ class HtmlAssetInliner { * * @return int|mixed */ - private function getUrlContents( + public function getUrlContents( $url, $timeout = 0, $userAgent = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10', @@ -335,14 +337,7 @@ class HtmlAssetInliner { curl_setopt($crl, CURLOPT_HEADER, 1); } - if (!empty($settings['context_skip_host_verifying'])) { - curl_setopt($crl, CURLOPT_SSL_VERIFYHOST, FALSE); - } - - if (!empty($settings['enable_basic_auth'])) { - curl_setopt($crl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - curl_setopt($crl, CURLOPT_USERPWD, $settings['basic_auth']['login'] . ':' . $settings['basic_auth']['password']); - } + $this->applySettingsToCurl($settings, $crl); curl_setopt($crl, CURLOPT_URL, $url); curl_setopt($crl, CURLOPT_RETURNTRANSFER, 1); # return result as string rather than direct output @@ -382,6 +377,21 @@ class HtmlAssetInliner { } /** + * @param $proj_settings + * @param $curl + */ + private function applySettingsToCurl($proj_settings, $curl) { + if (!empty($proj_settings['context_skip_host_verifying'])) { + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, FALSE); + } + + if (!empty($proj_settings['enable_basic_auth'])) { + curl_setopt($curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_setopt($curl, CURLOPT_USERPWD, $proj_settings['basic_auth']['login'] . ':' . $proj_settings['basic_auth']['password']); + } + } + + /** * Converts relative URLs to absolute URLs * * @param $url